From 705ba09e76d0bb4af88b7b7cde1f90c64c9e3753 Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Mon, 21 Feb 2022 12:36:29 -0600 Subject: Fix cache sanity checking code by moving functions to wider scope (#1435) --- src/H5C.c | 177 ++++++++++++++++++++++++------------------------------- src/H5Cprivate.h | 10 ++++ 2 files changed, 88 insertions(+), 99 deletions(-) diff --git a/src/H5C.c b/src/H5C.c index 22df59c..3868204 100644 --- a/src/H5C.c +++ b/src/H5C.c @@ -138,16 +138,6 @@ static herr_t H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t static herr_t H5C__verify_len_eoa(H5F_t *f, const H5C_class_t *type, haddr_t addr, size_t *len, hbool_t actual); -#if H5C_DO_SLIST_SANITY_CHECKS -static hbool_t H5C__entry_in_skip_list(H5C_t *cache_ptr, H5C_cache_entry_t *target_ptr); -#endif /* H5C_DO_SLIST_SANITY_CHECKS */ - -#if H5C_DO_EXTREME_SANITY_CHECKS -static herr_t H5C__validate_lru_list(H5C_t *cache_ptr); -static herr_t H5C__validate_pinned_entry_list(H5C_t *cache_ptr); -static herr_t H5C__validate_protected_entry_list(H5C_t *cache_ptr); -#endif /* H5C_DO_EXTREME_SANITY_CHECKS */ - #ifndef NDEBUG static void H5C__assert_flush_dep_nocycle(const H5C_cache_entry_t *entry, const H5C_cache_entry_t *base_entry); @@ -996,7 +986,7 @@ H5C_expunge_entry(H5F_t *f, const H5C_class_t *type, haddr_t addr, unsigned flag HDassert(H5F_addr_defined(addr)); #if H5C_DO_EXTREME_SANITY_CHECKS - if (H5C__validate_lru_list(cache_ptr) < 0) + if (H5C_validate_lru_list(cache_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "LRU extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -1031,7 +1021,7 @@ H5C_expunge_entry(H5F_t *f, const H5C_class_t *type, haddr_t addr, unsigned flag done: #if H5C_DO_EXTREME_SANITY_CHECKS - if (H5C__validate_lru_list(cache_ptr) < 0) + if (H5C_validate_lru_list(cache_ptr) < 0) HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "LRU extreme sanity check failed on exit") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -1138,8 +1128,8 @@ H5C_flush_cache(H5F_t *f, unsigned flags) #endif /* H5C_DO_SANITY_CHECKS */ #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -1314,8 +1304,8 @@ H5C_insert_entry(H5F_t *f, const H5C_class_t *type, haddr_t addr, void *thing, u #if H5C_DO_EXTREME_SANITY_CHECKS /* no need to verify that entry is not already in the index as */ /* we already make that check below. */ - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -1503,8 +1493,8 @@ H5C_insert_entry(H5F_t *f, const H5C_class_t *type, haddr_t addr, void *thing, u H5C__UPDATE_RP_FOR_INSERTION(cache_ptr, entry_ptr, FAIL) #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed just before done") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -1542,8 +1532,8 @@ H5C_insert_entry(H5F_t *f, const H5C_class_t *type, haddr_t addr, void *thing, u done: #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on exit") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -1867,8 +1857,8 @@ H5C_move_entry(H5C_t *cache_ptr, const H5C_class_t *type, haddr_t old_addr, hadd HDassert(H5F_addr_ne(old_addr, new_addr)); #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -1973,8 +1963,8 @@ H5C_move_entry(H5C_t *cache_ptr, const H5C_class_t *type, haddr_t old_addr, hadd done: #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on exit") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -2020,8 +2010,7 @@ H5C_resize_entry(void *thing, size_t new_size) HGOTO_ERROR(H5E_CACHE, H5E_BADTYPE, FAIL, "Entry isn't pinned or protected??") #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || (H5C_validate_pinned_entry_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -2117,8 +2106,7 @@ H5C_resize_entry(void *thing, size_t new_size) done: #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || (H5C_validate_pinned_entry_list(cache_ptr) < 0)) HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on exit") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -2158,8 +2146,8 @@ H5C_pin_protected_entry(void *thing) HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -2173,8 +2161,8 @@ H5C_pin_protected_entry(void *thing) done: #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on exit") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -2237,8 +2225,8 @@ H5C_protect(H5F_t *f, const H5C_class_t *type, haddr_t addr, void *udata, unsign HDassert(H5F_addr_defined(addr)); #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, NULL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -2625,8 +2613,8 @@ H5C_protect(H5F_t *f, const H5C_class_t *type, haddr_t addr, void *udata, unsign done: #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, NULL, "an extreme sanity check failed on exit") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -3099,8 +3087,8 @@ H5C_unpin_entry(void *_entry_ptr) HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -3110,8 +3098,8 @@ H5C_unpin_entry(void *_entry_ptr) done: #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on exit") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -3278,8 +3266,8 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags) was_clean = !(entry_ptr->is_dirty); #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -3545,8 +3533,8 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags) done: #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on exit") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -6080,8 +6068,8 @@ H5C__flush_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) HDassert(ring < H5C_RING_NTYPES); #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ @@ -7756,7 +7744,7 @@ done: /*------------------------------------------------------------------------- * - * Function: H5C__validate_lru_list + * Function: H5C_validate_lru_list * * Purpose: Debugging function that scans the LRU list for errors. * @@ -7771,15 +7759,15 @@ done: *------------------------------------------------------------------------- */ #if H5C_DO_EXTREME_SANITY_CHECKS -static herr_t -H5C__validate_lru_list(H5C_t *cache_ptr) +herr_t +H5C_validate_lru_list(H5C_t *cache_ptr) { int32_t len = 0; size_t size = 0; H5C_cache_entry_t *entry_ptr = NULL; herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC + FUNC_ENTER_NOAPI(FAIL) HDassert(cache_ptr); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); @@ -7788,51 +7776,48 @@ H5C__validate_lru_list(H5C_t *cache_ptr) (cache_ptr->LRU_head_ptr != cache_ptr->LRU_tail_ptr)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 1 failed") - if (cache_ptr->LRU_list_len < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 2 failed") - if ((cache_ptr->LRU_list_len == 1) && ((cache_ptr->LRU_head_ptr != cache_ptr->LRU_tail_ptr) || (cache_ptr->LRU_head_ptr == NULL) || (cache_ptr->LRU_head_ptr->size != cache_ptr->LRU_list_size))) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 3 failed") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 2 failed") if ((cache_ptr->LRU_list_len >= 1) && ((cache_ptr->LRU_head_ptr == NULL) || (cache_ptr->LRU_head_ptr->prev != NULL) || (cache_ptr->LRU_tail_ptr == NULL) || (cache_ptr->LRU_tail_ptr->next != NULL))) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 4 failed") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 3 failed") entry_ptr = cache_ptr->LRU_head_ptr; while (entry_ptr != NULL) { if ((entry_ptr != cache_ptr->LRU_head_ptr) && ((entry_ptr->prev == NULL) || (entry_ptr->prev->next != entry_ptr))) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 5 failed") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 4 failed") if ((entry_ptr != cache_ptr->LRU_tail_ptr) && ((entry_ptr->next == NULL) || (entry_ptr->next->prev != entry_ptr))) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 6 failed") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 5 failed") if ((entry_ptr->is_pinned) || (entry_ptr->pinned_from_client) || (entry_ptr->pinned_from_cache)) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 7 failed") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 6 failed") len++; size += entry_ptr->size; entry_ptr = entry_ptr->next; } - if ((cache_ptr->LRU_list_len != len) || (cache_ptr->LRU_list_size != size)) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 8 failed") + if ((cache_ptr->LRU_list_len != (uint32_t)len) || (cache_ptr->LRU_list_size != size)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 7 failed") done: if (ret_value != SUCCEED) HDassert(0); FUNC_LEAVE_NOAPI(ret_value) -} /* H5C__validate_lru_list() */ +} /* H5C_validate_lru_list() */ #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ /*------------------------------------------------------------------------- * - * Function: H5C__validate_pinned_entry_list + * Function: H5C_validate_pinned_entry_list * * Purpose: Debugging function that scans the pinned entry list for * errors. @@ -7848,15 +7833,15 @@ done: *------------------------------------------------------------------------- */ #if H5C_DO_EXTREME_SANITY_CHECKS -static herr_t -H5C__validate_pinned_entry_list(H5C_t *cache_ptr) +herr_t +H5C_validate_pinned_entry_list(H5C_t *cache_ptr) { int32_t len = 0; size_t size = 0; H5C_cache_entry_t *entry_ptr = NULL; herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC + FUNC_ENTER_NOAPI(FAIL) HDassert(cache_ptr); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); @@ -7865,54 +7850,51 @@ H5C__validate_pinned_entry_list(H5C_t *cache_ptr) (cache_ptr->pel_head_ptr != cache_ptr->pel_tail_ptr)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 1 failed") - if (cache_ptr->pel_len < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 2 failed") - if ((cache_ptr->pel_len == 1) && ((cache_ptr->pel_head_ptr != cache_ptr->pel_tail_ptr) || (cache_ptr->pel_head_ptr == NULL) || (cache_ptr->pel_head_ptr->size != cache_ptr->pel_size))) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 3 failed") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 2 failed") if ((cache_ptr->pel_len >= 1) && ((cache_ptr->pel_head_ptr == NULL) || (cache_ptr->pel_head_ptr->prev != NULL) || (cache_ptr->pel_tail_ptr == NULL) || (cache_ptr->pel_tail_ptr->next != NULL))) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 4 failed") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 3 failed") entry_ptr = cache_ptr->pel_head_ptr; while (entry_ptr != NULL) { if ((entry_ptr != cache_ptr->pel_head_ptr) && ((entry_ptr->prev == NULL) || (entry_ptr->prev->next != entry_ptr))) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 5 failed") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 4 failed") if ((entry_ptr != cache_ptr->pel_tail_ptr) && ((entry_ptr->next == NULL) || (entry_ptr->next->prev != entry_ptr))) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 6 failed") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 5 failed") if (!entry_ptr->is_pinned) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 7 failed") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 6 failed") if (!(entry_ptr->pinned_from_client || entry_ptr->pinned_from_cache)) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 8 failed") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 7 failed") len++; size += entry_ptr->size; entry_ptr = entry_ptr->next; } - if ((cache_ptr->pel_len != len) || (cache_ptr->pel_size != size)) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 9 failed") + if ((cache_ptr->pel_len != (uint32_t)len) || (cache_ptr->pel_size != size)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 8 failed") done: if (ret_value != SUCCEED) HDassert(0); FUNC_LEAVE_NOAPI(ret_value) -} /* H5C__validate_pinned_entry_list() */ +} /* H5C_validate_pinned_entry_list() */ #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ /*------------------------------------------------------------------------- * - * Function: H5C__validate_protected_entry_list + * Function: H5C_validate_protected_entry_list * * Purpose: Debugging function that scans the protected entry list for * errors. @@ -7928,15 +7910,15 @@ done: *------------------------------------------------------------------------- */ #if H5C_DO_EXTREME_SANITY_CHECKS -static herr_t -H5C__validate_protected_entry_list(H5C_t *cache_ptr) +herr_t +H5C_validate_protected_entry_list(H5C_t *cache_ptr) { int32_t len = 0; size_t size = 0; H5C_cache_entry_t *entry_ptr = NULL; herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC + FUNC_ENTER_NOAPI(FAIL) HDassert(cache_ptr); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); @@ -7945,54 +7927,51 @@ H5C__validate_protected_entry_list(H5C_t *cache_ptr) (cache_ptr->pl_head_ptr != cache_ptr->pl_tail_ptr)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 1 failed") - if (cache_ptr->pl_len < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 2 failed") - if ((cache_ptr->pl_len == 1) && ((cache_ptr->pl_head_ptr != cache_ptr->pl_tail_ptr) || (cache_ptr->pl_head_ptr == NULL) || (cache_ptr->pl_head_ptr->size != cache_ptr->pl_size))) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 3 failed") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 2 failed") if ((cache_ptr->pl_len >= 1) && ((cache_ptr->pl_head_ptr == NULL) || (cache_ptr->pl_head_ptr->prev != NULL) || (cache_ptr->pl_tail_ptr == NULL) || (cache_ptr->pl_tail_ptr->next != NULL))) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 4 failed") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 3 failed") entry_ptr = cache_ptr->pl_head_ptr; while (entry_ptr != NULL) { if ((entry_ptr != cache_ptr->pl_head_ptr) && ((entry_ptr->prev == NULL) || (entry_ptr->prev->next != entry_ptr))) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 5 failed") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 4 failed") if ((entry_ptr != cache_ptr->pl_tail_ptr) && ((entry_ptr->next == NULL) || (entry_ptr->next->prev != entry_ptr))) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 6 failed") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 5 failed") if (!entry_ptr->is_protected) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 7 failed") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 6 failed") if (entry_ptr->is_read_only && (entry_ptr->ro_ref_count <= 0)) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 8 failed") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 7 failed") len++; size += entry_ptr->size; entry_ptr = entry_ptr->next; } - if ((cache_ptr->pl_len != len) || (cache_ptr->pl_size != size)) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 9 failed") + if ((cache_ptr->pl_len != (uint32_t)len) || (cache_ptr->pl_size != size)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Check 8 failed") done: if (ret_value != SUCCEED) HDassert(0); FUNC_LEAVE_NOAPI(ret_value) -} /* H5C__validate_protected_entry_list() */ +} /* H5C_validate_protected_entry_list() */ #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ /*------------------------------------------------------------------------- * - * Function: H5C__entry_in_skip_list + * Function: H5C_entry_in_skip_list * * Purpose: Debugging function that scans skip list to see if it * is in present. We need this, as it is possible for @@ -8006,8 +7985,8 @@ done: *------------------------------------------------------------------------- */ #if H5C_DO_SLIST_SANITY_CHECKS -static hbool_t -H5C__entry_in_skip_list(H5C_t *cache_ptr, H5C_cache_entry_t *target_ptr) +hbool_t +H5C_entry_in_skip_list(H5C_t *cache_ptr, H5C_cache_entry_t *target_ptr) { H5SL_node_t *node_ptr; hbool_t in_slist; @@ -8035,7 +8014,7 @@ H5C__entry_in_skip_list(H5C_t *cache_ptr, H5C_cache_entry_t *target_ptr) } return (in_slist); -} /* H5C__entry_in_skip_list() */ +} /* H5C_entry_in_skip_list() */ #endif /* H5C_DO_SLIST_SANITY_CHECKS */ /*------------------------------------------------------------------------- @@ -8525,8 +8504,8 @@ H5C__serialize_cache(H5F_t *f) #endif /* H5C_DO_SANITY_CHECKS */ #if H5C_DO_EXTREME_SANITY_CHECKS - if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || - (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + if ((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || (H5C_validate_lru_list(cache_ptr) < 0)) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index 8a1043e..9514443 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -2292,6 +2292,16 @@ H5_DLL herr_t H5C_cache_image_status(H5F_t *f, hbool_t *load_ci_ptr, hbool_t * H5_DLL hbool_t H5C_cache_image_pending(const H5C_t *cache_ptr); H5_DLL herr_t H5C_get_mdc_image_info(const H5C_t *cache_ptr, haddr_t *image_addr, hsize_t *image_len); +#if H5C_DO_SLIST_SANITY_CHECKS +H5_DLL hbool_t H5C_entry_in_skip_list(H5C_t *cache_ptr, H5C_cache_entry_t *target_ptr); +#endif + +#if H5C_DO_EXTREME_SANITY_CHECKS +H5_DLL herr_t H5C_validate_lru_list(H5C_t *cache_ptr); +H5_DLL herr_t H5C_validate_pinned_entry_list(H5C_t *cache_ptr); +H5_DLL herr_t H5C_validate_protected_entry_list(H5C_t *cache_ptr); +#endif /* H5C_DO_EXTREME_SANITY_CHECKS */ + /* Logging functions */ H5_DLL herr_t H5C_start_logging(H5C_t *cache); H5_DLL herr_t H5C_stop_logging(H5C_t *cache); -- cgit v0.12 From c302773438a4db0b56d32eaf9cf8a92b56848c0a Mon Sep 17 00:00:00 2001 From: David Young Date: Mon, 21 Feb 2022 15:31:57 -0600 Subject: Sprinkle H5_ATTR_FORMAT over printf(3)-like functions in tools and fix issues (#1423) * Correct some conversion specifications. * Replace many "%lld" occurrences with "%" PRIuHSIZE except for a few instances where PRIdHSIZE was appropriate. Remove a couple of casts and use correct format strings, instead. * Copy values from a possibly unaligned buffer to aligned local variables instead of casting and dereferencing pointers into the buffer. * Committing clang-format changes Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- tools/lib/h5diff_array.c | 94 ++++++++++++++------------- tools/lib/h5diff_util.c | 4 +- tools/lib/h5tools.h | 3 - tools/lib/h5tools_dump.c | 55 ++++++++-------- tools/lib/h5tools_str.c | 23 ++++--- tools/lib/h5tools_str.h | 2 +- tools/lib/h5tools_utils.h | 6 +- tools/lib/h5trav.c | 7 +- tools/src/h5dump/h5dump_xml.c | 33 +++++----- tools/src/h5format_convert/h5format_convert.c | 2 +- tools/src/h5ls/h5ls.c | 50 +++++++------- tools/src/h5perf/pio_perf.c | 2 +- tools/src/h5repack/h5repack_main.c | 4 +- 13 files changed, 139 insertions(+), 146 deletions(-) diff --git a/tools/lib/h5diff_array.c b/tools/lib/h5diff_array.c index e8cfacb..a7156b5 100644 --- a/tools/lib/h5diff_array.c +++ b/tools/lib/h5diff_array.c @@ -204,7 +204,8 @@ diff_array(void *_mem1, void *_mem2, diff_opt_t *opts, hid_t container1_id, hid_ mcomp_t members; H5T_class_t type_class; - H5TOOLS_START_DEBUG(" - rank:%d hs_nelmts:%lld errstat:%d", opts->rank, opts->hs_nelmts, opts->err_stat); + H5TOOLS_START_DEBUG(" - rank:%d hs_nelmts:%" PRIuHSIZE " errstat:%d", opts->rank, opts->hs_nelmts, + opts->err_stat); opts->print_header = 1; /* enable print header */ /* get the size. */ @@ -388,7 +389,8 @@ diff_array(void *_mem1, void *_mem2, diff_opt_t *opts, hid_t container1_id, hid_ HDmemset(&members, 0, sizeof(mcomp_t)); get_member_types(opts->m_tid, &members); for (i = 0; i < opts->hs_nelmts; i++) { - H5TOOLS_DEBUG("opts->pos[%lld]:%lld - nelmts:%lld", i, opts->pos[i], opts->hs_nelmts); + H5TOOLS_DEBUG("opts->pos[%" PRIuHSIZE "]:%" PRIuHSIZE " - nelmts:%" PRIuHSIZE, i, + opts->pos[i], opts->hs_nelmts); nfound += diff_datum(mem1 + i * size, mem2 + i * size, i, opts, container1_id, container2_id, &members); if (opts->count_bool && nfound >= opts->count) @@ -396,7 +398,7 @@ diff_array(void *_mem1, void *_mem2, diff_opt_t *opts, hid_t container1_id, hid_ } /* i */ close_member_types(&members); } /* switch */ - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -452,7 +454,8 @@ diff_datum(void *_mem1, void *_mem2, hsize_t elemtno, diff_opt_t *opts, hid_t co hsize_t nfound = 0; /* differences found */ diff_err_t ret_value = opts->err_stat; - H5TOOLS_START_DEBUG("ph:%d elemtno:%lld - errstat:%d", opts->print_header, elemtno, opts->err_stat); + H5TOOLS_START_DEBUG("ph:%d elemtno:%" PRIuHSIZE " - errstat:%d", opts->print_header, elemtno, + opts->err_stat); type_size = H5Tget_size(opts->m_tid); type_class = H5Tget_class(opts->m_tid); @@ -702,8 +705,8 @@ diff_datum(void *_mem1, void *_mem2, hsize_t elemtno, diff_opt_t *opts, hid_t co H5TOOLS_DEBUG("H5T_ARRAY ph=%d", opts->print_header); arr_opts = *opts; - H5TOOLS_DEBUG("Check opts: hs_nelmts:%lld to %lld rank:%d to %d", opts->hs_nelmts, - arr_opts.hs_nelmts, opts->rank, arr_opts.rank); + H5TOOLS_DEBUG("Check opts: hs_nelmts:%" PRIuHSIZE " to %" PRIuHSIZE " rank:%d to %d", + opts->hs_nelmts, arr_opts.hs_nelmts, opts->rank, arr_opts.rank); /* get the array's base datatype for each element */ arr_opts.m_tid = H5Tget_super(opts->m_tid); size = H5Tget_size(arr_opts.m_tid); @@ -1051,8 +1054,8 @@ diff_datum(void *_mem1, void *_mem2, hsize_t elemtno, diff_opt_t *opts, hid_t co H5TOOLS_INFO("H5Rdestroy H5R_OBJECT1 failed"); if (H5Rdestroy(ref1_buf) < 0) H5TOOLS_INFO("H5Rdestroy H5R_OBJECT1 failed"); - H5TOOLS_DEBUG("H5T_REFERENCE - H5T_STD_REF complete nfound:%lld - errstat:%d", nfound, - ref_opts.err_stat); + H5TOOLS_DEBUG("H5T_REFERENCE - H5T_STD_REF complete nfound:%" PRIuHSIZE " - errstat:%d", + nfound, ref_opts.err_stat); } /*------------------------------------------------------------------------- * H5T_STD_REF_DSETREG @@ -1263,7 +1266,7 @@ diff_datum(void *_mem1, void *_mem2, hsize_t elemtno, diff_opt_t *opts, hid_t co done: opts->err_stat = opts->err_stat | ret_value; - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -1374,8 +1377,8 @@ diff_region(hid_t obj1_id, hid_t obj2_id, hid_t region1_id, hid_t region2_id, di npoints2 = H5Sget_select_elem_npoints(region2_id); } H5E_END_TRY; - H5TOOLS_DEBUG("blocks: 1=%lld-2=%lld", nblocks1, nblocks2); - H5TOOLS_DEBUG("points: 1=%lld-2=%lld", npoints1, npoints2); + H5TOOLS_DEBUG("blocks: 1=%" PRIdHSIZE "-2=%" PRIdHSIZE, nblocks1, nblocks2); + H5TOOLS_DEBUG("points: 1=%" PRIdHSIZE "-2=%" PRIdHSIZE, npoints1, npoints2); if (nblocks1 != nblocks2 || npoints1 != npoints2 || ndims1 != ndims2) { opts->not_cmp = 1; @@ -1518,10 +1521,10 @@ diff_region(hid_t obj1_id, hid_t obj2_id, hid_t region1_id, hid_t region2_id, di #if defined(H5DIFF_DEBUG) for (i = 0; i < npoints1; i++) { - parallel_print("%sPt%lu: ", i ? "," : "", (unsigned long)i); + parallel_print("%sPt%d: ", i ? "," : "", i); for (j = 0; j < ndims1; j++) - parallel_print("%s%lu", j ? "," : "(", (unsigned long)(ptdata1[i * ndims1 + j])); + parallel_print("%s%" PRIuHSIZE, j ? "," : "(", ptdata1[i * ndims1 + j]); parallel_print(")"); } @@ -1538,7 +1541,7 @@ diff_region(hid_t obj1_id, hid_t obj2_id, hid_t region1_id, hid_t region2_id, di ret_value = nfound_p + nfound_b; done: - H5TOOLS_ENDDEBUG(" with diffs:%lld", ret_value); + H5TOOLS_ENDDEBUG(" with diffs:%" PRIuHSIZE, ret_value); return ret_value; } @@ -1575,7 +1578,7 @@ character_compare(char *mem1, char *mem2, hsize_t elemtno, size_t u, diff_opt_t } nfound++; } - H5TOOLS_ENDDEBUG(": %lld", nfound); + H5TOOLS_ENDDEBUG(": %" PRIuHSIZE, nfound); return nfound; } @@ -1649,7 +1652,7 @@ character_compare_opt(unsigned char *mem1, unsigned char *mem2, hsize_t elemtno, nfound++; } - H5TOOLS_ENDDEBUG(": %lld zero:%d", nfound, both_zero); + H5TOOLS_ENDDEBUG(": %" PRIuHSIZE " zero:%d", nfound, both_zero); return nfound; } @@ -1830,7 +1833,7 @@ diff_float_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, d } } - H5TOOLS_ENDDEBUG(": %lld zero:%d", nfound, both_zero); + H5TOOLS_ENDDEBUG(": %" PRIuHSIZE " zero:%d", nfound, both_zero); return nfound; } @@ -2001,7 +2004,7 @@ diff_double_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, nfound++; } } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2175,7 +2178,7 @@ diff_ldouble_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, nfound++; } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2263,7 +2266,7 @@ diff_schar_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, d nfound++; } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2351,7 +2354,7 @@ diff_uchar_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, d nfound++; } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2439,7 +2442,7 @@ diff_short_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, d nfound++; } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2531,7 +2534,7 @@ diff_ushort_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, nfound++; } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2619,7 +2622,7 @@ diff_int_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, dif nfound++; } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2707,7 +2710,7 @@ diff_uint_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, di nfound++; } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2795,7 +2798,7 @@ diff_long_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, di nfound++; } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2885,7 +2888,7 @@ diff_ulong_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, d nfound++; } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -2978,7 +2981,7 @@ diff_llong_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, d } } - H5TOOLS_ENDDEBUG(":%lld - errstat:%d", nfound, opts->err_stat); + H5TOOLS_ENDDEBUG(":%" PRIuHSIZE " - errstat:%d", nfound, opts->err_stat); return nfound; } @@ -3078,7 +3081,7 @@ diff_ullong_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, } } - H5TOOLS_ENDDEBUG(": %lld zero:%d", nfound, both_zero); + H5TOOLS_ENDDEBUG(": %" PRIuHSIZE " zero:%d", nfound, both_zero); return nfound; } @@ -3323,7 +3326,7 @@ print_header(diff_opt_t *opts) static void print_pos(diff_opt_t *opts, hsize_t idx, size_t u) { - H5TOOLS_START_DEBUG(" -- idx:%lld", idx); + H5TOOLS_START_DEBUG(" -- idx:%" PRIuHSIZE, idx); if (print_data(opts)) { hsize_t curr_pos = idx; @@ -3336,8 +3339,8 @@ print_pos(diff_opt_t *opts, hsize_t idx, size_t u) H5TOOLS_DEBUG("rank=%d", opts->rank); if (opts->rank > 0) { parallel_print("[ "); - H5TOOLS_DEBUG("do calc_acc_pos[%lld] nelmts:%lld - errstat:%d", idx, opts->hs_nelmts, - opts->err_stat); + H5TOOLS_DEBUG("do calc_acc_pos[%" PRIuHSIZE "] nelmts:%" PRIuHSIZE " - errstat:%d", idx, + opts->hs_nelmts, opts->err_stat); if (opts->sset[0] != NULL) { /* Subsetting is used - calculate total position */ hsize_t curr_idx = 0; /* current pos in the selection space for each dimension */ @@ -3364,33 +3367,36 @@ print_pos(diff_opt_t *opts, hsize_t idx, size_t u) j = opts->rank - i - 1; prev_total_dim_size *= prev_dim_size; dim_size = opts->dims[j]; - H5TOOLS_DEBUG("j=%d, dim_size=%lld, prev_dim_size=%lld, total_dim_size=%lld, " - "prev_total_dim_size=%lld", + H5TOOLS_DEBUG("j=%d, dim_size=%" PRIuHSIZE ", prev_dim_size=%" PRIuHSIZE + ", total_dim_size=%" PRIuHSIZE ", " + "prev_total_dim_size=%" PRIuHSIZE, j, dim_size, prev_dim_size, total_dim_size, prev_total_dim_size); count = opts->sset[0]->count.data[j]; block = opts->sset[0]->block.data[j]; stride = opts->sset[0]->stride.data[j]; - H5TOOLS_DEBUG("stride=%lld, count=%lld, block=%lld", stride, count, block); + H5TOOLS_DEBUG("stride=%" PRIuHSIZE ", count=%" PRIuHSIZE ", block=%" PRIuHSIZE, + stride, count, block); tmp = count * block; k0 = curr_idx / tmp; k1 = curr_idx % tmp; curr_pos += k1 * stride * prev_total_dim_size; - H5TOOLS_DEBUG("curr_idx=%lld, k0=%lld, k1=%lld, curr_pos=%lld", curr_idx, k0, k1, - curr_pos); + H5TOOLS_DEBUG("curr_idx=%" PRIuHSIZE ", k0=%" PRIuHSIZE ", k1=%" PRIuHSIZE + ", curr_pos=%" PRIuHSIZE, + curr_idx, k0, k1, curr_pos); if (k0 > 0) curr_idx = k0 * total_dim_size; - H5TOOLS_DEBUG("curr_idx=%lld, tmp=%lld", curr_idx, tmp); + H5TOOLS_DEBUG("curr_idx=%" PRIuHSIZE ", tmp=%" PRIuHSIZE, curr_idx, tmp); total_dim_size *= dim_size; /* if last calculation exists within in current dimension */ if (k0 == 0) break; - H5TOOLS_DEBUG("j=%d, curr_pos=%lld", j, curr_pos); + H5TOOLS_DEBUG("j=%d, curr_pos=%" PRIuHSIZE, j, curr_pos); prev_dim_size = dim_size; } /* check if there is a final calculation needed for slowest changing dimension */ if (k0 > 0) curr_pos += k0 * stride * prev_total_dim_size; - H5TOOLS_DEBUG("4:curr_idx=%lld, curr_pos=%lld", curr_idx, curr_pos); + H5TOOLS_DEBUG("4:curr_idx=%" PRIuHSIZE ", curr_pos=%" PRIuHSIZE, curr_idx, curr_pos); } } /* @@ -3400,11 +3406,11 @@ print_pos(diff_opt_t *opts, hsize_t idx, size_t u) calc_acc_pos((unsigned)opts->rank, curr_pos, opts->acc, opts->pos); for (int i = 0; i < opts->rank; i++) { - H5TOOLS_DEBUG("pos loop:%d with opts->pos=%lld opts->sm_pos=%lld", i, opts->pos[i], - opts->sm_pos[i]); + H5TOOLS_DEBUG("pos loop:%d with opts->pos=%" PRIuHSIZE " opts->sm_pos=%" PRIuHSIZE, i, + opts->pos[i], opts->sm_pos[i]); opts->pos[i] += (unsigned long)opts->sm_pos[i]; - H5TOOLS_DEBUG("pos loop:%d with opts->pos=%lld", i, opts->pos[i]); - parallel_print(HSIZE_T_FORMAT, (unsigned long long)opts->pos[i]); + H5TOOLS_DEBUG("pos loop:%d with opts->pos=%" PRIuHSIZE, i, opts->pos[i]); + parallel_print("%" PRIuHSIZE, opts->pos[i]); parallel_print(" "); } parallel_print("]"); diff --git a/tools/lib/h5diff_util.c b/tools/lib/h5diff_util.c index c40de9d..05f380a 100644 --- a/tools/lib/h5diff_util.c +++ b/tools/lib/h5diff_util.c @@ -39,11 +39,11 @@ print_dimensions(int rank, hsize_t *dims) else { parallel_print("["); for (i = 0; i < rank - 1; i++) { - parallel_print(HSIZE_T_FORMAT, dims[i]); + parallel_print("%" PRIuHSIZE, dims[i]); parallel_print("x"); } - parallel_print(HSIZE_T_FORMAT, dims[rank - 1]); + parallel_print("%" PRIuHSIZE, dims[rank - 1]); parallel_print("]"); } } diff --git a/tools/lib/h5tools.h b/tools/lib/h5tools.h index cc6183a..d3501b3 100644 --- a/tools/lib/h5tools.h +++ b/tools/lib/h5tools.h @@ -29,9 +29,6 @@ #define START_OF_DATA 0x0001 #define END_OF_DATA 0x0002 -/* format for hsize_t */ -#define HSIZE_T_FORMAT "%" H5_PRINTF_LL_WIDTH "u" - #define H5TOOLS_DUMP_MAX_RANK H5S_MAX_RANK /* Stream macros */ diff --git a/tools/lib/h5tools_dump.c b/tools/lib/h5tools_dump.c index b8c9a19..78dd3ff 100644 --- a/tools/lib/h5tools_dump.c +++ b/tools/lib/h5tools_dump.c @@ -745,11 +745,11 @@ h5tools_dump_region_data_blocks(hid_t region_space, hid_t region_id, FILE *strea /* Start coordinates and opposite corner */ for (loop_indx = 0; loop_indx < ndims; loop_indx++) - h5tools_str_append(buffer, "%s" HSIZE_T_FORMAT, loop_indx ? "," : "(", + h5tools_str_append(buffer, "%s%" PRIuHSIZE, loop_indx ? "," : "(", ptdata[indx * 2 * ndims + loop_indx]); for (loop_indx = 0; loop_indx < ndims; loop_indx++) - h5tools_str_append(buffer, "%s" HSIZE_T_FORMAT, loop_indx ? "," : ")-(", + h5tools_str_append(buffer, "%s%" PRIuHSIZE, loop_indx ? "," : ")-(", ptdata[indx * 2 * ndims + loop_indx + ndims]); h5tools_str_append(buffer, ")"); @@ -1101,7 +1101,7 @@ h5tools_dump_region_data_points(hid_t region_space, hid_t region_id, FILE *strea (unsigned long)indx); for (loop_indx = 0; loop_indx < ndims; loop_indx++) - h5tools_str_append(buffer, "%s" HSIZE_T_FORMAT, loop_indx ? "," : "(", + h5tools_str_append(buffer, "%s%" PRIuHSIZE, loop_indx ? "," : "(", ptdata[indx * ndims + loop_indx]); h5tools_str_append(buffer, ")"); @@ -2608,7 +2608,7 @@ found_string_type: if (H5Tget_array_dims2(type, dims) >= 0) { /* Print array dimensions */ for (i = 0; i < ndims; i++) - h5tools_str_append(buffer, "[" HSIZE_T_FORMAT "]", dims[i]); + h5tools_str_append(buffer, "[%" PRIuHSIZE "]", dims[i]); h5tools_str_append(buffer, " "); } @@ -2683,12 +2683,12 @@ h5tools_print_dataspace(h5tools_str_t *buffer, hid_t space) case H5S_SIMPLE: /* simple dataspace */ - h5tools_str_append(buffer, "%s %s { %s " HSIZE_T_FORMAT, + h5tools_str_append(buffer, "%s %s { %s %" PRIuHSIZE, h5tools_dump_header_format->dataspacedescriptionbegin, S_SIMPLE, h5tools_dump_header_format->dataspacedimbegin, size[0]); for (i = 1; i < ndims; i++) - h5tools_str_append(buffer, ", " HSIZE_T_FORMAT, size[i]); + h5tools_str_append(buffer, ", %" PRIuHSIZE, size[i]); h5tools_str_append(buffer, " %s / ", h5tools_dump_header_format->dataspacedimend); @@ -2696,14 +2696,14 @@ h5tools_print_dataspace(h5tools_str_t *buffer, hid_t space) h5tools_str_append(buffer, "%s %s", h5tools_dump_header_format->dataspacedimbegin, "H5S_UNLIMITED"); else - h5tools_str_append(buffer, "%s " HSIZE_T_FORMAT, - h5tools_dump_header_format->dataspacedimbegin, maxsize[0]); + h5tools_str_append(buffer, "%s %" PRIuHSIZE, h5tools_dump_header_format->dataspacedimbegin, + maxsize[0]); for (i = 1; i < ndims; i++) if (maxsize[i] == H5S_UNLIMITED) h5tools_str_append(buffer, ", %s", "H5S_UNLIMITED"); else - h5tools_str_append(buffer, ", " HSIZE_T_FORMAT, maxsize[i]); + h5tools_str_append(buffer, ", %" PRIuHSIZE, maxsize[i]); h5tools_str_append(buffer, " %s }", h5tools_dump_header_format->dataspacedimend); break; @@ -2742,7 +2742,6 @@ h5tools_print_enum(FILE *stream, h5tools_str_t *buffer, const h5tool_format_t *i { char ** name = NULL; /*member names */ unsigned char *value = NULL; /*value array */ - unsigned char *copy = NULL; /*a pointer to value array */ unsigned i; unsigned nmembs = 0; /*number of members */ int snmembs; @@ -2834,16 +2833,16 @@ h5tools_print_enum(FILE *stream, h5tools_str_t *buffer, const h5tool_format_t *i h5tools_str_append(buffer, "%02x", value[i * dst_size + j]); } else if (H5T_SGN_NONE == H5Tget_sign(native)) { - /*On SGI Altix(cobalt), wrong values were printed out with "value+i*dst_size" - *strangely, unless use another pointer "copy".*/ - copy = value + i * dst_size; - h5tools_str_append(buffer, HSIZE_T_FORMAT, *((unsigned long long *)((void *)copy))); + unsigned long long copy; + + HDmemcpy(©, value + i * dst_size, sizeof(copy)); + h5tools_str_append(buffer, "%llu", copy); } else { - /*On SGI Altix(cobalt), wrong values were printed out with "value+i*dst_size" - *strangely, unless use another pointer "copy".*/ - copy = value + i * dst_size; - h5tools_str_append(buffer, "%" H5_PRINTF_LL_WIDTH "d", *((long long *)((void *)copy))); + long long copy; + + HDmemcpy(©, value + i * dst_size, sizeof(copy)); + h5tools_str_append(buffer, "%lld", copy); } h5tools_str_append(buffer, ";"); @@ -3174,10 +3173,10 @@ h5tools_dump_dcpl(FILE *stream, const h5tool_format_t *info, h5tools_context_t * h5tools_str_append(&buffer, "%s ", CHUNKED); rank = H5Pget_chunk(dcpl_id, (int)NELMTS(chsize), chsize); - h5tools_str_append(&buffer, "%s " HSIZE_T_FORMAT, h5tools_dump_header_format->dataspacedimbegin, + h5tools_str_append(&buffer, "%s %" PRIuHSIZE, h5tools_dump_header_format->dataspacedimbegin, chsize[0]); for (i = 1; i < rank; i++) - h5tools_str_append(&buffer, ", " HSIZE_T_FORMAT, chsize[i]); + h5tools_str_append(&buffer, ", %" PRIuHSIZE, chsize[i]); h5tools_str_append(&buffer, " %s", h5tools_dump_header_format->dataspacedimend); h5tools_render_element(stream, info, ctx, &buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0); @@ -3219,17 +3218,17 @@ h5tools_dump_dcpl(FILE *stream, const h5tool_format_t *info, h5tools_context_t * if (storage_size != 0) ratio = (double)uncomp_size / (double)storage_size; - h5tools_str_append(&buffer, "SIZE " HSIZE_T_FORMAT " (%.3f:1 COMPRESSION)", storage_size, + h5tools_str_append(&buffer, "SIZE %" PRIuHSIZE " (%.3f:1 COMPRESSION)", storage_size, ratio); } else - h5tools_str_append(&buffer, "SIZE " HSIZE_T_FORMAT, storage_size); + h5tools_str_append(&buffer, "SIZE %" PRIuHSIZE, storage_size); H5Sclose(sid); H5Tclose(tid); } else { - h5tools_str_append(&buffer, "SIZE " HSIZE_T_FORMAT, storage_size); + h5tools_str_append(&buffer, "SIZE %" PRIuHSIZE, storage_size); } h5tools_render_element(stream, info, ctx, &buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0); @@ -3247,7 +3246,7 @@ h5tools_dump_dcpl(FILE *stream, const h5tool_format_t *info, h5tools_context_t * ctx->need_prefix = TRUE; h5tools_str_reset(&buffer); - h5tools_str_append(&buffer, "SIZE " HSIZE_T_FORMAT, storage_size); + h5tools_str_append(&buffer, "SIZE %" PRIuHSIZE, storage_size); h5tools_render_element(stream, info, ctx, &buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0); ctx->indent_level--; @@ -3283,7 +3282,7 @@ h5tools_dump_dcpl(FILE *stream, const h5tool_format_t *info, h5tools_context_t * ctx->need_prefix = TRUE; h5tools_str_reset(&buffer); - h5tools_str_append(&buffer, "FILENAME %s SIZE " HSIZE_T_FORMAT, name, size); + h5tools_str_append(&buffer, "FILENAME %s SIZE %" PRIuHSIZE, name, size); h5tools_str_append(&buffer, " OFFSET %ld", offset); h5tools_render_element(stream, info, ctx, &buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0); @@ -3313,7 +3312,7 @@ h5tools_dump_dcpl(FILE *stream, const h5tool_format_t *info, h5tools_context_t * ctx->need_prefix = TRUE; h5tools_str_reset(&buffer); - h5tools_str_append(&buffer, "SIZE " HSIZE_T_FORMAT, storage_size); + h5tools_str_append(&buffer, "SIZE %" PRIuHSIZE, storage_size); h5tools_render_element(stream, info, ctx, &buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0); @@ -3909,7 +3908,7 @@ h5tools_print_dims(h5tools_str_t *buffer, hsize_t *s, int dims) int i; for (i = 0; i < dims; i++) { - h5tools_str_append(buffer, HSIZE_T_FORMAT, s[i]); + h5tools_str_append(buffer, "%" PRIuHSIZE, s[i]); if (i + 1 != dims) h5tools_str_append(buffer, ", "); @@ -4330,7 +4329,7 @@ h5tools_dump_data(FILE *stream, const h5tool_format_t *info, h5tools_context_t * if (string_dataformat.pindex) { string_dataformat.idx_fmt = "(%s): "; - string_dataformat.idx_n_fmt = HSIZE_T_FORMAT; + string_dataformat.idx_n_fmt = "%" PRIuHSIZE; string_dataformat.idx_sep = ","; string_dataformat.line_pre = "%s"; } diff --git a/tools/lib/h5tools_str.c b/tools/lib/h5tools_str.c index 4e8dcc1..1fb2c68 100644 --- a/tools/lib/h5tools_str.c +++ b/tools/lib/h5tools_str.c @@ -300,11 +300,11 @@ h5tools_str_prefix(h5tools_str_t *str /*in,out*/, const h5tool_format_t *info, h if (i) h5tools_str_append(str, "%s", OPT(info->idx_sep, ",")); - h5tools_str_append(str, OPT(info->idx_n_fmt, HSIZE_T_FORMAT), (hsize_t)ctx->pos[i]); + h5tools_str_append(str, OPT(info->idx_n_fmt, "%" PRIuHSIZE), (hsize_t)ctx->pos[i]); } } else /* Scalar */ - h5tools_str_append(str, OPT(info->idx_n_fmt, HSIZE_T_FORMAT), (hsize_t)elmtno); + h5tools_str_append(str, OPT(info->idx_n_fmt, "%" PRIuHSIZE), (hsize_t)elmtno); H5TOOLS_DEBUG("str=%s", str->s); H5TOOLS_ENDDEBUG(" "); @@ -341,11 +341,11 @@ h5tools_str_region_prefix(h5tools_str_t *str /*in,out*/, const h5tool_format_t * if (i) h5tools_str_append(str, "%s", OPT(info->idx_sep, ",")); - h5tools_str_append(str, OPT(info->idx_n_fmt, HSIZE_T_FORMAT), (hsize_t)ctx->pos[i]); + h5tools_str_append(str, OPT(info->idx_n_fmt, "%" PRIuHSIZE), (hsize_t)ctx->pos[i]); } } else /* Scalar */ - h5tools_str_append(str, OPT(info->idx_n_fmt, HSIZE_T_FORMAT), (hsize_t)0); + h5tools_str_append(str, OPT(info->idx_n_fmt, "%" PRIuHSIZE), (hsize_t)0); H5TOOLS_DEBUG("str=%s", str->s); H5TOOLS_ENDDEBUG(" "); @@ -385,7 +385,7 @@ h5tools_str_dump_space_slabs(h5tools_str_t *str, hid_t rspace, const h5tool_form /* Start coordinates */ h5tools_str_append(str, "%s%s ", info->line_indent, START); for (j = 0; j < ndims; j++) - h5tools_str_append(str, "%s" HSIZE_T_FORMAT, j ? "," : "(", start[j]); + h5tools_str_append(str, "%s%" PRIuHSIZE, j ? "," : "(", start[j]); h5tools_str_append(str, ")"); h5tools_str_append(str, "%s", "\n"); h5tools_str_indent(str, info, ctx); @@ -393,7 +393,7 @@ h5tools_str_dump_space_slabs(h5tools_str_t *str, hid_t rspace, const h5tool_form /* Stride coordinates */ h5tools_str_append(str, "%s ", STRIDE); for (j = 0; j < ndims; j++) - h5tools_str_append(str, "%s" HSIZE_T_FORMAT, j ? "," : "(", stride[j]); + h5tools_str_append(str, "%s%" PRIuHSIZE, j ? "," : "(", stride[j]); h5tools_str_append(str, ")"); h5tools_str_append(str, "%s", "\n"); h5tools_str_indent(str, info, ctx); @@ -404,7 +404,7 @@ h5tools_str_dump_space_slabs(h5tools_str_t *str, hid_t rspace, const h5tool_form if (count[j] == H5S_UNLIMITED) h5tools_str_append(str, "%s%s", j ? "," : "(", "H5S_UNLIMITED"); else - h5tools_str_append(str, "%s" HSIZE_T_FORMAT, j ? "," : "(", count[j]); + h5tools_str_append(str, "%s%" PRIuHSIZE, j ? "," : "(", count[j]); } h5tools_str_append(str, ")"); h5tools_str_append(str, "%s", "\n"); @@ -416,7 +416,7 @@ h5tools_str_dump_space_slabs(h5tools_str_t *str, hid_t rspace, const h5tool_form if (block[j] == H5S_UNLIMITED) h5tools_str_append(str, "%s%s", j ? "," : "(", "H5S_UNLIMITED"); else - h5tools_str_append(str, "%s" HSIZE_T_FORMAT, j ? "," : "(", block[j]); + h5tools_str_append(str, "%s%" PRIuHSIZE, j ? "," : "(", block[j]); } h5tools_str_append(str, ")"); } @@ -468,11 +468,10 @@ h5tools_str_dump_space_blocks(h5tools_str_t *str, hid_t rspace, const h5tool_for /* Start coordinates and opposite corner */ for (v = 0; v < ndims; v++) - h5tools_str_append(str, "%s" HSIZE_T_FORMAT, v ? "," : "(", ptdata[u * 2 * ndims + v]); + h5tools_str_append(str, "%s%" PRIuHSIZE, v ? "," : "(", ptdata[u * 2 * ndims + v]); for (v = 0; v < ndims; v++) - h5tools_str_append(str, "%s" HSIZE_T_FORMAT, v ? "," : ")-(", - ptdata[u * 2 * ndims + v + ndims]); + h5tools_str_append(str, "%s%" PRIuHSIZE, v ? "," : ")-(", ptdata[u * 2 * ndims + v + ndims]); h5tools_str_append(str, ")"); } @@ -527,7 +526,7 @@ h5tools_str_dump_space_points(h5tools_str_t *str, hid_t rspace, const h5tool_for (unsigned long)u); for (v = 0; v < ndims; v++) - h5tools_str_append(str, "%s" HSIZE_T_FORMAT, v ? "," : "(", (ptdata[u * ndims + v])); + h5tools_str_append(str, "%s%" PRIuHSIZE, v ? "," : "(", (ptdata[u * ndims + v])); h5tools_str_append(str, ")"); } diff --git a/tools/lib/h5tools_str.h b/tools/lib/h5tools_str.h index eee87c9..f4610f0 100644 --- a/tools/lib/h5tools_str.h +++ b/tools/lib/h5tools_str.h @@ -26,7 +26,7 @@ typedef struct h5tools_str_t { H5TOOLS_DLL void h5tools_str_close(h5tools_str_t *str); H5TOOLS_DLL size_t h5tools_str_len(h5tools_str_t *str); -H5TOOLS_DLL char * h5tools_str_append(h5tools_str_t *str, const char *fmt, ...); +H5TOOLS_DLL char * h5tools_str_append(h5tools_str_t *str, const char *fmt, ...) H5_ATTR_FORMAT(printf, 2, 3); H5TOOLS_DLL char * h5tools_str_reset(h5tools_str_t *str); H5TOOLS_DLL char * h5tools_str_trunc(h5tools_str_t *str, size_t size); H5TOOLS_DLL char * h5tools_str_fmt(h5tools_str_t *str, size_t start, const char *fmt); diff --git a/tools/lib/h5tools_utils.h b/tools/lib/h5tools_utils.h index 3733742..4cf4e40 100644 --- a/tools/lib/h5tools_utils.h +++ b/tools/lib/h5tools_utils.h @@ -70,11 +70,11 @@ H5TOOLS_DLLVAR unsigned h5tools_nCols; /*max number of columns for outputting * /* Definitions of useful routines */ H5TOOLS_DLL void indentation(unsigned); H5TOOLS_DLL void print_version(const char *progname); -H5TOOLS_DLL void parallel_print(const char *format, ...); +H5TOOLS_DLL void parallel_print(const char *format, ...) H5_ATTR_FORMAT(printf, 1, 2); H5TOOLS_DLL herr_t parse_tuple(const char *start, int sep, char **cpy_out, unsigned *nelems, char ***ptrs_out); -H5TOOLS_DLL void error_msg(const char *fmt, ...); -H5TOOLS_DLL void warn_msg(const char *fmt, ...); +H5TOOLS_DLL void error_msg(const char *fmt, ...) H5_ATTR_FORMAT(printf, 1, 2); +H5TOOLS_DLL void warn_msg(const char *fmt, ...) H5_ATTR_FORMAT(printf, 1, 2); H5TOOLS_DLL void help_ref_msg(FILE *output); H5TOOLS_DLL void free_table(table_t *table); #ifdef H5DUMP_DEBUG diff --git a/tools/lib/h5trav.c b/tools/lib/h5trav.c index 1f4b0d6..cca36e4 100644 --- a/tools/lib/h5trav.c +++ b/tools/lib/h5trav.c @@ -53,11 +53,6 @@ typedef struct trav_path_op_data_t { const char *path; } trav_path_op_data_t; -/* format for hsize_t */ -#ifdef H5TRAV_PRINT_SPACE -#define HSIZE_T_FORMAT "%" H5_PRINTF_LL_WIDTH "u" -#endif /* H5TRAV_PRINT_SPACE */ - /*------------------------------------------------------------------------- * local functions *------------------------------------------------------------------------- @@ -856,7 +851,7 @@ trav_attr(hid_t /* simple dataspace */ HDprintf(" {"); for (i = 0; i < ndims; i++) { - HDprintf("%s" HSIZE_T_FORMAT, i ? ", " : "", size[i]); + HDprintf("%s%" PRIuHSIZE, i ? ", " : "", size[i]); } HDprintf("}\n"); break; diff --git a/tools/src/h5dump/h5dump_xml.c b/tools/src/h5dump/h5dump_xml.c index 5f64d6c..827daf3 100644 --- a/tools/src/h5dump/h5dump_xml.c +++ b/tools/src/h5dump/h5dump_xml.c @@ -1779,8 +1779,7 @@ xml_dump_dataspace(hid_t space) /* Render the element */ h5tools_str_reset(&buffer); h5tools_str_append(&buffer, - "<%sDimension DimSize=\"%" H5_PRINTF_LL_WIDTH - "u\" MaxDimSize=\"UNLIMITED\"/>", + "<%sDimension DimSize=\"%" PRIuHSIZE "\" MaxDimSize=\"UNLIMITED\"/>", xmlnsprefix, size[i]); h5tools_render_element(rawoutstream, outputformat, &ctx, &buffer, &curr_pos, (size_t)outputformat->line_ncols, (hsize_t)0, (hsize_t)0); @@ -1790,10 +1789,9 @@ xml_dump_dataspace(hid_t space) /* Render the element */ h5tools_str_reset(&buffer); - h5tools_str_append(&buffer, - "<%sDimension DimSize=\"%" H5_PRINTF_LL_WIDTH - "u\" MaxDimSize=\"%" H5_PRINTF_LL_WIDTH "u\"/>", - xmlnsprefix, size[i], size[i]); + h5tools_str_append( + &buffer, "<%sDimension DimSize=\"%" PRIuHSIZE "\" MaxDimSize=\"%" PRIuHSIZE "\"/>", + xmlnsprefix, size[i], size[i]); h5tools_render_element(rawoutstream, outputformat, &ctx, &buffer, &curr_pos, (size_t)outputformat->line_ncols, (hsize_t)0, (hsize_t)0); } @@ -1802,10 +1800,9 @@ xml_dump_dataspace(hid_t space) /* Render the element */ h5tools_str_reset(&buffer); - h5tools_str_append(&buffer, - "<%sDimension DimSize=\"%" H5_PRINTF_LL_WIDTH - "u\" MaxDimSize=\"%" H5_PRINTF_LL_WIDTH "u\"/>", - xmlnsprefix, size[i], maxsize[i]); + h5tools_str_append( + &buffer, "<%sDimension DimSize=\"%" PRIuHSIZE "\" MaxDimSize=\"%" PRIuHSIZE "\"/>", + xmlnsprefix, size[i], maxsize[i]); h5tools_render_element(rawoutstream, outputformat, &ctx, &buffer, &curr_pos, (size_t)outputformat->line_ncols, (hsize_t)0, (hsize_t)0); } @@ -3897,8 +3894,8 @@ xml_dump_dataset(hid_t did, const char *name, struct subset_t H5_ATTR_UNUSED *ss /* Render the element */ h5tools_str_reset(&buffer); - h5tools_str_append(&buffer, "<%sChunkDimension DimSize=\"%" H5_PRINTF_LL_WIDTH "u\" />", - xmlnsprefix, chsize[i]); + h5tools_str_append(&buffer, "<%sChunkDimension DimSize=\"%" PRIuHSIZE "\" />", xmlnsprefix, + chsize[i]); h5tools_render_element(rawoutstream, outputformat, &ctx, &buffer, &curr_pos, (size_t)outputformat->line_ncols, (hsize_t)0, (hsize_t)0); } @@ -4529,12 +4526,16 @@ xml_print_enum(hid_t type) h5tools_str_append(&buffer, "%02x", value[i * dst_size + j]); } else if (H5T_SGN_NONE == H5Tget_sign(native)) { - h5tools_str_append(&buffer, "%" H5_PRINTF_LL_WIDTH "u", - *((unsigned long long *)((void *)(value + i * dst_size)))); + unsigned long long copy; + + HDmemcpy(©, value + i * dst_size, sizeof(copy)); + h5tools_str_append(&buffer, "%llu", copy); } else { - h5tools_str_append(&buffer, "%" H5_PRINTF_LL_WIDTH "d", - *((long long *)((void *)(value + i * dst_size)))); + long long copy; + + HDmemcpy(©, value + i * dst_size, sizeof(copy)); + h5tools_str_append(&buffer, "%lld", copy); } h5tools_render_element(rawoutstream, outputformat, &ctx, &buffer, &curr_pos, (size_t)outputformat->line_ncols, (hsize_t)0, (hsize_t)0); diff --git a/tools/src/h5format_convert/h5format_convert.c b/tools/src/h5format_convert/h5format_convert.c index b9572fe..50d0f5d 100644 --- a/tools/src/h5format_convert/h5format_convert.c +++ b/tools/src/h5format_convert/h5format_convert.c @@ -127,7 +127,7 @@ parse_command_line(int argc, const char *const *argv) dname_g = HDstrdup(H5_optarg); if (dname_g == NULL) { h5tools_setstatus(EXIT_FAILURE); - error_msg("No dataset name\n", H5_optarg); + error_msg("No dataset name `%s`\n", H5_optarg); usage(h5tools_getprogname()); goto error; } diff --git a/tools/src/h5ls/h5ls.c b/tools/src/h5ls/h5ls.c index 0d34574..a27a8cc 100644 --- a/tools/src/h5ls/h5ls.c +++ b/tools/src/h5ls/h5ls.c @@ -78,9 +78,9 @@ static h5tool_format_t ls_dataformat = { ",", /*elmt_suf1 */ " ", /*elmt_suf2 */ - HSIZE_T_FORMAT, /*idx_n_fmt */ - ",", /*idx_sep */ - "(%s)", /*idx_fmt */ + "%" PRIuHSIZE, /*idx_n_fmt */ + ",", /*idx_sep */ + "(%s)", /*idx_fmt */ 65535, /*line_ncols */ /*standard default columns */ @@ -942,8 +942,7 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind) /* Print members */ for (i = 0; i < (unsigned)nmembs; i++) { - unsigned char *copy; /* a pointer to value array */ - int nchars; /* number of output characters */ + int nchars; /* number of output characters */ h5tools_str_append(buffer, "\n%*s", ind + 4, ""); nchars = print_string(buffer, name[i], TRUE); @@ -957,16 +956,16 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind) h5tools_str_append(buffer, "%02x", value[i * dst_size + j]); } else if (H5T_SGN_NONE == H5Tget_sign(native)) { - /*On SGI Altix(cobalt), wrong values were printed out with "value+i*dst_size" - *strangely, unless use another pointer "copy".*/ - copy = value + i * dst_size; - h5tools_str_append(buffer, HSIZE_T_FORMAT, *((unsigned long long *)((void *)copy))); + unsigned long long copy; + + HDmemcpy(©, value + i * dst_size, sizeof(copy)); + h5tools_str_append(buffer, "%llu", copy); } else { - /*On SGI Altix(cobalt), wrong values were printed out with "value+i*dst_size" - *strangely, unless use another pointer "copy".*/ - copy = value + i * dst_size; - h5tools_str_append(buffer, "%lld", *((long long *)((void *)copy))); + long long copy; + + HDmemcpy(©, value + i * dst_size, sizeof(copy)); + h5tools_str_append(buffer, "%lld", copy); } } @@ -1185,13 +1184,13 @@ print_array_type(h5tools_str_t *buffer, hid_t type, int ind) /* Print dimensions */ for (i = 0; i < ndims; i++) - h5tools_str_append(buffer, "%s" HSIZE_T_FORMAT, i ? "," : "[", dims[i]); + h5tools_str_append(buffer, "%s%" PRIuHSIZE, i ? "," : "[", dims[i]); h5tools_str_append(buffer, "]"); HDfree(dims); } else - h5tools_str_append(buffer, " [SCALAR]\n", rawoutstream); + h5tools_str_append(buffer, " [SCALAR]\n"); /* Print parent type */ h5tools_str_append(buffer, " "); @@ -1701,7 +1700,7 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain /* simple dataspace */ h5tools_str_append(&buffer, " {"); for (i = 0; i < ndims; i++) { - h5tools_str_append(&buffer, "%s" HSIZE_T_FORMAT, i ? ", " : "", size[i]); + h5tools_str_append(&buffer, "%s%" PRIuHSIZE, i ? ", " : "", size[i]); nelmts *= size[i]; } h5tools_str_append(&buffer, "}\n"); @@ -1789,12 +1788,12 @@ dataset_list1(hid_t dset) ndims = H5Sget_simple_extent_dims(space, cur_size, max_size); h5tools_str_append(&buffer, " {"); for (i = 0; i < ndims; i++) { - h5tools_str_append(&buffer, "%s" HSIZE_T_FORMAT, i ? ", " : "", cur_size[i]); + h5tools_str_append(&buffer, "%s%" PRIuHSIZE, i ? ", " : "", cur_size[i]); if (max_size[i] == H5S_UNLIMITED) { h5tools_str_append(&buffer, "/%s", "Inf"); } else if (max_size[i] != cur_size[i] || verbose_g > 0) { - h5tools_str_append(&buffer, "/" HSIZE_T_FORMAT, max_size[i]); + h5tools_str_append(&buffer, "/%" PRIuHSIZE, max_size[i]); } } if (space_type == H5S_SCALAR) @@ -1868,10 +1867,10 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name) h5tools_str_append(&buffer, " %-10s {", "Chunks:"); total = H5Tget_size(type); for (i = 0; i < ndims; i++) { - h5tools_str_append(&buffer, "%s" HSIZE_T_FORMAT, i ? ", " : "", chsize[i]); + h5tools_str_append(&buffer, "%s%" PRIuHSIZE, i ? ", " : "", chsize[i]); total *= chsize[i]; } - h5tools_str_append(&buffer, "} " HSIZE_T_FORMAT " bytes\n", total); + h5tools_str_append(&buffer, "} %" PRIuHSIZE " bytes\n", total); } break; case H5D_COMPACT: break; @@ -1897,15 +1896,13 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name) if (H5Pget_external(dcpl, (unsigned)i, sizeof(f_name), f_name, &f_offset, &f_size) < 0) { h5tools_str_append( - &buffer, - " #%03d %10" H5_PRINTF_LL_WIDTH "u %10s %10s ***ERROR*** %s\n", i, + &buffer, " #%03d %10" PRIuHSIZE " %10s %10s ***ERROR*** %s\n", i, total, "", "", i + 1 < nf ? "Following addresses are incorrect" : ""); } else if (H5S_UNLIMITED == f_size) { h5tools_str_append(&buffer, - " #%03d %10" H5_PRINTF_LL_WIDTH - "u %10" H5_PRINTF_LL_WIDTH "u %10s ", - i, total, (hsize_t)f_offset, "INF"); + " #%03d %10" PRIuHSIZE " %10" PRIuHSIZE " %10s ", i, + total, (hsize_t)f_offset, "INF"); print_string(&buffer, f_name, TRUE); } else { @@ -1985,8 +1982,7 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name) case H5T_ARRAY: case H5T_NCLASSES: default: - h5tools_str_append(&buffer, - HSIZE_T_FORMAT " logical byte%s, " HSIZE_T_FORMAT " allocated byte%s", + h5tools_str_append(&buffer, "%" PRIuHSIZE " logical byte%s, %" PRIuHSIZE " allocated byte%s", total, 1 == total ? "" : "s", used, 1 == used ? "" : "s"); if (used > 0) { utilization = ((double)total * 100.0) / (double)used; diff --git a/tools/src/h5perf/pio_perf.c b/tools/src/h5perf/pio_perf.c index c233684..7e12bed 100644 --- a/tools/src/h5perf/pio_perf.c +++ b/tools/src/h5perf/pio_perf.c @@ -199,7 +199,7 @@ static int destroy_comm_world(void); static void output_results(const struct options *options, const char *name, minmax *table, int table_size, off_t data_size); static void output_times(const struct options *options, const char *name, minmax *table, int table_size); -static void output_report(const char *fmt, ...); +static void output_report(const char *fmt, ...) H5_ATTR_FORMAT(printf, 1, 2); static void print_indent(register int indent); static void usage(const char *prog); static void report_parameters(struct options *opts); diff --git a/tools/src/h5repack/h5repack_main.c b/tools/src/h5repack/h5repack_main.c index 68e37fd..4c88bd7 100644 --- a/tools/src/h5repack/h5repack_main.c +++ b/tools/src/h5repack/h5repack_main.c @@ -701,7 +701,7 @@ parse_command_line(int argc, const char *const *argv, pack_opt_t *options) case 'a': options->alignment = HDstrtoull(H5_optarg, NULL, 0); if (options->alignment < 1) { - error_msg("invalid alignment size\n", H5_optarg); + error_msg("invalid alignment size `%s`\n", H5_optarg); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; goto done; @@ -721,7 +721,7 @@ parse_command_line(int argc, const char *const *argv, pack_opt_t *options) else if (!HDstrcmp(strategy, "NONE")) options->fs_strategy = H5F_FSPACE_STRATEGY_NONE; else { - error_msg("invalid file space management strategy\n", H5_optarg); + error_msg("invalid file space management strategy `%s`\n", H5_optarg); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; goto done; -- cgit v0.12 From 3107fafa2ce858b598617463334d3d6d01dc1e4c Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Mon, 21 Feb 2022 15:32:29 -0600 Subject: Oess 168 utils mirror vfd (#1444) * Committing clang-format changes * Spelling of preceed was corrected to proceed, but should have been corrected to precede. * Correct spelling correction of 'preceed' incorrectly to 'proceed'. It should be 'precede'. * OESS-168: Remove clang warnings. * OESS-168: Address @lrknox and @gnuoyd reviews. * Eliminate clang warnings listed in PR #1310 without adding new ssize_t variables. * Committing clang-format changes * Add H5_ATTR_UNUSED to wait_for_child call. Remove unneeded casts in mirror_log calls. * Keep ssize_t in mirror_server.c line 479. Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Hyo-Kyung Lee --- src/H5FDmirror_priv.h | 4 ++-- utils/mirror_vfd/mirror_remote.c | 2 +- utils/mirror_vfd/mirror_remote.h | 2 +- utils/mirror_vfd/mirror_server.c | 27 +++++++++++++-------------- utils/mirror_vfd/mirror_writer.c | 8 +++----- 5 files changed, 20 insertions(+), 23 deletions(-) diff --git a/src/H5FDmirror_priv.h b/src/H5FDmirror_priv.h index 6a7b13e..f647c21 100644 --- a/src/H5FDmirror_priv.h +++ b/src/H5FDmirror_priv.h @@ -28,10 +28,10 @@ extern "C" { * = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = */ -/* The maximum allowed size for a receiving buffer when accepting bytes to +/* Define the maximum allowed size for a receiving buffer when accepting bytes to * write. Writes larger than this size are performed by multiple accept-write * steps by the Writer. */ -#define H5FD_MIRROR_DATA_BUFFER_MAX H5_GB /* 1 Gigabyte */ +#define H5FD_MIRROR_DATA_BUFFER_MAX (1024 * 1024 * 1024) /* 1 Gigabyte */ #define H5FD_MIRROR_XMIT_CURR_VERSION 1 #define H5FD_MIRROR_XMIT_MAGIC 0x87F8005B diff --git a/utils/mirror_vfd/mirror_remote.c b/utils/mirror_vfd/mirror_remote.c index f61580d..8bb9544 100644 --- a/utils/mirror_vfd/mirror_remote.c +++ b/utils/mirror_vfd/mirror_remote.c @@ -147,7 +147,7 @@ mirror_log_bytes(struct mirror_log_info *info, unsigned int level, size_t n_byte * ---------------------------------------------------------------------------- */ loginfo_t * -mirror_log_init(char *path, char *prefix, unsigned int verbosity) +mirror_log_init(char *path, const char *prefix, unsigned int verbosity) { loginfo_t *info = NULL; diff --git a/utils/mirror_vfd/mirror_remote.h b/utils/mirror_vfd/mirror_remote.h index 6f37b0b..9415f7f 100644 --- a/utils/mirror_vfd/mirror_remote.h +++ b/utils/mirror_vfd/mirror_remote.h @@ -42,7 +42,7 @@ typedef struct mirror_log_info { void mirror_log(loginfo_t *info, unsigned int level, const char *format, ...); void mirror_log_bytes(loginfo_t *info, unsigned int level, size_t n_bytes, const unsigned char *buf); -loginfo_t *mirror_log_init(char *path, char *prefix, unsigned int verbosity); +loginfo_t *mirror_log_init(char *path, const char *prefix, unsigned int verbosity); int mirror_log_term(loginfo_t *loginfo); herr_t run_writer(int socketfd, H5FD_mirror_xmit_open_t *xmit_open); diff --git a/utils/mirror_vfd/mirror_server.c b/utils/mirror_vfd/mirror_server.c index f6a8bfe..5381d95 100644 --- a/utils/mirror_vfd/mirror_server.c +++ b/utils/mirror_vfd/mirror_server.c @@ -94,14 +94,14 @@ * --------------------------------------------------------------------------- */ struct op_args { - uint32_t magic; - int help; - int main_port; - int verbosity; - int log_prepend_serv; - int log_prepend_type; - char log_path[PATH_MAX + 1]; - char writer_log_path[PATH_MAX + 1]; + uint32_t magic; + int help; + int main_port; + unsigned int verbosity; + int log_prepend_serv; + int log_prepend_type; + char log_path[PATH_MAX + 1]; + char writer_log_path[PATH_MAX + 1]; }; /* --------------------------------------------------------------------------- @@ -224,7 +224,7 @@ parse_args(int argc, char **argv, struct op_args *args_out) } /* end if port */ else if (!HDstrncmp(argv[i], "--verbosity=", 12)) { mirror_log(NULL, V_INFO, "parsing 'verbosity' (%s)", argv[i] + 12); - args_out->verbosity = HDatoi(argv[i] + 12); + args_out->verbosity = (unsigned int)HDatoi(argv[i] + 12); } /* end if verbosity */ else if (!HDstrncmp(argv[i], "--logpath=", 10)) { mirror_log(NULL, V_INFO, "parsing 'logpath' (%s)", argv[i] + 10); @@ -456,7 +456,7 @@ error: * --------------------------------------------------------------------------- */ static void -wait_for_child(int sig) +wait_for_child(int H5_ATTR_UNUSED sig) { while (HDwaitpid(-1, NULL, WNOHANG) > 0) ; @@ -476,7 +476,7 @@ handle_requests(struct server_run *run) { int connfd = -1; /**/ char mybuf[H5FD_MIRROR_XMIT_OPEN_SIZE]; /**/ - int ret; /* general-purpose error-checking */ + ssize_t ret; /* general-purpose error-checking */ int pid; /* process ID of fork */ struct sigaction sa; int ret_value = 0; @@ -521,14 +521,13 @@ handle_requests(struct server_run *run) /* Read handshake from port connection. */ - ret = (int)HDread(connfd, &mybuf, H5FD_MIRROR_XMIT_OPEN_SIZE); - if (-1 == ret) { + if ((ret = HDread(connfd, &mybuf, H5FD_MIRROR_XMIT_OPEN_SIZE)) < 0) { mirror_log(run->loginfo, V_ERR, "read:%d", ret); goto error; } mirror_log(run->loginfo, V_INFO, "received %d bytes", ret); mirror_log(run->loginfo, V_ALL, "```"); - mirror_log_bytes(run->loginfo, V_ALL, ret, (const unsigned char *)mybuf); + mirror_log_bytes(run->loginfo, V_ALL, (size_t)ret, (const unsigned char *)mybuf); mirror_log(run->loginfo, V_ALL, "```"); /* Respond to handshake message. diff --git a/utils/mirror_vfd/mirror_writer.c b/utils/mirror_vfd/mirror_writer.c index d3f12de..5726db5 100644 --- a/utils/mirror_vfd/mirror_writer.c +++ b/utils/mirror_vfd/mirror_writer.c @@ -788,8 +788,7 @@ do_write(struct mirror_session *session, const unsigned char *xmit_buf) */ sum_bytes_written = 0; do { - nbytes_in_packet = HDread(session->sockfd, buf, H5FD_MIRROR_DATA_BUFFER_MAX); - if (-1 == nbytes_in_packet) { + if ((nbytes_in_packet = HDread(session->sockfd, buf, H5FD_MIRROR_DATA_BUFFER_MAX)) < 0) { mirror_log(session->loginfo, V_ERR, "can't read into databuffer"); reply_error(session, "can't read data buffer"); return -1; @@ -798,7 +797,7 @@ do_write(struct mirror_session *session, const unsigned char *xmit_buf) mirror_log(session->loginfo, V_INFO, "received %zd bytes", nbytes_in_packet); if (HEXDUMP_WRITEDATA) { mirror_log(session->loginfo, V_ALL, "DATA:\n```"); - mirror_log_bytes(session->loginfo, V_ALL, nbytes_in_packet, (const unsigned char *)buf); + mirror_log_bytes(session->loginfo, V_ALL, (size_t)nbytes_in_packet, (const unsigned char *)buf); mirror_log(session->loginfo, V_ALL, "```"); } @@ -859,8 +858,7 @@ receive_communique(struct mirror_session *session, struct sock_comm *comm) mirror_log(session->loginfo, V_INFO, "ready to receive"); /* TODO */ - read_ret = HDread(session->sockfd, comm->raw, H5FD_MIRROR_XMIT_BUFFER_MAX); - if (-1 == read_ret) { + if ((read_ret = HDread(session->sockfd, comm->raw, H5FD_MIRROR_XMIT_BUFFER_MAX)) < 0) { mirror_log(session->loginfo, V_ERR, "read:%zd", read_ret); goto error; } -- cgit v0.12 From 5a5a77040225a5c621cb75b578204c8b534a7652 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Tue, 22 Feb 2022 08:36:28 -0600 Subject: Enable creation of RPM/DEB binaries GH#1394 (#1398) Co-authored-by: Larry Knox --- CMakeInstallation.cmake | 8 +++++++- release_docs/RELEASE.txt | 9 +++++++++ src/CMakeLists.txt | 1 + 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CMakeInstallation.cmake b/CMakeInstallation.cmake index f5e0897..b5efebf 100644 --- a/CMakeInstallation.cmake +++ b/CMakeInstallation.cmake @@ -395,13 +395,19 @@ if (NOT HDF5_EXTERNALLY_CONFIGURED AND NOT HDF5_NO_PACKAGES) set (CPACK_PACKAGING_INSTALL_PREFIX "/${CPACK_PACKAGE_INSTALL_DIRECTORY}") set (CPACK_COMPONENTS_ALL_IN_ONE_PACKAGE ON) + list (APPEND CPACK_GENERATOR "DEB") set (CPACK_DEBIAN_PACKAGE_SECTION "Libraries") set (CPACK_DEBIAN_PACKAGE_MAINTAINER "${HDF5_PACKAGE_BUGREPORT}") -# list (APPEND CPACK_GENERATOR "RPM") + list (APPEND CPACK_GENERATOR "RPM") set (CPACK_RPM_PACKAGE_RELEASE "1") + set (CPACK_RPM_PACKAGE_RELEASE_DIST ON) set (CPACK_RPM_COMPONENT_INSTALL ON) set (CPACK_RPM_PACKAGE_RELOCATABLE ON) + set (CPACK_RPM_FILE_NAME "RPM-DEFAULT") + set (CPACK_RPM_PACKAGE_NAME "${CPACK_PACKAGE_NAME}") + set (CPACK_RPM_PACKAGE_VERSION "${CPACK_PACKAGE_VERSION}") + set (CPACK_RPM_PACKAGE_VENDOR "${CPACK_PACKAGE_VENDOR}") set (CPACK_RPM_PACKAGE_LICENSE "BSD-style") set (CPACK_RPM_PACKAGE_GROUP "Development/Libraries") set (CPACK_RPM_PACKAGE_URL "${HDF5_PACKAGE_URL}") diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index f68a7fe..10663f2 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -47,6 +47,15 @@ New Features Configuration: ------------- + - CPack will now generate RPM/DEB packages. + + Enabled the RPM and DEB CPack generators on linux. In addition to + generating STGZ and TGZ packages, CPack will try to package the + library for RPM and DEB packages. This is the initial attempt and + may change as issues are resolved. + + (ADB - 2022/01/27) + - Added new option to the h5cc scripts produced by CMake. Add -showconfig option to h5cc scripts that cat the diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index da99dc6..017b5d6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1329,6 +1329,7 @@ if (HDF5_EXPORTED_TARGETS) RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT libraries FRAMEWORK DESTINATION ${HDF5_INSTALL_FWRK_DIR} COMPONENT libraries INCLUDES DESTINATION include + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE ) endif () -- cgit v0.12 From 758e97c1e5f02869dbae81a7a428d270a016464e Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Thu, 24 Feb 2022 10:04:59 -0600 Subject: Parallel Compression improvements (#1302) --- CMakeLists.txt | 12 +- MANIFEST | 2 + config/cmake/H5pubconf.h.in | 3 + configure.ac | 17 +- examples/CMakeLists.txt | 42 +- examples/CMakeTests.cmake | 40 +- examples/Makefile.am | 6 +- examples/ph5_filtered_writes.c | 482 ++ examples/ph5_filtered_writes_no_sel.c | 366 ++ examples/ph5example.c | 8 +- release_docs/RELEASE.txt | 40 +- src/H5Dchunk.c | 378 +- src/H5Dint.c | 55 +- src/H5Dio.c | 128 +- src/H5Dmpio.c | 5397 ++++++++++++++----- src/H5Dpkg.h | 17 +- src/H5Dselect.c | 187 +- src/H5FDmpio.c | 44 +- src/H5Fmpi.c | 64 + src/H5Fprivate.h | 3 +- src/H5mpi.c | 233 + src/H5private.h | 30 + src/H5public.h | 7 +- testpar/t_2Gio.c | 275 +- testpar/t_dset.c | 277 +- testpar/t_filters_parallel.c | 9394 ++++++++++++++++++++------------- testpar/t_filters_parallel.h | 117 +- testpar/testphdf5.h | 4 - 28 files changed, 11790 insertions(+), 5838 deletions(-) create mode 100644 examples/ph5_filtered_writes.c create mode 100644 examples/ph5_filtered_writes_no_sel.c diff --git a/CMakeLists.txt b/CMakeLists.txt index fedce44..cad378b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -708,10 +708,14 @@ if (HDF5_ENABLE_PARALLEL) # Used by Parallel Compression feature set (PARALLEL_FILTERED_WRITES ON) - CHECK_SYMBOL_EXISTS (MPI_Mprobe "mpi.h" H5_HAVE_MPI_Mprobe) - CHECK_SYMBOL_EXISTS (MPI_Imrecv "mpi.h" H5_HAVE_MPI_Imrecv) - if (NOT H5_HAVE_MPI_Mprobe OR NOT H5_HAVE_MPI_Imrecv) - message (WARNING "The MPI_Mprobe and/or MPI_Imrecv functions could not be located. + CHECK_SYMBOL_EXISTS (MPI_Ibarrier "mpi.h" H5_HAVE_MPI_Ibarrier) + CHECK_SYMBOL_EXISTS (MPI_Issend "mpi.h" H5_HAVE_MPI_Issend) + CHECK_SYMBOL_EXISTS (MPI_Iprobe "mpi.h" H5_HAVE_MPI_Iprobe) + CHECK_SYMBOL_EXISTS (MPI_Irecv "mpi.h" H5_HAVE_MPI_Irecv) + if (H5_HAVE_MPI_Ibarrier AND H5_HAVE_MPI_Issend AND H5_HAVE_MPI_Iprobe AND H5_HAVE_MPI_Irecv) + set (H5_HAVE_PARALLEL_FILTERED_WRITES 1) + else () + message (WARNING "The MPI_Ibarrier/MPI_Issend/MPI_Iprobe/MPI_Irecv functions could not be located. Parallel writes of filtered data will be disabled.") set (PARALLEL_FILTERED_WRITES OFF) endif () diff --git a/MANIFEST b/MANIFEST index 0aa9957..37b84d4 100644 --- a/MANIFEST +++ b/MANIFEST @@ -333,6 +333,8 @@ ./examples/h5_ref2reg_deprec.c ./examples/h5_shared_mesg.c ./examples/ph5example.c +./examples/ph5_filtered_writes.c +./examples/ph5_filtered_writes_no_sel.c ./examples/h5_vds.c ./examples/h5_vds-exc.c ./examples/h5_vds-exclim.c diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index 1f7f4b1..4956c97 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -265,6 +265,9 @@ /* Define if we have parallel support */ #cmakedefine H5_HAVE_PARALLEL @H5_HAVE_PARALLEL@ +/* Define if we have support for writing to filtered datasets in parallel */ +#cmakedefine H5_HAVE_PARALLEL_FILTERED_WRITES @H5_HAVE_PARALLEL_FILTERED_WRITES@ + /* Define if both pread and pwrite exist. */ #cmakedefine H5_HAVE_PREADWRITE @H5_HAVE_PREADWRITE@ diff --git a/configure.ac b/configure.ac index 4cf329c..55468bf 100644 --- a/configure.ac +++ b/configure.ac @@ -2987,11 +2987,11 @@ if test -n "$PARALLEL"; then fi ## ---------------------------------------------------------------------- - ## Check for the MPI-3 functions necessary for the Parallel Compression + ## Check for the MPI functions necessary for the Parallel Compression ## feature. If these are not present, issue a warning that Parallel ## Compression will be disabled. ## - AC_MSG_CHECKING([for MPI_Mprobe and MPI_Imrecv functions]) + AC_MSG_CHECKING([for MPI_Ibarrier/MPI_Issend/MPI_Iprobe/MPI_Irecv functions]) AC_LINK_IFELSE( [AC_LANG_PROGRAM( @@ -2999,16 +2999,19 @@ if test -n "$PARALLEL"; then #include ]], [[ - MPI_Message message; + int flag; MPI_Init(0, (void *) 0); - MPI_Mprobe(0, 0, 0, &message, (void *) 0); - MPI_Imrecv((void *) 0, 0, 0, (void *) 0, (void *) 0); + MPI_Ibarrier(0, (void *) 0); + MPI_Issend((void *) 0, 0, 0, 0, 0, 0, (void *) 0); + MPI_Iprobe(0, 0, 0, &flag, (void *) 0); + MPI_Irecv((void *) 0, 0, 0, 0, 0, 0, (void *) 0); ]] )], [AC_MSG_RESULT([yes]) - PARALLEL_FILTERED_WRITES=yes], + PARALLEL_FILTERED_WRITES=yes + AC_DEFINE([HAVE_PARALLEL_FILTERED_WRITES], [1], [Define if we have support for writing to filtered datasets in parallel])], [AC_MSG_RESULT([no]) - AC_MSG_WARN([A simple MPI program using the MPI_Mprobe and MPI_Imrecv functions could not be compiled and linked. + AC_MSG_WARN([A simple MPI program using the MPI_Ibarrier, MPI_Issend, MPI_Iprobe and MPI_Irecv functions could not be compiled and linked. Parallel writes of filtered data will be disabled.]) PARALLEL_FILTERED_WRITES=no] ) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 9ab870f..3f329c1 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -42,6 +42,14 @@ set (examples h5_vds-percival-unlim-maxmin ) +if (H5_HAVE_PARALLEL) + set (parallel_examples + ph5example + ph5_filtered_writes + ph5_filtered_writes_no_sel + ) +endif () + foreach (example ${examples}) add_executable (${example} ${HDF5_EXAMPLES_SOURCE_DIR}/${example}.c) target_include_directories (${example} PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") @@ -63,23 +71,25 @@ foreach (example ${examples}) endforeach () if (H5_HAVE_PARALLEL) - add_executable (ph5example ${HDF5_EXAMPLES_SOURCE_DIR}/ph5example.c) - target_include_directories (ph5example PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") - if (NOT BUILD_SHARED_LIBS) - TARGET_C_PROPERTIES (ph5example STATIC) - target_link_libraries (ph5example PRIVATE ${HDF5_LIB_TARGET} ${MPI_C_LIBRARIES}) - else () - TARGET_C_PROPERTIES (ph5example SHARED) - target_link_libraries (ph5example PRIVATE ${HDF5_LIBSH_TARGET} ${MPI_C_LIBRARIES}) - endif () - set_target_properties (ph5example PROPERTIES FOLDER examples) + foreach (parallel_example ${parallel_examples}) + add_executable (${parallel_example} ${HDF5_EXAMPLES_SOURCE_DIR}/${parallel_example}.c) + target_include_directories (${parallel_example} PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + if (NOT BUILD_SHARED_LIBS) + TARGET_C_PROPERTIES (${parallel_example} STATIC) + target_link_libraries (${parallel_example} PRIVATE ${HDF5_LIB_TARGET} ${MPI_C_LIBRARIES}) + else () + TARGET_C_PROPERTIES (${parallel_example} SHARED) + target_link_libraries (${parallel_example} PRIVATE ${HDF5_LIBSH_TARGET} ${MPI_C_LIBRARIES}) + endif () + set_target_properties (${parallel_example} PROPERTIES FOLDER examples) - #----------------------------------------------------------------------------- - # Add Target to clang-format - #----------------------------------------------------------------------------- - if (HDF5_ENABLE_FORMATTERS) - clang_format (HDF5_EXAMPLES_ph5example_FORMAT ph5example) - endif () + #----------------------------------------------------------------------------- + # Add Target to clang-format + #----------------------------------------------------------------------------- + if (HDF5_ENABLE_FORMATTERS) + clang_format (HDF5_EXAMPLES_${parallel_example}_FORMAT ${parallel_example}) + endif () + endforeach () endif () if (BUILD_TESTING AND HDF5_TEST_EXAMPLES) diff --git a/examples/CMakeTests.cmake b/examples/CMakeTests.cmake index 70142c8..3e24ba0 100644 --- a/examples/CMakeTests.cmake +++ b/examples/CMakeTests.cmake @@ -101,22 +101,26 @@ if (H5_HAVE_PARALLEL AND HDF5_TEST_PARALLEL AND NOT WIN32) # Ensure that 24 is a multiple of the number of processes. # The number 24 corresponds to SPACE1_DIM1 and SPACE1_DIM2 defined in ph5example.c math(EXPR NUMPROCS "24 / ((24 + ${MPIEXEC_MAX_NUMPROCS} - 1) / ${MPIEXEC_MAX_NUMPROCS})") - if (HDF5_ENABLE_USING_MEMCHECKER) - add_test (NAME MPI_TEST_EXAMPLES-ph5example COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${NUMPROCS} ${MPIEXEC_PREFLAGS} $ ${MPIEXEC_POSTFLAGS}) - else () - add_test (NAME MPI_TEST_EXAMPLES-ph5example COMMAND "${CMAKE_COMMAND}" - -D "TEST_PROGRAM=${MPIEXEC_EXECUTABLE};${MPIEXEC_NUMPROC_FLAG};${NUMPROCS};${MPIEXEC_PREFLAGS};$;${MPIEXEC_POSTFLAGS}" - -D "TEST_ARGS:STRING=" - -D "TEST_EXPECT=0" - -D "TEST_OUTPUT=ph5example.out" - -D "TEST_REFERENCE:STRING=PHDF5 tests finished with no errors" - -D "TEST_FILTER:STRING=PHDF5 tests finished with no errors" - -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" - -P "${HDF_RESOURCES_EXT_DIR}/grepTest.cmake" - ) - endif () - if (last_test) - set_tests_properties (MPI_TEST_EXAMPLES-ph5example PROPERTIES DEPENDS ${last_test}) - endif () - set (last_test "MPI_TEST_EXAMPLES-ph5example") + + foreach (parallel_example ${parallel_examples}) + if (HDF5_ENABLE_USING_MEMCHECKER) + add_test (NAME MPI_TEST_EXAMPLES-${parallel_example} COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${NUMPROCS} ${MPIEXEC_PREFLAGS} $ ${MPIEXEC_POSTFLAGS}) + else () + add_test (NAME MPI_TEST_EXAMPLES-${parallel_example} COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=${MPIEXEC_EXECUTABLE};${MPIEXEC_NUMPROC_FLAG};${NUMPROCS};${MPIEXEC_PREFLAGS};$;${MPIEXEC_POSTFLAGS}" + -D "TEST_ARGS:STRING=" + -D "TEST_EXPECT=0" + -D "TEST_SKIP_COMPARE=TRUE" + -D "TEST_OUTPUT=${parallel_example}.out" + -D "TEST_REFERENCE:STRING=PHDF5 example finished with no errors" + #-D "TEST_FILTER:STRING=PHDF5 tests finished with no errors" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}" + -P "${HDF_RESOURCES_EXT_DIR}/grepTest.cmake" + ) + endif () + if (last_test) + set_tests_properties (MPI_TEST_EXAMPLES-${parallel_example} PROPERTIES DEPENDS ${last_test}) + endif () + set (last_test "MPI_TEST_EXAMPLES-${parallel_example}") + endforeach () endif () diff --git a/examples/Makefile.am b/examples/Makefile.am index 7b5aa63..161f789 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -20,7 +20,7 @@ include $(top_srcdir)/config/commence.am if BUILD_PARALLEL_CONDITIONAL - EXAMPLE_PROG_PARA = ph5example + EXAMPLE_PROG_PARA = ph5example ph5_filtered_writes ph5_filtered_writes_no_sel endif INSTALL_SCRIPT_FILES = run-c-ex.sh @@ -50,7 +50,7 @@ INSTALL_FILES = h5_write.c h5_read.c h5_extend_write.c h5_chunk_read.c h5_compou h5_group.c h5_select.c h5_attribute.c h5_mount.c h5_drivers.c \ h5_reference_deprec.c h5_ref_extern.c h5_ref_compat.c h5_ref2reg_deprec.c \ h5_extlink.c h5_elink_unix2win.c h5_shared_mesg.c h5_debug_trace.c \ - ph5example.c \ + ph5example.c ph5_filtered_writes.c ph5_filtered_writes_no_sel.c \ h5_vds.c h5_vds-exc.c h5_vds-exclim.c h5_vds-eiger.c h5_vds-simpleIO.c \ h5_vds-percival.c h5_vds-percival-unlim.c h5_vds-percival-unlim-maxmin.c @@ -119,6 +119,8 @@ h5_reference_deprec: $(srcdir)/h5_reference_deprec.c h5_ref2reg_deprec: $(srcdir)/h5_ref2reg_deprec.c h5_drivers: $(srcdir)/h5_drivers.c ph5example: $(srcdir)/ph5example.c +ph5_filtered_writes: $(srcdir)/ph5_filtered_writes.c +ph5_filtered_writes_no_sel: $(srcdir)/ph5_filtered_writes_no_sel.c h5_dtransform: $(srcdir)/h5_dtransform.c h5_extlink: $(srcdir)/h5_extlink.c $(EXTLINK_DIRS) h5_elink_unix2win: $(srcdir)/h5_elink_unix2win.c $(EXTLINK_DIRS) diff --git a/examples/ph5_filtered_writes.c b/examples/ph5_filtered_writes.c new file mode 100644 index 0000000..8b55528 --- /dev/null +++ b/examples/ph5_filtered_writes.c @@ -0,0 +1,482 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Example of using the parallel HDF5 library to write to datasets + * with filters applied to them. + * + * If the HDF5_NOCLEANUP environment variable is set, the file that + * this example creates will not be removed as the example finishes. + * + * The need of requirement of parallel file prefix is that in general + * the current working directory in which compiling is done, is not suitable + * for parallel I/O and there is no standard pathname for parallel file + * systems. In some cases, the parallel file name may even need some + * parallel file type prefix such as: "pfs:/GF/...". Therefore, this + * example parses the HDF5_PARAPREFIX environment variable for a prefix, + * if one is needed. + */ + +#include + +#include "hdf5.h" + +#if defined(H5_HAVE_PARALLEL) && defined(H5_HAVE_PARALLEL_FILTERED_WRITES) + +#define EXAMPLE_FILE "ph5_filtered_writes.h5" +#define EXAMPLE_DSET1_NAME "DSET1" +#define EXAMPLE_DSET2_NAME "DSET2" + +#define EXAMPLE_DSET_DIMS 2 +#define EXAMPLE_DSET_CHUNK_DIM_SIZE 10 + +/* Dataset datatype */ +#define HDF5_DATATYPE H5T_NATIVE_INT +typedef int C_DATATYPE; + +/* Global variables */ +int mpi_rank, mpi_size; + +/* + * Routine to set an HDF5 filter on the given DCPL + */ +static void +set_filter(hid_t dcpl_id) +{ + htri_t filter_avail; + + /* + * Check if 'deflate' filter is available + */ + filter_avail = H5Zfilter_avail(H5Z_FILTER_DEFLATE); + if (filter_avail < 0) + return; + else if (filter_avail) { + /* + * Set 'deflate' filter with reasonable + * compression level on DCPL + */ + H5Pset_deflate(dcpl_id, 6); + } + else { + /* + * Set Fletcher32 checksum filter on DCPL + * since it is always available in HDF5 + */ + H5Pset_fletcher32(dcpl_id); + } +} + +/* + * Routine to fill a data buffer with data. Assumes + * dimension rank is 2 and data is stored contiguous. + */ +void +fill_databuf(hsize_t start[], hsize_t count[], hsize_t stride[], C_DATATYPE *data) +{ + C_DATATYPE *dataptr = data; + hsize_t i, j; + + /* Use MPI rank value for data */ + for (i = 0; i < count[0]; i++) { + for (j = 0; j < count[1]; j++) { + *dataptr++ = mpi_rank; + } + } +} + +/* Cleanup created file */ +static void +cleanup(char *filename) +{ + hbool_t do_cleanup = getenv(HDF5_NOCLEANUP) ? 0 : 1; + + if (do_cleanup) + MPI_File_delete(filename, MPI_INFO_NULL); +} + +/* + * Routine to write to a dataset in a fashion + * where no chunks in the dataset are written + * to by more than 1 MPI rank. This will + * generally give the best performance as the + * MPI ranks will need the least amount of + * inter-process communication. + */ +static void +write_dataset_no_overlap(hid_t file_id, hid_t dxpl_id) +{ + C_DATATYPE data[EXAMPLE_DSET_CHUNK_DIM_SIZE][4 * EXAMPLE_DSET_CHUNK_DIM_SIZE]; + hsize_t dataset_dims[EXAMPLE_DSET_DIMS]; + hsize_t chunk_dims[EXAMPLE_DSET_DIMS]; + hsize_t start[EXAMPLE_DSET_DIMS]; + hsize_t stride[EXAMPLE_DSET_DIMS]; + hsize_t count[EXAMPLE_DSET_DIMS]; + size_t i, j; + hid_t dset_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; + hid_t file_dataspace = H5I_INVALID_HID; + + /* + * ------------------------------------ + * Setup Dataset Creation Property List + * ------------------------------------ + */ + + dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + + /* + * REQUIRED: Dataset chunking must be enabled to + * apply a data filter to the dataset. + * Chunks in the dataset are of size + * EXAMPLE_DSET_CHUNK_DIM_SIZE x EXAMPLE_DSET_CHUNK_DIM_SIZE. + */ + chunk_dims[0] = EXAMPLE_DSET_CHUNK_DIM_SIZE; + chunk_dims[1] = EXAMPLE_DSET_CHUNK_DIM_SIZE; + H5Pset_chunk(dcpl_id, EXAMPLE_DSET_DIMS, chunk_dims); + + /* Set filter to be applied to created datasets */ + set_filter(dcpl_id); + + /* + * ------------------------------------ + * Define the dimensions of the dataset + * and create it + * ------------------------------------ + */ + + /* + * Create a dataset composed of 4 chunks + * per MPI rank. The first dataset dimension + * scales according to the number of MPI ranks. + * The second dataset dimension stays fixed + * according to the chunk size. + */ + dataset_dims[0] = EXAMPLE_DSET_CHUNK_DIM_SIZE * mpi_size; + dataset_dims[1] = 4 * EXAMPLE_DSET_CHUNK_DIM_SIZE; + + file_dataspace = H5Screate_simple(EXAMPLE_DSET_DIMS, dataset_dims, NULL); + + /* Create the dataset */ + dset_id = H5Dcreate2(file_id, EXAMPLE_DSET1_NAME, HDF5_DATATYPE, file_dataspace, H5P_DEFAULT, dcpl_id, + H5P_DEFAULT); + + /* + * ------------------------------------ + * Setup selection in the dataset for + * each MPI rank + * ------------------------------------ + */ + + /* + * Each MPI rank's selection covers a + * single chunk in the first dataset + * dimension. Each MPI rank's selection + * covers 4 chunks in the second dataset + * dimension. This leads to each MPI rank + * writing to 4 chunks of the dataset. + */ + start[0] = mpi_rank * EXAMPLE_DSET_CHUNK_DIM_SIZE; + start[1] = 0; + stride[0] = 1; + stride[1] = 1; + count[0] = EXAMPLE_DSET_CHUNK_DIM_SIZE; + count[1] = 4 * EXAMPLE_DSET_CHUNK_DIM_SIZE; + + H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, NULL); + + /* + * -------------------------------------- + * Fill data buffer with MPI rank's rank + * value to make it easy to see which + * part of the dataset each rank wrote to + * -------------------------------------- + */ + + fill_databuf(start, count, stride, &data[0][0]); + + /* + * --------------------------------- + * Write to the dataset collectively + * --------------------------------- + */ + + H5Dwrite(dset_id, HDF5_DATATYPE, H5S_BLOCK, file_dataspace, dxpl_id, data); + + /* + * -------------- + * Close HDF5 IDs + * -------------- + */ + + H5Sclose(file_dataspace); + H5Pclose(dcpl_id); + H5Dclose(dset_id); +} + +/* + * Routine to write to a dataset in a fashion + * where every chunk in the dataset is written + * to by every MPI rank. This will generally + * give the worst performance as the MPI ranks + * will need the most amount of inter-process + * communication. + */ +static void +write_dataset_overlap(hid_t file_id, hid_t dxpl_id) +{ + C_DATATYPE data[mpi_size][EXAMPLE_DSET_CHUNK_DIM_SIZE]; + hsize_t dataset_dims[EXAMPLE_DSET_DIMS]; + hsize_t chunk_dims[EXAMPLE_DSET_DIMS]; + hsize_t start[EXAMPLE_DSET_DIMS]; + hsize_t stride[EXAMPLE_DSET_DIMS]; + hsize_t count[EXAMPLE_DSET_DIMS]; + size_t i, j; + hid_t dset_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; + hid_t file_dataspace = H5I_INVALID_HID; + + /* + * ------------------------------------ + * Setup Dataset Creation Property List + * ------------------------------------ + */ + + dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + + /* + * REQUIRED: Dataset chunking must be enabled to + * apply a data filter to the dataset. + * Chunks in the dataset are of size + * mpi_size x EXAMPLE_DSET_CHUNK_DIM_SIZE. + */ + chunk_dims[0] = mpi_size; + chunk_dims[1] = EXAMPLE_DSET_CHUNK_DIM_SIZE; + H5Pset_chunk(dcpl_id, EXAMPLE_DSET_DIMS, chunk_dims); + + /* Set filter to be applied to created datasets */ + set_filter(dcpl_id); + + /* + * ------------------------------------ + * Define the dimensions of the dataset + * and create it + * ------------------------------------ + */ + + /* + * Create a dataset composed of N chunks, + * where N is the number of MPI ranks. The + * first dataset dimension scales according + * to the number of MPI ranks. The second + * dataset dimension stays fixed according + * to the chunk size. + */ + dataset_dims[0] = mpi_size * chunk_dims[0]; + dataset_dims[1] = EXAMPLE_DSET_CHUNK_DIM_SIZE; + + file_dataspace = H5Screate_simple(EXAMPLE_DSET_DIMS, dataset_dims, NULL); + + /* Create the dataset */ + dset_id = H5Dcreate2(file_id, EXAMPLE_DSET2_NAME, HDF5_DATATYPE, file_dataspace, H5P_DEFAULT, dcpl_id, + H5P_DEFAULT); + + /* + * ------------------------------------ + * Setup selection in the dataset for + * each MPI rank + * ------------------------------------ + */ + + /* + * Each MPI rank's selection covers + * part of every chunk in the first + * dimension. Each MPI rank's selection + * covers all of every chunk in the + * second dimension. This leads to + * each MPI rank writing an equal + * amount of data to every chunk + * in the dataset. + */ + start[0] = mpi_rank; + start[1] = 0; + stride[0] = chunk_dims[0]; + stride[1] = 1; + count[0] = mpi_size; + count[1] = EXAMPLE_DSET_CHUNK_DIM_SIZE; + + H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, NULL); + + /* + * -------------------------------------- + * Fill data buffer with MPI rank's rank + * value to make it easy to see which + * part of the dataset each rank wrote to + * -------------------------------------- + */ + + fill_databuf(start, count, stride, &data[0][0]); + + /* + * --------------------------------- + * Write to the dataset collectively + * --------------------------------- + */ + + H5Dwrite(dset_id, HDF5_DATATYPE, H5S_BLOCK, file_dataspace, dxpl_id, data); + + /* + * -------------- + * Close HDF5 IDs + * -------------- + */ + + H5Sclose(file_dataspace); + H5Pclose(dcpl_id); + H5Dclose(dset_id); +} + +int +main(int argc, char **argv) +{ + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info = MPI_INFO_NULL; + hid_t file_id = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + hid_t dxpl_id = H5I_INVALID_HID; + char * par_prefix = NULL; + char filename[PATH_MAX]; + + MPI_Init(&argc, &argv); + MPI_Comm_size(comm, &mpi_size); + MPI_Comm_rank(comm, &mpi_rank); + + /* + * ---------------------------------- + * Start parallel access to HDF5 file + * ---------------------------------- + */ + + /* Setup File Access Property List with parallel I/O access */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(fapl_id, comm, info); + + /* + * OPTIONAL: Set collective metadata reads on FAPL to allow + * parallel writes to filtered datasets to perform + * better at scale. While not strictly necessary, + * this is generally recommended. + */ + H5Pset_all_coll_metadata_ops(fapl_id, true); + + /* + * OPTIONAL: Set the latest file format version for HDF5 in + * order to gain access to different dataset chunk + * index types and better data encoding methods. + * While not strictly necessary, this is generally + * recommended. + */ + H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + + /* Parse any parallel prefix and create filename */ + par_prefix = getenv("HDF5_PARAPREFIX"); + + snprintf(filename, PATH_MAX, "%s%s%s", par_prefix ? par_prefix : "", par_prefix ? "/" : "", EXAMPLE_FILE); + + /* Create HDF5 file */ + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + + /* + * -------------------------------------- + * Setup Dataset Transfer Property List + * with collective I/O + * -------------------------------------- + */ + + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + + /* + * REQUIRED: Setup collective I/O for the dataset + * write operations. Parallel writes to + * filtered datasets MUST be collective, + * even if some ranks have no data to + * contribute to the write operation. + * + * Refer to the 'ph5_filtered_writes_no_sel' + * example to see how to setup a dataset + * write when one or more MPI ranks have + * no data to contribute to the write + * operation. + */ + H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE); + + /* + * -------------------------------- + * Create and write to each dataset + * -------------------------------- + */ + + /* + * Write to a dataset in a fashion where no + * chunks in the dataset are written to by + * more than 1 MPI rank. This will generally + * give the best performance as the MPI ranks + * will need the least amount of inter-process + * communication. + */ + write_dataset_no_overlap(file_id, dxpl_id); + + /* + * Write to a dataset in a fashion where + * every chunk in the dataset is written + * to by every MPI rank. This will generally + * give the worst performance as the MPI ranks + * will need the most amount of inter-process + * communication. + */ + write_dataset_overlap(file_id, dxpl_id); + + /* + * ------------------ + * Close all HDF5 IDs + * ------------------ + */ + + H5Pclose(dxpl_id); + H5Pclose(fapl_id); + H5Fclose(file_id); + + printf("PHDF5 example finished with no errors\n"); + + /* + * ------------------------------------ + * Cleanup created HDF5 file and finish + * ------------------------------------ + */ + + cleanup(filename); + + MPI_Finalize(); + + return 0; +} + +#else + +int +main(void) +{ + printf("HDF5 not configured with parallel support or parallel filtered writes are disabled!\n"); + return 0; +} + +#endif diff --git a/examples/ph5_filtered_writes_no_sel.c b/examples/ph5_filtered_writes_no_sel.c new file mode 100644 index 0000000..14c68c8 --- /dev/null +++ b/examples/ph5_filtered_writes_no_sel.c @@ -0,0 +1,366 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Example of using the parallel HDF5 library to collectively write to + * datasets with filters applied to them when one or MPI ranks do not + * have data to contribute to the dataset. + * + * If the HDF5_NOCLEANUP environment variable is set, the file that + * this example creates will not be removed as the example finishes. + * + * The need of requirement of parallel file prefix is that in general + * the current working directory in which compiling is done, is not suitable + * for parallel I/O and there is no standard pathname for parallel file + * systems. In some cases, the parallel file name may even need some + * parallel file type prefix such as: "pfs:/GF/...". Therefore, this + * example parses the HDF5_PARAPREFIX environment variable for a prefix, + * if one is needed. + */ + +#include + +#include "hdf5.h" + +#if defined(H5_HAVE_PARALLEL) && defined(H5_HAVE_PARALLEL_FILTERED_WRITES) + +#define EXAMPLE_FILE "ph5_filtered_writes_no_sel.h5" +#define EXAMPLE_DSET_NAME "DSET" + +#define EXAMPLE_DSET_DIMS 2 +#define EXAMPLE_DSET_CHUNK_DIM_SIZE 10 + +/* Dataset datatype */ +#define HDF5_DATATYPE H5T_NATIVE_INT +typedef int C_DATATYPE; + +/* Global variables */ +int mpi_rank, mpi_size; + +/* + * Routine to set an HDF5 filter on the given DCPL + */ +static void +set_filter(hid_t dcpl_id) +{ + htri_t filter_avail; + + /* + * Check if 'deflate' filter is available + */ + filter_avail = H5Zfilter_avail(H5Z_FILTER_DEFLATE); + if (filter_avail < 0) + return; + else if (filter_avail) { + /* + * Set 'deflate' filter with reasonable + * compression level on DCPL + */ + H5Pset_deflate(dcpl_id, 6); + } + else { + /* + * Set Fletcher32 checksum filter on DCPL + * since it is always available in HDF5 + */ + H5Pset_fletcher32(dcpl_id); + } +} + +/* + * Routine to fill a data buffer with data. Assumes + * dimension rank is 2 and data is stored contiguous. + */ +void +fill_databuf(hsize_t start[], hsize_t count[], hsize_t stride[], C_DATATYPE *data) +{ + C_DATATYPE *dataptr = data; + hsize_t i, j; + + /* Use MPI rank value for data */ + for (i = 0; i < count[0]; i++) { + for (j = 0; j < count[1]; j++) { + *dataptr++ = mpi_rank; + } + } +} + +/* Cleanup created file */ +static void +cleanup(char *filename) +{ + hbool_t do_cleanup = getenv(HDF5_NOCLEANUP) ? 0 : 1; + + if (do_cleanup) + MPI_File_delete(filename, MPI_INFO_NULL); +} + +/* + * Routine to write to a dataset in a fashion + * where no chunks in the dataset are written + * to by more than 1 MPI rank. This will + * generally give the best performance as the + * MPI ranks will need the least amount of + * inter-process communication. + */ +static void +write_dataset_some_no_sel(hid_t file_id, hid_t dxpl_id) +{ + C_DATATYPE data[EXAMPLE_DSET_CHUNK_DIM_SIZE][4 * EXAMPLE_DSET_CHUNK_DIM_SIZE]; + hsize_t dataset_dims[EXAMPLE_DSET_DIMS]; + hsize_t chunk_dims[EXAMPLE_DSET_DIMS]; + hsize_t start[EXAMPLE_DSET_DIMS]; + hsize_t stride[EXAMPLE_DSET_DIMS]; + hsize_t count[EXAMPLE_DSET_DIMS]; + hbool_t no_selection; + size_t i, j; + hid_t dset_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; + hid_t file_dataspace = H5I_INVALID_HID; + + /* + * ------------------------------------ + * Setup Dataset Creation Property List + * ------------------------------------ + */ + + dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + + /* + * REQUIRED: Dataset chunking must be enabled to + * apply a data filter to the dataset. + * Chunks in the dataset are of size + * EXAMPLE_DSET_CHUNK_DIM_SIZE x EXAMPLE_DSET_CHUNK_DIM_SIZE. + */ + chunk_dims[0] = EXAMPLE_DSET_CHUNK_DIM_SIZE; + chunk_dims[1] = EXAMPLE_DSET_CHUNK_DIM_SIZE; + H5Pset_chunk(dcpl_id, EXAMPLE_DSET_DIMS, chunk_dims); + + /* Set filter to be applied to created datasets */ + set_filter(dcpl_id); + + /* + * ------------------------------------ + * Define the dimensions of the dataset + * and create it + * ------------------------------------ + */ + + /* + * Create a dataset composed of 4 chunks + * per MPI rank. The first dataset dimension + * scales according to the number of MPI ranks. + * The second dataset dimension stays fixed + * according to the chunk size. + */ + dataset_dims[0] = EXAMPLE_DSET_CHUNK_DIM_SIZE * mpi_size; + dataset_dims[1] = 4 * EXAMPLE_DSET_CHUNK_DIM_SIZE; + + file_dataspace = H5Screate_simple(EXAMPLE_DSET_DIMS, dataset_dims, NULL); + + /* Create the dataset */ + dset_id = H5Dcreate2(file_id, EXAMPLE_DSET_NAME, HDF5_DATATYPE, file_dataspace, H5P_DEFAULT, dcpl_id, + H5P_DEFAULT); + + /* + * ------------------------------------ + * Setup selection in the dataset for + * each MPI rank + * ------------------------------------ + */ + + /* + * Odd rank value MPI ranks do not + * contribute any data to the dataset. + */ + no_selection = (mpi_rank % 2) == 1; + + if (no_selection) { + /* + * MPI ranks not contributing data to + * the dataset should call H5Sselect_none + * on the file dataspace that will be + * passed to H5Dwrite. + */ + H5Sselect_none(file_dataspace); + } + else { + /* + * Even MPI ranks contribute data to + * the dataset. Each MPI rank's selection + * covers a single chunk in the first dataset + * dimension. Each MPI rank's selection + * covers 4 chunks in the second dataset + * dimension. This leads to each contributing + * MPI rank writing to 4 chunks of the dataset. + */ + start[0] = mpi_rank * EXAMPLE_DSET_CHUNK_DIM_SIZE; + start[1] = 0; + stride[0] = 1; + stride[1] = 1; + count[0] = EXAMPLE_DSET_CHUNK_DIM_SIZE; + count[1] = 4 * EXAMPLE_DSET_CHUNK_DIM_SIZE; + + H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, NULL); + + /* + * -------------------------------------- + * Fill data buffer with MPI rank's rank + * value to make it easy to see which + * part of the dataset each rank wrote to + * -------------------------------------- + */ + + fill_databuf(start, count, stride, &data[0][0]); + } + + /* + * --------------------------------- + * Write to the dataset collectively + * --------------------------------- + */ + + H5Dwrite(dset_id, HDF5_DATATYPE, no_selection ? H5S_ALL : H5S_BLOCK, file_dataspace, dxpl_id, data); + + /* + * -------------- + * Close HDF5 IDs + * -------------- + */ + + H5Sclose(file_dataspace); + H5Pclose(dcpl_id); + H5Dclose(dset_id); +} + +int +main(int argc, char **argv) +{ + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info = MPI_INFO_NULL; + hid_t file_id = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + hid_t dxpl_id = H5I_INVALID_HID; + char * par_prefix = NULL; + char filename[PATH_MAX]; + + MPI_Init(&argc, &argv); + MPI_Comm_size(comm, &mpi_size); + MPI_Comm_rank(comm, &mpi_rank); + + /* + * ---------------------------------- + * Start parallel access to HDF5 file + * ---------------------------------- + */ + + /* Setup File Access Property List with parallel I/O access */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(fapl_id, comm, info); + + /* + * OPTIONAL: Set collective metadata reads on FAPL to allow + * parallel writes to filtered datasets to perform + * better at scale. While not strictly necessary, + * this is generally recommended. + */ + H5Pset_all_coll_metadata_ops(fapl_id, true); + + /* + * OPTIONAL: Set the latest file format version for HDF5 in + * order to gain access to different dataset chunk + * index types and better data encoding methods. + * While not strictly necessary, this is generally + * recommended. + */ + H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + + /* Parse any parallel prefix and create filename */ + par_prefix = getenv("HDF5_PARAPREFIX"); + + snprintf(filename, PATH_MAX, "%s%s%s", par_prefix ? par_prefix : "", par_prefix ? "/" : "", EXAMPLE_FILE); + + /* Create HDF5 file */ + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + + /* + * -------------------------------------- + * Setup Dataset Transfer Property List + * with collective I/O + * -------------------------------------- + */ + + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + + /* + * REQUIRED: Setup collective I/O for the dataset + * write operations. Parallel writes to + * filtered datasets MUST be collective, + * even if some ranks have no data to + * contribute to the write operation. + */ + H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE); + + /* + * -------------------------------- + * Create and write to the dataset + * -------------------------------- + */ + + /* + * Write to a dataset in a fashion where no + * chunks in the dataset are written to by + * more than 1 MPI rank and some MPI ranks + * have nothing to contribute to the dataset. + * In this case, the MPI ranks that have no + * data to contribute must still participate + * in the collective H5Dwrite call, but should + * call H5Sselect_none on the file dataspace + * passed to the H5Dwrite call. + */ + write_dataset_some_no_sel(file_id, dxpl_id); + + /* + * ------------------ + * Close all HDF5 IDs + * ------------------ + */ + + H5Pclose(dxpl_id); + H5Pclose(fapl_id); + H5Fclose(file_id); + + printf("PHDF5 example finished with no errors\n"); + + /* + * ------------------------------------ + * Cleanup created HDF5 file and finish + * ------------------------------------ + */ + + cleanup(filename); + + MPI_Finalize(); + + return 0; +} + +#else + +int +main(void) +{ + printf("HDF5 not configured with parallel support or parallel filtered writes are disabled!\n"); + return 0; +} + +#endif diff --git a/examples/ph5example.c b/examples/ph5example.c index 23af477..36fbfd5 100644 --- a/examples/ph5example.c +++ b/examples/ph5example.c @@ -1073,11 +1073,11 @@ main(int argc, char **argv) finish: if (mpi_rank == 0) { /* only process 0 reports */ if (nerrors) - printf("***PHDF5 tests detected %d errors***\n", nerrors); + printf("***PHDF5 example detected %d errors***\n", nerrors); else { - printf("===================================\n"); - printf("PHDF5 tests finished with no errors\n"); - printf("===================================\n"); + printf("=====================================\n"); + printf("PHDF5 example finished with no errors\n"); + printf("=====================================\n"); } } if (docleanup) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 10663f2..a165433 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -883,8 +883,46 @@ New Features Parallel Library: ----------------- - - + - Several improvements to parallel compression feature, including: + + * Improved support for collective I/O (for both writes and reads) + + * Significant reduction of memory usage for the feature as a whole + + * Reduction of copying of application data buffers passed to H5Dwrite + + * Addition of support for incremental file space allocation for filtered + datasets created in parallel. Incremental file space allocation is the + default for these types of datasets (early file space allocation is + also still supported), while early file space allocation is still the + default (and only supported allocation time) for unfiltered datasets + created in parallel. Incremental file space allocation should help with + parallel HDF5 applications that wish to use fill values on filtered + datasets, but would typically avoid doing so since dataset creation in + parallel would often take an excessive amount of time. Since these + datasets previously used early file space allocation, HDF5 would + allocate space for and write fill values to every chunk in the dataset + at creation time, leading to noticeable overhead. Instead, with + incremental file space allocation, allocation of file space for chunks + and writing of fill values to those chunks will be delayed until each + individual chunk is initially written to. + + * Addition of support for HDF5's "don't filter partial edge chunks" flag + (https://portal.hdfgroup.org/display/HDF5/H5P_SET_CHUNK_OPTS) + + * Addition of proper support for HDF5 fill values with the feature + + * Addition of 'H5_HAVE_PARALLEL_FILTERED_WRITES' macro to H5pubconf.h + so HDF5 applications can determine at compile-time whether the feature + is available + + * Addition of simple examples (ph5_filtered_writes.c and + ph5_filtered_writes_no_sel.c) under examples directory to demonstrate + usage of the feature + + * Improved coverage of regression testing for the feature + (JTH - 2022/2/23) Fortran Library: ---------------- diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 1b0e579..e4d8706 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -239,10 +239,14 @@ typedef struct H5D_chunk_file_iter_ud_t { #ifdef H5_HAVE_PARALLEL /* information to construct a collective I/O operation for filling chunks */ -typedef struct H5D_chunk_coll_info_t { - size_t num_io; /* Number of write operations */ - haddr_t *addr; /* array of the file addresses of the write operation */ -} H5D_chunk_coll_info_t; +typedef struct H5D_chunk_coll_fill_info_t { + size_t num_chunks; /* Number of chunks in the write operation */ + struct chunk_coll_fill_info { + haddr_t addr; /* File address of the chunk */ + size_t chunk_size; /* Size of the chunk in the file */ + hbool_t unfiltered_partial_chunk; + } * chunk_info; +} H5D_chunk_coll_fill_info_t; #endif /* H5_HAVE_PARALLEL */ typedef struct H5D_chunk_iter_ud_t { @@ -287,9 +291,6 @@ static int H5D__chunk_format_convert_cb(const H5D_chunk_rec_t *chunk_rec, void * /* Helper routines */ static herr_t H5D__chunk_set_info_real(H5O_layout_chunk_t *layout, unsigned ndims, const hsize_t *curr_dims, const hsize_t *max_dims); -static void * H5D__chunk_mem_alloc(size_t size, const H5O_pline_t *pline); -static void * H5D__chunk_mem_xfree(void *chk, const void *pline); -static void * H5D__chunk_mem_realloc(void *chk, size_t size, const H5O_pline_t *pline); static herr_t H5D__chunk_cinfo_cache_reset(H5D_chunk_cached_t *last); static herr_t H5D__chunk_cinfo_cache_update(H5D_chunk_cached_t *last, const H5D_chunk_ud_t *udata); static hbool_t H5D__chunk_cinfo_cache_found(const H5D_chunk_cached_t *last, H5D_chunk_ud_t *udata); @@ -306,8 +307,6 @@ static herr_t H5D__chunk_mem_cb(void *elem, const H5T_t *type, unsigned ndims, static unsigned H5D__chunk_hash_val(const H5D_shared_t *shared, const hsize_t *scaled); static herr_t H5D__chunk_flush_entry(const H5D_t *dset, H5D_rdcc_ent_t *ent, hbool_t reset); static herr_t H5D__chunk_cache_evict(const H5D_t *dset, H5D_rdcc_ent_t *ent, hbool_t flush); -static hbool_t H5D__chunk_is_partial_edge_chunk(unsigned dset_ndims, const uint32_t *chunk_dims, - const hsize_t *chunk_scaled, const hsize_t *dset_dims); static void * H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata, hbool_t relax, hbool_t prev_unfilt_chunk); static herr_t H5D__chunk_unlock(const H5D_io_info_t *io_info, const H5D_chunk_ud_t *udata, hbool_t dirty, @@ -315,9 +314,9 @@ static herr_t H5D__chunk_unlock(const H5D_io_info_t *io_info, const H5D_chunk_ static herr_t H5D__chunk_cache_prune(const H5D_t *dset, size_t size); static herr_t H5D__chunk_prune_fill(H5D_chunk_it_ud1_t *udata, hbool_t new_unfilt_chunk); #ifdef H5_HAVE_PARALLEL -static herr_t H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_info_t *chunk_info, - size_t chunk_size, const void *fill_buf); -static int H5D__chunk_cmp_addr(const void *addr1, const void *addr2); +static herr_t H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_fill_info, + const void *fill_buf, const void *partial_chunk_fill_buf); +static int H5D__chunk_cmp_coll_fill_info(const void *_entry1, const void *_entry2); #endif /* H5_HAVE_PARALLEL */ /* Debugging helper routine callback */ @@ -1362,7 +1361,7 @@ done: * *------------------------------------------------------------------------- */ -static void * +void * H5D__chunk_mem_alloc(size_t size, const H5O_pline_t *pline) { void *ret_value = NULL; /* Return value */ @@ -1393,7 +1392,7 @@ H5D__chunk_mem_alloc(size_t size, const H5O_pline_t *pline) * *------------------------------------------------------------------------- */ -static void * +void * H5D__chunk_mem_xfree(void *chk, const void *_pline) { const H5O_pline_t *pline = (const H5O_pline_t *)_pline; @@ -1417,7 +1416,7 @@ H5D__chunk_mem_xfree(void *chk, const void *_pline) * calls H5D__chunk_mem_xfree and discards the return value. *------------------------------------------------------------------------- */ -static void +void H5D__chunk_mem_free(void *chk, const void *_pline) { (void)H5D__chunk_mem_xfree(chk, _pline); @@ -1437,7 +1436,7 @@ H5D__chunk_mem_free(void *chk, const void *_pline) * *------------------------------------------------------------------------- */ -static void * +void * H5D__chunk_mem_realloc(void *chk, size_t size, const H5O_pline_t *pline) { void *ret_value = NULL; /* Return value */ @@ -4320,8 +4319,8 @@ H5D__chunk_allocate(const H5D_io_info_t *io_info, hbool_t full_overwrite, const hbool_t blocks_written = FALSE; /* Flag to indicate that chunk was actually written */ hbool_t using_mpi = FALSE; /* Flag to indicate that the file is being accessed with an MPI-capable file driver */ - H5D_chunk_coll_info_t chunk_info; /* chunk address information for doing I/O */ -#endif /* H5_HAVE_PARALLEL */ + H5D_chunk_coll_fill_info_t chunk_fill_info; /* chunk address information for doing I/O */ +#endif /* H5_HAVE_PARALLEL */ hbool_t carry; /* Flag to indicate that chunk increment carrys to higher dimension (sorta) */ unsigned space_ndims; /* Dataset's space rank */ const hsize_t * space_dim; /* Dataset's dataspace dimensions */ @@ -4368,8 +4367,8 @@ H5D__chunk_allocate(const H5D_io_info_t *io_info, hbool_t full_overwrite, const using_mpi = TRUE; /* init chunk info stuff for collective I/O */ - chunk_info.num_io = 0; - chunk_info.addr = NULL; + chunk_fill_info.num_chunks = 0; + chunk_fill_info.chunk_info = NULL; } /* end if */ #endif /* H5_HAVE_PARALLEL */ @@ -4641,19 +4640,26 @@ H5D__chunk_allocate(const H5D_io_info_t *io_info, hbool_t full_overwrite, const if (using_mpi) { /* collect all chunk addresses to be written to write collectively at the end */ - /* allocate/resize address array if no more space left */ - /* Note that if we add support for parallel filters we must - * also store an array of chunk sizes and pass it to the - * apporpriate collective write function */ - if (0 == chunk_info.num_io % 1024) - if (NULL == (chunk_info.addr = (haddr_t *)H5MM_realloc( - chunk_info.addr, (chunk_info.num_io + 1024) * sizeof(haddr_t)))) + + /* allocate/resize chunk info array if no more space left */ + if (0 == chunk_fill_info.num_chunks % 1024) { + void *tmp_realloc; + + if (NULL == (tmp_realloc = H5MM_realloc(chunk_fill_info.chunk_info, + (chunk_fill_info.num_chunks + 1024) * + sizeof(struct chunk_coll_fill_info)))) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, - "memory allocation failed for chunk addresses") + "memory allocation failed for chunk fill info") + + chunk_fill_info.chunk_info = tmp_realloc; + } - /* Store the chunk's address for later */ - chunk_info.addr[chunk_info.num_io] = udata.chunk_block.offset; - chunk_info.num_io++; + /* Store info about the chunk for later */ + chunk_fill_info.chunk_info[chunk_fill_info.num_chunks].addr = udata.chunk_block.offset; + chunk_fill_info.chunk_info[chunk_fill_info.num_chunks].chunk_size = chunk_size; + chunk_fill_info.chunk_info[chunk_fill_info.num_chunks].unfiltered_partial_chunk = + (*fill_buf == unfilt_fill_buf); + chunk_fill_info.num_chunks++; /* Indicate that blocks will be written */ blocks_written = TRUE; @@ -4726,7 +4732,7 @@ H5D__chunk_allocate(const H5D_io_info_t *io_info, hbool_t full_overwrite, const #ifdef H5_HAVE_PARALLEL /* do final collective I/O */ if (using_mpi && blocks_written) - if (H5D__chunk_collective_fill(dset, &chunk_info, chunk_size, fb_info.fill_buf) < 0) + if (H5D__chunk_collective_fill(dset, &chunk_fill_info, fb_info.fill_buf, unfilt_fill_buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file") #endif /* H5_HAVE_PARALLEL */ @@ -4742,8 +4748,8 @@ done: unfilt_fill_buf = H5D__chunk_mem_xfree(unfilt_fill_buf, &def_pline); #ifdef H5_HAVE_PARALLEL - if (using_mpi && chunk_info.addr) - H5MM_free(chunk_info.addr); + if (using_mpi && chunk_fill_info.chunk_info) + H5MM_free(chunk_fill_info.chunk_info); #endif FUNC_LEAVE_NOAPI(ret_value) @@ -4937,27 +4943,35 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_info_t *chunk_info, size_t chunk_size, - const void *fill_buf) +H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_fill_info_t *chunk_fill_info, + const void *fill_buf, const void *partial_chunk_fill_buf) { - MPI_Comm mpi_comm = MPI_COMM_NULL; /* MPI communicator for file */ - int mpi_rank = (-1); /* This process's rank */ - int mpi_size = (-1); /* MPI Comm size */ - int mpi_code; /* MPI return code */ - size_t num_blocks; /* Number of blocks between processes. */ - size_t leftover_blocks; /* Number of leftover blocks to handle */ - int blocks, leftover, block_len; /* converted to int for MPI */ + MPI_Comm mpi_comm = MPI_COMM_NULL; /* MPI communicator for file */ + int mpi_rank = (-1); /* This process's rank */ + int mpi_size = (-1); /* MPI Comm size */ + int mpi_code; /* MPI return code */ + size_t num_blocks; /* Number of blocks between processes. */ + size_t leftover_blocks; /* Number of leftover blocks to handle */ + int blocks, leftover; /* converted to int for MPI */ MPI_Aint * chunk_disp_array = NULL; + MPI_Aint * block_disps = NULL; int * block_lens = NULL; MPI_Datatype mem_type = MPI_BYTE, file_type = MPI_BYTE; H5FD_mpio_xfer_t prev_xfer_mode; /* Previous data xfer mode */ hbool_t have_xfer_mode = FALSE; /* Whether the previous xffer mode has been retrieved */ - hbool_t need_addr_sort = FALSE; - int i; /* Local index variable */ + hbool_t need_sort = FALSE; + size_t i; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC + /* + * If a separate fill buffer is provided for partial chunks, ensure + * that the "don't filter partial edge chunks" flag is set. + */ + if (partial_chunk_fill_buf) + HDassert(dset->shared->layout.u.chunk.flags & H5O_LAYOUT_CHUNK_DONT_FILTER_PARTIAL_BOUND_CHUNKS); + /* Get the MPI communicator */ if (MPI_COMM_NULL == (mpi_comm = H5F_mpi_get_comm(dset->oloc.file))) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI communicator") @@ -4973,39 +4987,89 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_info_t *chunk_info, /* Distribute evenly the number of blocks between processes. */ if (mpi_size == 0) HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "Resulted in division by zero") - num_blocks = (size_t)(chunk_info->num_io / (size_t)mpi_size); /* value should be the same on all procs */ + num_blocks = + (size_t)(chunk_fill_info->num_chunks / (size_t)mpi_size); /* value should be the same on all procs */ /* After evenly distributing the blocks between processes, are there any * leftover blocks for each individual process (round-robin)? */ - leftover_blocks = (size_t)(chunk_info->num_io % (size_t)mpi_size); + leftover_blocks = (size_t)(chunk_fill_info->num_chunks % (size_t)mpi_size); /* Cast values to types needed by MPI */ H5_CHECKED_ASSIGN(blocks, int, num_blocks, size_t); H5_CHECKED_ASSIGN(leftover, int, leftover_blocks, size_t); - H5_CHECKED_ASSIGN(block_len, int, chunk_size, size_t); /* Check if we have any chunks to write on this rank */ if (num_blocks > 0 || (leftover && leftover > mpi_rank)) { + MPI_Aint partial_fill_buf_disp = 0; + hbool_t all_same_block_len = TRUE; + /* Allocate buffers */ - /* (MSC - should not need block_lens if MPI_type_create_hindexed_block is working) */ - if (NULL == (block_lens = (int *)H5MM_malloc((size_t)(blocks + 1) * sizeof(int)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk lengths buffer") if (NULL == (chunk_disp_array = (MPI_Aint *)H5MM_malloc((size_t)(blocks + 1) * sizeof(MPI_Aint)))) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk file displacement buffer") - for (i = 0; i < blocks; i++) { - /* store the chunk address as an MPI_Aint */ - chunk_disp_array[i] = (MPI_Aint)(chunk_info->addr[i + (mpi_rank * blocks)]); + if (partial_chunk_fill_buf) { + MPI_Aint fill_buf_addr; + MPI_Aint partial_fill_buf_addr; + + /* Calculate the displacement between the fill buffer and partial chunk fill buffer */ + if (MPI_SUCCESS != (mpi_code = MPI_Get_address(fill_buf, &fill_buf_addr))) + HMPI_GOTO_ERROR(FAIL, "MPI_Get_address failed", mpi_code) + if (MPI_SUCCESS != (mpi_code = MPI_Get_address(partial_chunk_fill_buf, &partial_fill_buf_addr))) + HMPI_GOTO_ERROR(FAIL, "MPI_Get_address failed", mpi_code) - /* MSC - should not need this if MPI_type_create_hindexed_block is working */ - block_lens[i] = block_len; +#if MPI_VERSION >= 3 && MPI_SUBVERSION >= 1 + partial_fill_buf_disp = MPI_Aint_diff(partial_fill_buf_addr, fill_buf_addr); +#else + partial_fill_buf_disp = partial_fill_buf_addr - fill_buf_addr; +#endif - /* Make sure that the addresses in the datatype are - * monotonically non-decreasing + /* + * Allocate all-zero block displacements array. If a block's displacement + * is left as zero, that block will be written to from the regular fill + * buffer. If a block represents an unfiltered partial edge chunk, its + * displacement will be set so that the block is written to from the + * unfiltered fill buffer. */ - if (i && (chunk_disp_array[i] < chunk_disp_array[i - 1])) - need_addr_sort = TRUE; + if (NULL == (block_disps = (MPI_Aint *)H5MM_calloc((size_t)(blocks + 1) * sizeof(MPI_Aint)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate block displacements buffer") + } + + /* + * Perform initial scan of chunk info list to: + * - make sure that chunk addresses are monotonically non-decreasing + * - check if all blocks have the same length + */ + for (i = 1; i < chunk_fill_info->num_chunks; i++) { + if (chunk_fill_info->chunk_info[i].addr < chunk_fill_info->chunk_info[i - 1].addr) + need_sort = TRUE; + + if (chunk_fill_info->chunk_info[i].chunk_size != chunk_fill_info->chunk_info[i - 1].chunk_size) + all_same_block_len = FALSE; + } + + if (need_sort) + HDqsort(chunk_fill_info->chunk_info, chunk_fill_info->num_chunks, + sizeof(struct chunk_coll_fill_info), H5D__chunk_cmp_coll_fill_info); + + /* Allocate buffer for block lengths if necessary */ + if (!all_same_block_len) + if (NULL == (block_lens = (int *)H5MM_malloc((size_t)(blocks + 1) * sizeof(int)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk lengths buffer") + + for (i = 0; i < (size_t)blocks; i++) { + size_t idx = i + (size_t)(mpi_rank * blocks); + + /* store the chunk address as an MPI_Aint */ + chunk_disp_array[i] = (MPI_Aint)(chunk_fill_info->chunk_info[idx].addr); + + if (!all_same_block_len) + H5_CHECKED_ASSIGN(block_lens[i], int, chunk_fill_info->chunk_info[idx].chunk_size, size_t); + + if (chunk_fill_info->chunk_info[idx].unfiltered_partial_chunk) { + HDassert(partial_chunk_fill_buf); + block_disps[i] = partial_fill_buf_disp; + } } /* end for */ /* Calculate if there are any leftover blocks after evenly @@ -5013,32 +5077,71 @@ H5D__chunk_collective_fill(const H5D_t *dset, H5D_chunk_coll_info_t *chunk_info, * to processes 0 -> leftover. */ if (leftover && leftover > mpi_rank) { - chunk_disp_array[blocks] = (MPI_Aint)chunk_info->addr[(blocks * mpi_size) + mpi_rank]; - if (blocks && (chunk_disp_array[blocks] < chunk_disp_array[blocks - 1])) - need_addr_sort = TRUE; - block_lens[blocks] = block_len; + chunk_disp_array[blocks] = + (MPI_Aint)chunk_fill_info->chunk_info[(blocks * mpi_size) + mpi_rank].addr; + + if (!all_same_block_len) + H5_CHECKED_ASSIGN(block_lens[blocks], int, + chunk_fill_info->chunk_info[(blocks * mpi_size) + mpi_rank].chunk_size, + size_t); + + if (chunk_fill_info->chunk_info[(blocks * mpi_size) + mpi_rank].unfiltered_partial_chunk) { + HDassert(partial_chunk_fill_buf); + block_disps[blocks] = partial_fill_buf_disp; + } + blocks++; } - /* Ensure that the blocks are sorted in monotonically non-decreasing - * order of offset in the file. - */ - if (need_addr_sort) - HDqsort(chunk_disp_array, (size_t)blocks, sizeof(MPI_Aint), H5D__chunk_cmp_addr); + /* Create file and memory types for the write operation */ + if (all_same_block_len) { + int block_len; + + H5_CHECKED_ASSIGN(block_len, int, chunk_fill_info->chunk_info[0].chunk_size, size_t); + + mpi_code = + MPI_Type_create_hindexed_block(blocks, block_len, chunk_disp_array, MPI_BYTE, &file_type); + if (mpi_code != MPI_SUCCESS) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed_block failed", mpi_code) + + if (partial_chunk_fill_buf) { + /* + * If filters are disabled for partial edge chunks, those chunks could + * potentially have the same block length as the other chunks, but still + * need to be written to using the unfiltered fill buffer. Use an hindexed + * block type rather than an hvector. + */ + mpi_code = + MPI_Type_create_hindexed_block(blocks, block_len, block_disps, MPI_BYTE, &mem_type); + if (mpi_code != MPI_SUCCESS) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed_block failed", mpi_code) + } + else { + mpi_code = MPI_Type_create_hvector(blocks, block_len, 0, MPI_BYTE, &mem_type); + if (mpi_code != MPI_SUCCESS) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code) + } + } + else { + /* + * Currently, different block lengths implies that there are partial + * edge chunks and the "don't filter partial edge chunks" flag is set. + */ + HDassert(partial_chunk_fill_buf); + HDassert(block_lens); + HDassert(block_disps); + + mpi_code = MPI_Type_create_hindexed(blocks, block_lens, chunk_disp_array, MPI_BYTE, &file_type); + if (mpi_code != MPI_SUCCESS) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) + + mpi_code = MPI_Type_create_hindexed(blocks, block_lens, block_disps, MPI_BYTE, &mem_type); + if (mpi_code != MPI_SUCCESS) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) + } - /* MSC - should use this if MPI_type_create_hindexed block is working: - * mpi_code = MPI_Type_create_hindexed_block(blocks, block_len, chunk_disp_array, MPI_BYTE, - * &file_type); - */ - mpi_code = MPI_Type_create_hindexed(blocks, block_lens, chunk_disp_array, MPI_BYTE, &file_type); - if (mpi_code != MPI_SUCCESS) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&file_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - - mpi_code = MPI_Type_create_hvector(blocks, block_len, 0, MPI_BYTE, &mem_type); - if (mpi_code != MPI_SUCCESS) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code) if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&mem_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) } /* end if */ @@ -5081,39 +5184,25 @@ done: if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type))) HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) H5MM_xfree(chunk_disp_array); + H5MM_xfree(block_disps); H5MM_xfree(block_lens); FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__chunk_collective_fill() */ static int -H5D__chunk_cmp_addr(const void *addr1, const void *addr2) +H5D__chunk_cmp_coll_fill_info(const void *_entry1, const void *_entry2) { - MPI_Aint _addr1 = (MPI_Aint)0, _addr2 = (MPI_Aint)0; - int ret_value = 0; + const struct chunk_coll_fill_info *entry1; + const struct chunk_coll_fill_info *entry2; FUNC_ENTER_STATIC_NOERR - _addr1 = *((const MPI_Aint *)addr1); - _addr2 = *((const MPI_Aint *)addr2); + entry1 = (const struct chunk_coll_fill_info *)_entry1; + entry2 = (const struct chunk_coll_fill_info *)_entry2; -#if MPI_VERSION >= 3 && MPI_SUBVERSION >= 1 - { - MPI_Aint diff = MPI_Aint_diff(_addr1, _addr2); - - if (diff < (MPI_Aint)0) - ret_value = -1; - else if (diff > (MPI_Aint)0) - ret_value = 1; - else - ret_value = 0; - } -#else - ret_value = (_addr1 > _addr2) - (_addr1 < _addr2); -#endif - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__chunk_cmp_addr() */ + FUNC_LEAVE_NOAPI(H5F_addr_cmp(entry1->addr, entry2->addr)) +} /* end H5D__chunk_cmp_coll_fill_info() */ #endif /* H5_HAVE_PARALLEL */ /*------------------------------------------------------------------------- @@ -6827,7 +6916,7 @@ done: * *------------------------------------------------------------------------- */ -static hbool_t +hbool_t H5D__chunk_is_partial_edge_chunk(unsigned dset_ndims, const uint32_t *chunk_dims, const hsize_t scaled[], const hsize_t *dset_dims) { @@ -7122,6 +7211,89 @@ done: } /* end H5D__chunk_format_convert() */ /*------------------------------------------------------------------------- + * Function: H5D__chunk_index_empty_cb + * + * Purpose: Callback function that simply stops iteration and sets the + * `empty` parameter to FALSE if called. If this callback is + * entered, it means that the chunk index contains at least + * one chunk, so is not empty. + * + * Return: H5_ITER_STOP + * + *------------------------------------------------------------------------- + */ +static int +H5D__chunk_index_empty_cb(const H5D_chunk_rec_t H5_ATTR_UNUSED *chunk_rec, void *_udata) +{ + hbool_t *empty = (hbool_t *)_udata; + int ret_value = H5_ITER_STOP; + + FUNC_ENTER_STATIC_NOERR + + *empty = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__chunk_index_empty_cb() */ + +/*------------------------------------------------------------------------- + * Function: H5D__chunk_index_empty + * + * Purpose: Determines whether a chunk index is empty (has no chunks + * inserted into it yet). + * + * Note: This routine is meant to be a little more performant than + * just counting the number of chunks in the index. In the + * future, this is probably a callback that the chunk index + * ops structure should provide. + * + * Return: Non-negative on Success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__chunk_index_empty(const H5D_t *dset, hbool_t *empty) +{ + H5D_chk_idx_info_t idx_info; /* Chunked index info */ + H5D_rdcc_ent_t * ent; /* Cache entry */ + const H5D_rdcc_t * rdcc = NULL; /* Raw data chunk cache */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr) + + HDassert(dset); + HDassert(dset->shared); + HDassert(empty); + + rdcc = &(dset->shared->cache.chunk); /* raw data chunk cache */ + HDassert(rdcc); + + /* Search for cached chunks that haven't been written out */ + for (ent = rdcc->head; ent; ent = ent->next) + /* Flush the chunk out to disk, to make certain the size is correct later */ + if (H5D__chunk_flush_entry(dset, ent, FALSE) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer") + + /* Compose chunked index info struct */ + idx_info.f = dset->oloc.file; + idx_info.pline = &dset->shared->dcpl_cache.pline; + idx_info.layout = &dset->shared->layout.u.chunk; + idx_info.storage = &dset->shared->layout.storage.u.chunk; + + *empty = TRUE; + + if (H5F_addr_defined(idx_info.storage->idx_addr)) { + /* Iterate over the allocated chunks */ + if ((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__chunk_index_empty_cb, empty) < + 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "unable to retrieve allocated chunk information from index") + } + +done: + FUNC_LEAVE_NOAPI_TAG(ret_value) +} /* end H5D__chunk_index_empty() */ + +/*------------------------------------------------------------------------- * Function: H5D__get_num_chunks_cb * * Purpose: Callback function that increments the number of written diff --git a/src/H5Dint.c b/src/H5Dint.c index c9ea6bd..cc17265 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -378,40 +378,18 @@ H5D__get_space_status(const H5D_t *dset, H5D_space_status_t *allocation) /* Check for chunked layout */ if (dset->shared->layout.type == H5D_CHUNKED) { - hsize_t space_allocated; /* The number of bytes allocated for chunks */ - hssize_t snelmts; /* Temporary holder for number of elements in dataspace */ - hsize_t nelmts; /* Number of elements in dataspace */ - size_t dt_size; /* Size of datatype */ - hsize_t full_size; /* The number of bytes in the dataset when fully populated */ - - /* For chunked layout set the space status by the storage size */ - /* Get the dataset's dataspace */ - HDassert(dset->shared->space); - - /* Get the total number of elements in dataset's dataspace */ - if ((snelmts = H5S_GET_EXTENT_NPOINTS(dset->shared->space)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve number of elements in dataspace") - nelmts = (hsize_t)snelmts; - - /* Get the size of the dataset's datatype */ - if (0 == (dt_size = H5T_GET_SIZE(dset->shared->type))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve size of datatype") - - /* Compute the maximum size of the dataset in bytes */ - full_size = nelmts * dt_size; - - /* Check for overflow during multiplication */ - if (nelmts != (full_size / dt_size)) - HGOTO_ERROR(H5E_DATASET, H5E_OVERFLOW, FAIL, "size of dataset's storage overflowed") - - /* Difficult to error check, since the error value is 0 and 0 is a valid value... :-/ */ - if (H5D__get_storage_size(dset, &space_allocated) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get size of dataset's storage") - - /* Decide on how much of the space is allocated */ - if (space_allocated == 0) + hsize_t n_chunks_total = dset->shared->layout.u.chunk.nchunks; + hsize_t n_chunks_alloc = 0; + + if (H5D__get_num_chunks(dset, dset->shared->space, &n_chunks_alloc) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "unable to retrieve number of allocated chunks in dataset") + + HDassert(n_chunks_alloc <= n_chunks_total); + + if (n_chunks_alloc == 0) *allocation = H5D_SPACE_STATUS_NOT_ALLOCATED; - else if (space_allocated == full_size) + else if (n_chunks_alloc == n_chunks_total) *allocation = H5D_SPACE_STATUS_ALLOCATED; else *allocation = H5D_SPACE_STATUS_PART_ALLOCATED; @@ -1301,10 +1279,19 @@ H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id, hid_t HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set latest indexing") } /* end if */ - /* Check if this dataset is going into a parallel file and set space allocation time */ + /* Check if the file driver would like to force early space allocation */ if (H5F_HAS_FEATURE(file, H5FD_FEAT_ALLOCATE_EARLY)) new_dset->shared->dcpl_cache.fill.alloc_time = H5D_ALLOC_TIME_EARLY; + /* + * Check if this dataset is going into a parallel file and set space allocation time. + * If the dataset has filters applied to it, writes to the dataset must be collective, + * so we don't need to force early space allocation. Otherwise, we force early space + * allocation to facilitate independent raw data operations. + */ + if (H5F_HAS_FEATURE(file, H5FD_FEAT_HAS_MPI) && (new_dset->shared->dcpl_cache.pline.nused == 0)) + new_dset->shared->dcpl_cache.fill.alloc_time = H5D_ALLOC_TIME_EARLY; + /* Set the dataset's I/O operations */ if (H5D__layout_set_io_ops(new_dset) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to initialize I/O operations") diff --git a/src/H5Dio.c b/src/H5Dio.c index 1ea3f07..e226a0a 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -300,6 +300,7 @@ H5D__write(H5D_t *dataset, hid_t mem_type_id, H5S_t *mem_space, H5S_t *file_spac H5D_io_info_t io_info; /* Dataset I/O info */ H5D_type_info_t type_info; /* Datatype info for operation */ hbool_t type_info_init = FALSE; /* Whether the datatype info has been initialized */ + hbool_t should_alloc_space = FALSE; /* Whether or not to initialize dataset's storage */ H5S_t * projected_mem_space = NULL; /* If not NULL, ptr to dataspace containing a */ /* projection of the supplied mem_space to a new */ /* dataspace with rank equal to that of */ @@ -432,8 +433,20 @@ H5D__write(H5D_t *dataset, hid_t mem_type_id, H5S_t *mem_space, H5S_t *file_spac HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set up I/O operation") /* Allocate dataspace and initialize it if it hasn't been. */ - if (nelmts > 0 && dataset->shared->dcpl_cache.efl.nused == 0 && - !(*dataset->shared->layout.ops->is_space_alloc)(&dataset->shared->layout.storage)) { + should_alloc_space = dataset->shared->dcpl_cache.efl.nused == 0 && + !(*dataset->shared->layout.ops->is_space_alloc)(&dataset->shared->layout.storage); + + /* + * If not using an MPI-based VFD, we only need to allocate + * and initialize storage if there's a selection in the + * dataset's dataspace. Otherwise, we always need to participate + * in the storage allocation since this may use collective + * operations and we will hang if we don't participate. + */ + if (!H5F_HAS_FEATURE(dataset->oloc.file, H5FD_FEAT_HAS_MPI)) + should_alloc_space = should_alloc_space && (nelmts > 0); + + if (should_alloc_space) { hssize_t file_nelmts; /* Number of elements in file dataset's dataspace */ hbool_t full_overwrite; /* Whether we are over-writing all the elements */ @@ -808,98 +821,35 @@ H5D__ioinfo_adjust(H5D_io_info_t *io_info, const H5D_t *dset, const H5S_t *file_ io_info->io_ops.single_write = H5D__mpio_select_write; } /* end if */ else { - int comm_size = 0; - - /* Retrieve size of MPI communicator used for file */ - if ((comm_size = H5F_shared_mpi_get_size(io_info->f_sh)) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get MPI communicator size") - /* Check if there are any filters in the pipeline. If there are, * we cannot break to independent I/O if this is a write operation * with multiple ranks involved; otherwise, there will be metadata * inconsistencies in the file. */ - if (comm_size > 1 && io_info->op_type == H5D_IO_OP_WRITE && - io_info->dset->shared->dcpl_cache.pline.nused > 0) { - H5D_mpio_no_collective_cause_t cause; - uint32_t local_no_collective_cause; - uint32_t global_no_collective_cause; - hbool_t local_error_message_previously_written = FALSE; - hbool_t global_error_message_previously_written = FALSE; - size_t idx; - size_t cause_strings_len; - char local_no_collective_cause_string[512] = ""; - char global_no_collective_cause_string[512] = ""; - const char * cause_strings[] = { - "independent I/O was requested", - "datatype conversions were required", - "data transforms needed to be applied", - "optimized MPI types flag wasn't set", - "one of the dataspaces was neither simple nor scalar", - "dataset was not contiguous or chunked", - "parallel writes to filtered datasets are disabled", - "an error occurred while checking if collective I/O was possible"}; - - cause_strings_len = sizeof(cause_strings) / sizeof(cause_strings[0]); - - if (H5CX_get_mpio_local_no_coll_cause(&local_no_collective_cause) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, - "unable to get local no collective cause value") - if (H5CX_get_mpio_global_no_coll_cause(&global_no_collective_cause) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, - "unable to get global no collective cause value") - - /* Append each of the "reason for breaking collective I/O" error messages to the - * local and global no collective cause strings */ - for (cause = 1, idx = 0; - (cause < H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE) && (idx < cause_strings_len); - cause <<= 1, idx++) { - if (cause & local_no_collective_cause) { - size_t local_buffer_space = sizeof(local_no_collective_cause_string) - - HDstrlen(local_no_collective_cause_string) - 1; - - /* Check if there were any previous error messages included. If so, prepend a - * semicolon to separate the messages. - */ - if (local_buffer_space && local_error_message_previously_written) { - HDstrncat(local_no_collective_cause_string, "; ", local_buffer_space); - local_buffer_space -= MIN(local_buffer_space, 2); - } - - if (local_buffer_space) - HDstrncat(local_no_collective_cause_string, cause_strings[idx], - local_buffer_space); - - local_error_message_previously_written = TRUE; - } /* end if */ - - if (cause & global_no_collective_cause) { - size_t global_buffer_space = sizeof(global_no_collective_cause_string) - - HDstrlen(global_no_collective_cause_string) - 1; - - /* Check if there were any previous error messages included. If so, prepend a - * semicolon to separate the messages. - */ - if (global_buffer_space && global_error_message_previously_written) { - HDstrncat(global_no_collective_cause_string, "; ", global_buffer_space); - global_buffer_space -= MIN(global_buffer_space, 2); - } - - if (global_buffer_space) - HDstrncat(global_no_collective_cause_string, cause_strings[idx], - global_buffer_space); - - global_error_message_previously_written = TRUE; - } /* end if */ - } /* end for */ - - HGOTO_ERROR(H5E_IO, H5E_NO_INDEPENDENT, FAIL, - "Can't perform independent write with filters in pipeline.\n" - " The following caused a break from collective I/O:\n" - " Local causes: %s\n" - " Global causes: %s", - local_no_collective_cause_string, global_no_collective_cause_string); - } /* end if */ + if (io_info->op_type == H5D_IO_OP_WRITE && io_info->dset->shared->dcpl_cache.pline.nused > 0) { + int comm_size = 0; + + /* Retrieve size of MPI communicator used for file */ + if ((comm_size = H5F_shared_mpi_get_size(io_info->f_sh)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get MPI communicator size") + + if (comm_size > 1) { + char local_no_coll_cause_string[512]; + char global_no_coll_cause_string[512]; + + if (H5D__mpio_get_no_coll_cause_strings(local_no_coll_cause_string, 512, + global_no_coll_cause_string, 512) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "can't get reasons for breaking collective I/O") + + HGOTO_ERROR(H5E_IO, H5E_NO_INDEPENDENT, FAIL, + "Can't perform independent write with filters in pipeline.\n" + " The following caused a break from collective I/O:\n" + " Local causes: %s\n" + " Global causes: %s", + local_no_coll_cause_string, global_no_coll_cause_string); + } + } /* If we won't be doing collective I/O, but the user asked for * collective I/O, change the request to use independent I/O diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c index ce790c4..527fc7b 100644 --- a/src/H5Dmpio.c +++ b/src/H5Dmpio.c @@ -36,6 +36,7 @@ #include "H5Eprivate.h" /* Error handling */ #include "H5Fprivate.h" /* File access */ #include "H5FDprivate.h" /* File drivers */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory management */ #include "H5Oprivate.h" /* Object headers */ @@ -43,6 +44,15 @@ #include "H5Sprivate.h" /* Dataspaces */ #include "H5VMprivate.h" /* Vector */ +/* uthash is an external, header-only hash table implementation. + * + * We include the file directly in src/ and #define a few functions + * to use our internal memory calls. + */ +#define uthash_malloc(sz) H5MM_malloc(sz) +#define uthash_free(ptr, sz) H5MM_free(ptr) /* Ignoring sz is intentional */ +#include "uthash.h" + #ifdef H5_HAVE_PARALLEL /****************/ @@ -81,9 +91,54 @@ /* Macros to represent the regularity of the selection for multiple chunk IO case. */ #define H5D_CHUNK_SELECT_REG 1 +/* + * Threshold value for redistributing shared filtered chunks + * on all MPI ranks, or just MPI rank 0 + */ +#define H5D_CHUNK_REDISTRIBUTE_THRES ((size_t)((25 * H5_MB) / sizeof(H5D_chunk_redistribute_info_t))) + +/* + * Initial allocation size for the arrays that hold + * buffers for chunk modification data that is sent + * to other ranks and the MPI_Request objects for + * those send operations + */ +#define H5D_CHUNK_NUM_SEND_MSGS_INIT 64 + +/* + * Define a tag value for the MPI messages sent/received for + * chunk modification data + */ +#define H5D_CHUNK_MOD_DATA_TAG 64 + +/* + * Macro to initialize a H5D_chk_idx_info_t + * structure, given a pointer to a H5D_io_info_t + * structure + */ +#define H5D_MPIO_INIT_CHUNK_IDX_INFO(index_info, io_info_ptr) \ + do { \ + index_info.f = (io_info_ptr)->dset->oloc.file; \ + index_info.pline = &((io_info_ptr)->dset->shared->dcpl_cache.pline); \ + index_info.layout = &((io_info_ptr)->dset->shared->layout.u.chunk); \ + index_info.storage = &((io_info_ptr)->dset->shared->layout.storage.u.chunk); \ + } while (0) + +/* + * Macro to initialize a H5D_chunk_ud_t structure + * given a pointer to a H5D_chk_idx_info_t structure + */ +#define H5D_MPIO_INIT_CHUNK_UD_INFO(chunk_ud, index_info_ptr) \ + do { \ + HDmemset(&chunk_ud, 0, sizeof(H5D_chunk_ud_t)); \ + chunk_ud.common.layout = (index_info_ptr)->layout; \ + chunk_ud.common.storage = (index_info_ptr)->storage; \ + } while (0) + /******************/ /* Local Typedefs */ /******************/ + /* Combine chunk address and chunk info into a struct for better performance. */ typedef struct H5D_chunk_addr_info_t { haddr_t chunk_addr; @@ -100,115 +155,137 @@ typedef enum H5D_mpio_no_rank0_bcast_cause_t { } H5D_mpio_no_rank0_bcast_cause_t; /* + * Information necessary for re-allocating file space for a chunk + * during a parallel write of a chunked dataset with filters + * applied. + */ +typedef struct H5D_chunk_alloc_info_t { + H5F_block_t chunk_current; + H5F_block_t chunk_new; + hsize_t chunk_idx; +} H5D_chunk_alloc_info_t; + +/* + * Information for a chunk pertaining to the dataset's chunk + * index entry for the chunk + */ +typedef struct H5D_chunk_index_info_t { + hsize_t chunk_idx; + unsigned filter_mask; + hbool_t need_insert; +} H5D_chunk_index_info_t; + +/* * Information about a single chunk when performing collective filtered I/O. All * of the fields of one of these structs are initialized at the start of collective - * filtered I/O in the function H5D__construct_filtered_io_info_list(). - * - * This struct's fields are as follows: - * - * index - The "Index" of the chunk in the dataset. The index of a chunk is used during - * the collective re-insertion of chunks into the chunk index after the collective - * I/O has been performed. - * - * scaled - The scaled coordinates of the chunk in the dataset's file dataspace. The - * coordinates are used in both the collective re-allocation of space in the file - * and the collective re-insertion of chunks into the chunk index after the collective - * I/O has been performed. + * filtered I/O in the function H5D__mpio_collective_filtered_chunk_io_setup(). This + * struct's fields are as follows: * - * full_overwrite - A flag which determines whether or not a chunk needs to be read from the - * file when being updated. If a chunk is being fully overwritten (the entire - * extent is selected in its file dataspace), then it is not necessary to - * read the chunk from the file. However, if the chunk is not being fully - * overwritten, it has to be read from the file in order to update the chunk - * without trashing the parts of the chunk that are not selected. + * index_info - A structure containing the information needed when collectively + * re-inserting the chunk into the dataset's chunk index. The structure + * is distributed to all ranks during the re-insertion operation. Its fields + * are as follows: * - * num_writers - The total number of processors writing to this chunk. This field is used - * when the new owner of a chunk is receiving messages, which contain selections in - * the chunk and data to update the chunk with, from other processors which have this - * chunk selected in the I/O operation. The new owner must know how many processors it - * should expect messages from so that it can post an equal number of receive calls. + * chunk_idx - The index of the chunk in the dataset's chunk index. * - * io_size - The total size of I/O to this chunk. This field is an accumulation of the size of - * I/O to the chunk from each processor which has the chunk selected and is used to - * determine the value for the previous full_overwrite flag. + * filter_mask - A bit-mask that indicates which filters are to be applied to the + * chunk. Each filter in a chunk's filter pipeline has a bit position + * that can be masked to disable that particular filter for the chunk. + * This filter mask is saved alongside the chunk in the file. * - * buf - A pointer which serves the dual purpose of holding either the chunk data which is to be - * written to the file or the chunk data which has been read from the file. + * need_insert - A flag which determines whether or not a chunk needs to be re-inserted into + * the chunk index after the write operation. * - * chunk_states - In the case of dataset writes only, this struct is used to track a chunk's size and - * address in the file before and after the filtering operation has occurred. + * chunk_info - A pointer to the chunk's H5D_chunk_info_t structure, which contains useful + * information like the dataspaces containing the selection in the chunk. * - * Its fields are as follows: + * chunk_current - The address in the file and size of this chunk before the filtering + * operation. When reading a chunk from the file, this field is used to + * read the correct amount of bytes. It is also used when redistributing + * shared chunks among MPI ranks and as a parameter to the chunk file + * space reallocation function. * - * chunk_current - The address in the file and size of this chunk before the filtering - * operation. When reading a chunk from the file, this field is used to - * read the correct amount of bytes. It is also used when redistributing - * shared chunks among processors and as a parameter to the chunk file - * space reallocation function. + * chunk_new - The address in the file and size of this chunk after the filtering + * operation. This field is relevant when collectively re-allocating space + * in the file for all of the chunks written to in the I/O operation, as + * their sizes may have changed after their data has been filtered. * - * new_chunk - The address in the file and size of this chunk after the filtering - * operation. This field is relevant when collectively re-allocating space - * in the file for all of the chunks written to in the I/O operation, as - * their sizes may have changed after their data has been filtered. + * need_read - A flag which determines whether or not a chunk needs to be read from the + * file. During writes, if a chunk is being fully overwritten (the entire extent + * is selected in its file dataspace), then it is not necessary to read the chunk + * from the file. However, if the chunk is not being fully overwritten, it has to + * be read from the file in order to update the chunk without trashing the parts + * of the chunk that are not selected. During reads, this field should generally + * be true, but may be false if the chunk isn't allocated, for example. * - * owners - In the case of dataset writes only, this struct is used to manage which single processor - * will ultimately write data out to the chunk. It allows the other processors to act according - * to the decision and send their selection in the chunk, as well as the data they wish - * to update the chunk with, to the processor which is writing to the chunk. + * skip_filter_pline - A flag which determines whether to skip calls to the filter pipeline + * for this chunk. This flag is mostly useful for correct handling of + * partial edge chunks when the "don't filter partial edge chunks" flag + * is set on the dataset's DCPL. * - * Its fields are as follows: + * io_size - The total size of I/O to this chunk. This field is an accumulation of the size of + * I/O to the chunk from each MPI rank which has the chunk selected and is used to + * determine the value for the previous `full_overwrite` flag. * - * original_owner - The processor which originally had this chunk selected at the beginning of - * the collective filtered I/O operation. This field is currently used when - * redistributing shared chunks among processors. + * chunk_buf_size - The size in bytes of the data buffer allocated for the chunk * - * new_owner - The processor which has been selected to perform the write to this chunk. + * orig_owner - The MPI rank which originally had this chunk selected at the beginning of + * the collective filtered I/O operation. This field is currently used when + * redistributing shared chunks among MPI ranks. * - * async_info - In the case of dataset writes only, this struct is used by the owning processor of the - * chunk in order to manage the MPI send and receive calls made between it and all of - * the other processors which have this chunk selected in the I/O operation. + * new_owner - The MPI rank which has been selected to perform the modifications to this chunk. * - * Its fields are as follows: + * num_writers - The total number of MPI ranks writing to this chunk. This field is used when + * the new owner of a chunk is receiving messages from other MPI ranks that + * contain their selections in the chunk and the data to update the chunk with. + * The new owner must know how many MPI ranks it should expect messages from so + * that it can post an equal number of receive calls. * - * receive_requests_array - An array containing one MPI_Request for each of the - * asynchronous MPI receive calls the owning processor of this - * chunk makes to another processor in order to receive that - * processor's chunk modification data and selection in the chunk. + * buf - A pointer which serves the dual purpose of holding either the chunk data which is to be + * written to the file or the chunk data which has been read from the file. * - * receive_buffer_array - An array of buffers into which the owning processor of this chunk - * will store chunk modification data and the selection in the chunk - * received from another processor. + * hh - A handle for hash tables provided by the uthash.h header * - * num_receive_requests - The number of entries in the receive_request_array and - * receive_buffer_array fields. */ typedef struct H5D_filtered_collective_io_info_t { - hsize_t index; - hsize_t scaled[H5O_LAYOUT_NDIMS]; - hbool_t full_overwrite; - size_t num_writers; - size_t io_size; - void * buf; - - struct { - H5F_block_t chunk_current; - H5F_block_t new_chunk; - } chunk_states; - - struct { - int original_owner; - int new_owner; - } owners; - - struct { - MPI_Request * receive_requests_array; - unsigned char **receive_buffer_array; - int num_receive_requests; - } async_info; + H5D_chunk_index_info_t index_info; + + H5D_chunk_info_t *chunk_info; + H5F_block_t chunk_current; + H5F_block_t chunk_new; + hbool_t need_read; + hbool_t skip_filter_pline; + size_t io_size; + size_t chunk_buf_size; + int orig_owner; + int new_owner; + int num_writers; + void * buf; + + UT_hash_handle hh; } H5D_filtered_collective_io_info_t; -/* Function pointer typedef for sort function */ -typedef int (*H5D_mpio_sort_func_cb_t)(const void *, const void *); +/* + * Information necessary for redistributing shared chunks during + * a parallel write of a chunked dataset with filters applied. + */ +typedef struct H5D_chunk_redistribute_info_t { + H5F_block_t chunk_block; + hsize_t chunk_idx; + int orig_owner; + int new_owner; + int num_writers; +} H5D_chunk_redistribute_info_t; + +/* + * Information used when re-inserting a chunk into a dataset's + * chunk index during a parallel write of a chunked dataset with + * filters applied. + */ +typedef struct H5D_chunk_insert_info_t { + H5F_block_t chunk_block; + H5D_chunk_index_info_t index_info; +} H5D_chunk_insert_info_t; /********************/ /* Local Prototypes */ @@ -216,53 +293,98 @@ typedef int (*H5D_mpio_sort_func_cb_t)(const void *, const void *); static herr_t H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, H5D_chunk_map_t *fm); static herr_t H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - H5D_chunk_map_t *fm); + H5D_chunk_map_t *fm, int mpi_rank, int mpi_size); static herr_t H5D__multi_chunk_filtered_collective_io(H5D_io_info_t * io_info, - const H5D_type_info_t *type_info, H5D_chunk_map_t *fm); + const H5D_type_info_t *type_info, H5D_chunk_map_t *fm, + int mpi_rank, int mpi_size); static herr_t H5D__link_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - H5D_chunk_map_t *fm, int sum_chunk); + H5D_chunk_map_t *fm, int sum_chunk, int mpi_rank, int mpi_size); static herr_t H5D__link_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - H5D_chunk_map_t *fm); + H5D_chunk_map_t *fm, int mpi_rank, int mpi_size); static herr_t H5D__inter_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, const H5S_t *file_space, const H5S_t *mem_space); static herr_t H5D__final_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, MPI_Datatype mpi_file_type, MPI_Datatype mpi_buf_type); static herr_t H5D__sort_chunk(H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, - H5D_chunk_addr_info_t chunk_addr_info_array[], int many_chunk_opt); + H5D_chunk_addr_info_t chunk_addr_info_array[], int many_chunk_opt, int mpi_rank, + int mpi_size); static herr_t H5D__obtain_mpio_mode(H5D_io_info_t *io_info, H5D_chunk_map_t *fm, uint8_t assign_io_mode[], - haddr_t chunk_addr[]); + haddr_t chunk_addr[], int mpi_rank, int mpi_size); static herr_t H5D__mpio_get_sum_chunk(const H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, int *sum_chunkf); -static herr_t H5D__construct_filtered_io_info_list(const H5D_io_info_t * io_info, - const H5D_type_info_t * type_info, - const H5D_chunk_map_t * fm, - H5D_filtered_collective_io_info_t **chunk_list, - size_t * num_entries); -#if MPI_VERSION >= 3 -static herr_t H5D__chunk_redistribute_shared_chunks(const H5D_io_info_t * io_info, - const H5D_type_info_t * type_info, - const H5D_chunk_map_t * fm, - H5D_filtered_collective_io_info_t *local_chunk_array, - size_t *local_chunk_array_num_entries); -#endif -static herr_t H5D__mpio_array_gatherv(void *local_array, size_t local_array_num_entries, - size_t array_entry_size, void **gathered_array, - size_t *gathered_array_num_entries, hbool_t allgather, int root, - MPI_Comm comm, int (*sort_func)(const void *, const void *)); -static herr_t H5D__mpio_filtered_collective_write_type(H5D_filtered_collective_io_info_t *chunk_list, - size_t num_entries, MPI_Datatype *new_mem_type, - hbool_t *mem_type_derived, MPI_Datatype *new_file_type, - hbool_t *file_type_derived); -static herr_t H5D__filtered_collective_chunk_entry_io(H5D_filtered_collective_io_info_t *chunk_entry, - const H5D_io_info_t * io_info, - const H5D_type_info_t * type_info, - const H5D_chunk_map_t * fm); +static herr_t H5D__mpio_collective_filtered_chunk_io_setup(const H5D_io_info_t * io_info, + const H5D_type_info_t * type_info, + const H5D_chunk_map_t * fm, + H5D_filtered_collective_io_info_t **chunk_list, + size_t *num_entries, int mpi_rank); +static herr_t H5D__mpio_redistribute_shared_chunks(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, + const H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, + int mpi_rank, int mpi_size, + size_t **rank_chunks_assigned_map); +static herr_t H5D__mpio_redistribute_shared_chunks_int(H5D_filtered_collective_io_info_t *chunk_list, + size_t * num_chunks_assigned_map, + hbool_t all_ranks_involved, + const H5D_io_info_t * io_info, + const H5D_chunk_map_t *fm, int mpi_rank, int mpi_size); +static herr_t H5D__mpio_share_chunk_modification_data(H5D_filtered_collective_io_info_t *chunk_list, + size_t *chunk_list_num_entries, H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, int mpi_rank, + int mpi_size, + H5D_filtered_collective_io_info_t **chunk_hash_table, + unsigned char *** chunk_msg_bufs, + int * chunk_msg_bufs_len); +static herr_t H5D__mpio_collective_filtered_chunk_common_io(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, + const H5D_io_info_t * io_info, + const H5D_type_info_t *type_info, int mpi_size); +static herr_t H5D__mpio_collective_filtered_chunk_read(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, + const H5D_io_info_t * io_info, + const H5D_type_info_t *type_info, int mpi_rank, + int mpi_size); +static herr_t H5D__mpio_collective_filtered_chunk_update(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, + H5D_filtered_collective_io_info_t *chunk_hash_table, + unsigned char ** chunk_msg_bufs, + int chunk_msg_bufs_len, const H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, int mpi_rank, + int mpi_size); +static herr_t H5D__mpio_collective_filtered_chunk_reallocate(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, + size_t * num_chunks_assigned_map, + H5D_io_info_t * io_info, + H5D_chk_idx_info_t *idx_info, int mpi_rank, + int mpi_size); +static herr_t H5D__mpio_collective_filtered_chunk_reinsert(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, + size_t * num_chunks_assigned_map, + H5D_io_info_t * io_info, + H5D_chk_idx_info_t *idx_info, int mpi_rank, + int mpi_size); +static herr_t H5D__mpio_get_chunk_redistribute_info_types(MPI_Datatype *contig_type, + hbool_t * contig_type_derived, + MPI_Datatype *resized_type, + hbool_t * resized_type_derived); +static herr_t H5D__mpio_get_chunk_alloc_info_types(MPI_Datatype *contig_type, hbool_t *contig_type_derived, + MPI_Datatype *resized_type, hbool_t *resized_type_derived); +static herr_t H5D__mpio_get_chunk_insert_info_types(MPI_Datatype *contig_type, hbool_t *contig_type_derived, + MPI_Datatype *resized_type, + hbool_t * resized_type_derived); +static herr_t H5D__mpio_collective_filtered_io_type(H5D_filtered_collective_io_info_t *chunk_list, + size_t num_entries, H5D_io_op_type_t op_type, + MPI_Datatype *new_mem_type, hbool_t *mem_type_derived, + MPI_Datatype *new_file_type, hbool_t *file_type_derived); static int H5D__cmp_chunk_addr(const void *chunk_addr_info1, const void *chunk_addr_info2); static int H5D__cmp_filtered_collective_io_info_entry(const void *filtered_collective_io_info_entry1, const void *filtered_collective_io_info_entry2); -#if MPI_VERSION >= 3 -static int H5D__cmp_filtered_collective_io_info_entry_owner(const void *filtered_collective_io_info_entry1, - const void *filtered_collective_io_info_entry2); +static int H5D__cmp_chunk_redistribute_info(const void *entry1, const void *entry2); +static int H5D__cmp_chunk_redistribute_info_orig_owner(const void *entry1, const void *entry2); + +#ifdef H5Dmpio_DEBUG +static herr_t H5D__mpio_debug_init(void); +static herr_t H5D__mpio_dump_collective_filtered_chunk_list(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, int mpi_rank); #endif /*********************/ @@ -273,6 +395,188 @@ static int H5D__cmp_filtered_collective_io_info_entry_owner(const void *filtered /* Local Variables */ /*******************/ +/* Declare extern free list to manage the H5S_sel_iter_t struct */ +H5FL_EXTERN(H5S_sel_iter_t); + +#ifdef H5Dmpio_DEBUG + +/* Flags to control debug actions in this file. + * (Meant to be indexed by characters) + * + * These flags can be set with either (or both) the environment variable + * "H5D_mpio_Debug" set to a string containing one or more characters + * (flags) or by setting them as a string value for the + * "H5D_mpio_debug_key" MPI Info key. + * + * Supported characters in 'H5D_mpio_Debug' string: + * 't' trace function entry and exit + * 'f' log to file rather than debugging stream + * 'm' show (rough) memory usage statistics + * 'c' show critical timing information + * + * To only show output from a particular MPI rank, specify its rank + * number as a character, e.g.: + * + * '0' only show output from rank 0 + * + * To only show output from a particular range (up to 8 ranks supported + * between 0-9) of MPI ranks, specify the start and end ranks separated + * by a hyphen, e.g.: + * + * '0-7' only show output from ranks 0 through 7 + * + */ +static int H5D_mpio_debug_flags_s[256]; +static int H5D_mpio_debug_rank_s[8] = {-1, -1, -1, -1, -1, -1, -1, -1}; +static hbool_t H5D_mpio_debug_inited = FALSE; +static const char *const trace_in_pre = "-> "; +static const char *const trace_out_pre = "<- "; +static int debug_indent = 0; +static FILE * debug_stream = NULL; + +/* Determine if this rank should output debugging info */ +#define H5D_MPIO_DEBUG_THIS_RANK(rank) \ + (H5D_mpio_debug_rank_s[0] < 0 || rank == H5D_mpio_debug_rank_s[0] || rank == H5D_mpio_debug_rank_s[1] || \ + rank == H5D_mpio_debug_rank_s[2] || rank == H5D_mpio_debug_rank_s[3] || \ + rank == H5D_mpio_debug_rank_s[4] || rank == H5D_mpio_debug_rank_s[5] || \ + rank == H5D_mpio_debug_rank_s[6] || rank == H5D_mpio_debug_rank_s[7]) + +/* Print some debugging string */ +#define H5D_MPIO_DEBUG(rank, string) \ + do { \ + if (debug_stream && H5D_MPIO_DEBUG_THIS_RANK(rank)) { \ + HDfprintf(debug_stream, "%*s(Rank %d) " string "\n", debug_indent, "", rank); \ + fflush(debug_stream); \ + } \ + } while (0) + +/* Print some debugging string with printf-style arguments */ +#define H5D_MPIO_DEBUG_VA(rank, string, ...) \ + do { \ + if (debug_stream && H5D_MPIO_DEBUG_THIS_RANK(rank)) { \ + HDfprintf(debug_stream, "%*s(Rank %d) " string "\n", debug_indent, "", rank, __VA_ARGS__); \ + fflush(debug_stream); \ + } \ + } while (0) + +#define H5D_MPIO_TRACE_ENTER(rank) \ + do { \ + hbool_t trace_flag = H5D_mpio_debug_flags_s[(int)'t']; \ + \ + if (trace_flag) { \ + H5D_MPIO_DEBUG_VA(rank, "%s%s", trace_in_pre, __func__); \ + debug_indent += (int)strlen(trace_in_pre); \ + } \ + } while (0) + +#define H5D_MPIO_TRACE_EXIT(rank) \ + do { \ + hbool_t trace_flag = H5D_mpio_debug_flags_s[(int)'t']; \ + \ + if (trace_flag) { \ + debug_indent -= (int)strlen(trace_out_pre); \ + H5D_MPIO_DEBUG_VA(rank, "%s%s", trace_out_pre, __func__); \ + } \ + } while (0) + +#define H5D_MPIO_TIME_START(rank, op_name) \ + { \ + hbool_t time_flag = H5D_mpio_debug_flags_s[(int)'c']; \ + double start_time = 0.0, end_time = 0.0; \ + const char *const op = op_name; \ + \ + if (time_flag) { \ + start_time = MPI_Wtime(); \ + } + +#define H5D_MPIO_TIME_STOP(rank) \ + if (time_flag) { \ + end_time = MPI_Wtime(); \ + H5D_MPIO_DEBUG_VA(rank, "'%s' took %f seconds", op, (end_time - start_time)); \ + } \ + } + +/*--------------------------------------------------------------------------- + * Function: H5D__mpio_parse_debug_str + * + * Purpose: Parse a string for H5Dmpio-related debugging flags + * + * Returns: N/A + * + *--------------------------------------------------------------------------- + */ +static void +H5D__mpio_parse_debug_str(const char *s) +{ + FUNC_ENTER_STATIC_NOERR + + HDassert(s); + + while (*s) { + int c = (int)(*s); + + if (c >= (int)'0' && c <= (int)'9') { + hbool_t range = FALSE; + + if (*(s + 1) && *(s + 2)) + range = (int)*(s + 1) == '-' && (int)*(s + 2) >= (int)'0' && (int)*(s + 2) <= (int)'9'; + + if (range) { + int start_rank = c - (int)'0'; + int end_rank = (int)*(s + 2) - '0'; + int num_ranks = end_rank - start_rank + 1; + int i; + + if (num_ranks > 8) { + end_rank = start_rank + 7; + num_ranks = 8; + } + + for (i = 0; i < num_ranks; i++) + H5D_mpio_debug_rank_s[i] = start_rank++; + + s += 3; + } + else + H5D_mpio_debug_rank_s[0] = c - (int)'0'; + } + else + H5D_mpio_debug_flags_s[c]++; + + s++; + } + + FUNC_LEAVE_NOAPI_VOID +} + +static herr_t +H5D__mpio_debug_init(void) +{ + const char *debug_str; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC_NOERR + + HDassert(!H5D_mpio_debug_inited); + + /* Clear the debug flag buffer */ + HDmemset(H5D_mpio_debug_flags_s, 0, sizeof(H5D_mpio_debug_flags_s)); + + /* Retrieve and parse the H5Dmpio debug string */ + debug_str = HDgetenv("H5D_mpio_Debug"); + if (debug_str) + H5D__mpio_parse_debug_str(debug_str); + + if (H5DEBUG(D)) + debug_stream = H5DEBUG(D); + + H5D_mpio_debug_inited = TRUE; + + FUNC_LEAVE_NOAPI(ret_value) +} + +#endif + /*------------------------------------------------------------------------- * Function: H5D__mpio_opt_possible * @@ -347,14 +651,9 @@ H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space, co * use collective IO will defer until each chunk IO is reached. */ -#if MPI_VERSION < 3 - /* - * Don't allow parallel writes to filtered datasets if the MPI version - * is less than 3. The functions needed (MPI_Mprobe and MPI_Imrecv) will - * not be available. - */ - if (io_info->op_type == H5D_IO_OP_WRITE && io_info->dset->shared->layout.type == H5D_CHUNKED && - io_info->dset->shared->dcpl_cache.pline.nused > 0) +#ifndef H5_HAVE_PARALLEL_FILTERED_WRITES + /* Don't allow writes to filtered datasets if the functionality is disabled */ + if (io_info->op_type == H5D_IO_OP_WRITE && io_info->dset->shared->dcpl_cache.pline.nused > 0) local_cause[0] |= H5D_MPIO_PARALLEL_FILTERED_WRITES_DISABLED; #endif @@ -437,6 +736,150 @@ done: } /* H5D__mpio_opt_possible() */ /*------------------------------------------------------------------------- + * Function: H5D__mpio_get_no_coll_cause_strings + * + * Purpose: When collective I/O is broken internally, it can be useful + * for users to see a representative string for the reason(s) + * why it was broken. This routine inspects the current + * "cause" flags from the API context and prints strings into + * the caller's buffers for the local and global reasons that + * collective I/O was broken. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__mpio_get_no_coll_cause_strings(char *local_cause, size_t local_cause_len, char *global_cause, + size_t global_cause_len) +{ + uint32_t local_no_coll_cause; + uint32_t global_no_coll_cause; + size_t local_cause_bytes_written = 0; + size_t global_cause_bytes_written = 0; + int nbits; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + HDassert((local_cause && local_cause_len > 0) || (global_cause && global_cause_len > 0)); + + /* + * Use compile-time assertion so this routine is updated + * when any new "no collective cause" values are added + */ + HDcompile_assert(H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE == (H5D_mpio_no_collective_cause_t)256); + + /* Initialize output buffers */ + if (local_cause) + *local_cause = '\0'; + if (global_cause) + *global_cause = '\0'; + + /* Retrieve the local and global cause flags from the API context */ + if (H5CX_get_mpio_local_no_coll_cause(&local_no_coll_cause) < 0) + HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "unable to get local no collective cause value") + if (H5CX_get_mpio_global_no_coll_cause(&global_no_coll_cause) < 0) + HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "unable to get global no collective cause value") + + /* + * Append each of the "reason for breaking collective I/O" + * error messages to the local and global cause string buffers + */ + nbits = 8 * sizeof(local_no_coll_cause); + for (int bit_pos = 0; bit_pos < nbits; bit_pos++) { + H5D_mpio_no_collective_cause_t cur_cause; + const char * cause_str; + size_t buf_space_left; + + cur_cause = (H5D_mpio_no_collective_cause_t)(1 << bit_pos); + if (cur_cause == H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE) + break; + + switch (cur_cause) { + case H5D_MPIO_SET_INDEPENDENT: + cause_str = "independent I/O was requested"; + break; + case H5D_MPIO_DATATYPE_CONVERSION: + cause_str = "datatype conversions were required"; + break; + case H5D_MPIO_DATA_TRANSFORMS: + cause_str = "data transforms needed to be applied"; + break; + case H5D_MPIO_MPI_OPT_TYPES_ENV_VAR_DISABLED: + cause_str = "optimized MPI types flag wasn't set"; + break; + case H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES: + cause_str = "one of the dataspaces was neither simple nor scalar"; + break; + case H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET: + cause_str = "dataset was not contiguous or chunked"; + break; + case H5D_MPIO_PARALLEL_FILTERED_WRITES_DISABLED: + cause_str = "parallel writes to filtered datasets are disabled"; + break; + case H5D_MPIO_ERROR_WHILE_CHECKING_COLLECTIVE_POSSIBLE: + cause_str = "an error occurred while checking if collective I/O was possible"; + break; + case H5D_MPIO_COLLECTIVE: + case H5D_MPIO_NO_COLLECTIVE_MAX_CAUSE: + default: + HDassert(0 && "invalid no collective cause reason"); + break; + } + + /* + * Determine if the local reasons for breaking collective I/O + * included the current cause + */ + if (local_cause && (cur_cause & local_no_coll_cause)) { + buf_space_left = local_cause_len - local_cause_bytes_written; + + /* + * Check if there were any previous error messages included. If + * so, prepend a semicolon to separate the messages. + */ + if (buf_space_left && local_cause_bytes_written) { + HDstrncat(local_cause, "; ", buf_space_left); + local_cause_bytes_written += MIN(buf_space_left, 2); + buf_space_left -= MIN(buf_space_left, 2); + } + + if (buf_space_left) { + HDstrncat(local_cause, cause_str, buf_space_left); + local_cause_bytes_written += MIN(buf_space_left, HDstrlen(cause_str)); + } + } + + /* + * Determine if the global reasons for breaking collective I/O + * included the current cause + */ + if (global_cause && (cur_cause & global_no_coll_cause)) { + buf_space_left = global_cause_len - global_cause_bytes_written; + + /* + * Check if there were any previous error messages included. If + * so, prepend a semicolon to separate the messages. + */ + if (buf_space_left && global_cause_bytes_written) { + HDstrncat(global_cause, "; ", buf_space_left); + global_cause_bytes_written += MIN(buf_space_left, 2); + buf_space_left -= MIN(buf_space_left, 2); + } + + if (buf_space_left) { + HDstrncat(global_cause, cause_str, buf_space_left); + global_cause_bytes_written += MIN(buf_space_left, HDstrlen(cause_str)); + } + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_get_no_coll_cause_strings() */ + +/*------------------------------------------------------------------------- * Function: H5D__mpio_select_read * * Purpose: MPI-IO function to read directly from app buffer to file. @@ -500,145 +943,6 @@ done: } /* end H5D__mpio_select_write() */ /*------------------------------------------------------------------------- - * Function: H5D__mpio_array_gatherv - * - * Purpose: Given an array, specified in local_array, by each processor - * calling this function, collects each array into a single - * array which is then either gathered to the processor - * specified by root, when allgather is false, or is - * distributed back to all processors when allgather is true. - * - * The number of entries in the array contributed by an - * individual processor and the size of each entry should be - * specified in local_array_num_entries and array_entry_size, - * respectively. - * - * The MPI communicator to use should be specified for comm. - * - * If the sort_func argument is supplied, the array is sorted - * before the function returns. - * - * Note: if allgather is specified as true, root is ignored. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Jordan Henderson - * Sunday, April 9th, 2017 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5D__mpio_array_gatherv(void *local_array, size_t local_array_num_entries, size_t array_entry_size, - void **_gathered_array, size_t *_gathered_array_num_entries, hbool_t allgather, - int root, MPI_Comm comm, H5D_mpio_sort_func_cb_t sort_func) -{ - size_t gathered_array_num_entries = 0; /* The size of the newly-constructed array */ - void * gathered_array = NULL; /* The newly-constructed array returned to the caller */ - int *receive_counts_array = NULL; /* Array containing number of entries each processor is contributing */ - int *displacements_array = - NULL; /* Array of displacements where each processor places its data in the final array */ - int mpi_code, mpi_rank, mpi_size; - int sendcount; - herr_t ret_value = SUCCEED; - - FUNC_ENTER_STATIC - - HDassert(_gathered_array); - HDassert(_gathered_array_num_entries); - - MPI_Comm_size(comm, &mpi_size); - MPI_Comm_rank(comm, &mpi_rank); - - /* Determine the size of the end result array by collecting the number - * of entries contributed by each processor into a single total. - */ - if (MPI_SUCCESS != (mpi_code = MPI_Allreduce(&local_array_num_entries, &gathered_array_num_entries, 1, - MPI_INT, MPI_SUM, comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code) - - /* If 0 entries resulted from the collective operation, no processor is contributing anything and there is - * nothing to do */ - if (gathered_array_num_entries > 0) { - /* - * If gathering to all processors, all processors need to allocate space for the resulting array, as - * well as the receive counts and displacements arrays for the collective MPI_Allgatherv call. - * Otherwise, only the root processor needs to allocate the space for an MPI_Gatherv call. - */ - if (allgather || (mpi_rank == root)) { - if (NULL == (gathered_array = H5MM_malloc(gathered_array_num_entries * array_entry_size))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate gathered array") - - if (NULL == (receive_counts_array = (int *)H5MM_malloc((size_t)mpi_size * sizeof(int)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate receive counts array") - - if (NULL == (displacements_array = (int *)H5MM_malloc((size_t)mpi_size * sizeof(int)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate receive displacements array") - } /* end if */ - - /* - * If gathering to all processors, inform each processor of how many entries each other processor is - * contributing to the resulting array by collecting the counts into each processor's "receive counts" - * array. Otherwise, inform only the root processor of how many entries each other processor is - * contributing. - */ - if (allgather) { - if (MPI_SUCCESS != (mpi_code = MPI_Allgather(&local_array_num_entries, 1, MPI_INT, - receive_counts_array, 1, MPI_INT, comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code) - } /* end if */ - else { - if (MPI_SUCCESS != (mpi_code = MPI_Gather(&local_array_num_entries, 1, MPI_INT, - receive_counts_array, 1, MPI_INT, root, comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Gather failed", mpi_code) - } /* end else */ - - if (allgather || (mpi_rank == root)) { - size_t i; - - /* Multiply each receive count by the size of the array entry, since the data is sent as bytes. */ - for (i = 0; i < (size_t)mpi_size; i++) - H5_CHECKED_ASSIGN(receive_counts_array[i], int, - (size_t)receive_counts_array[i] * array_entry_size, size_t); - - /* Set receive buffer offsets for the collective MPI_Allgatherv/MPI_Gatherv call. */ - displacements_array[0] = 0; - for (i = 1; i < (size_t)mpi_size; i++) - displacements_array[i] = displacements_array[i - 1] + receive_counts_array[i - 1]; - } /* end if */ - - /* As the data is sent as bytes, calculate the true sendcount for the data. */ - H5_CHECKED_ASSIGN(sendcount, int, local_array_num_entries *array_entry_size, size_t); - - if (allgather) { - if (MPI_SUCCESS != - (mpi_code = MPI_Allgatherv(local_array, sendcount, MPI_BYTE, gathered_array, - receive_counts_array, displacements_array, MPI_BYTE, comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Allgatherv failed", mpi_code) - } /* end if */ - else { - if (MPI_SUCCESS != - (mpi_code = MPI_Gatherv(local_array, sendcount, MPI_BYTE, gathered_array, - receive_counts_array, displacements_array, MPI_BYTE, root, comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Gatherv failed", mpi_code) - } /* end else */ - - if (sort_func && (allgather || (mpi_rank == root))) - HDqsort(gathered_array, gathered_array_num_entries, array_entry_size, sort_func); - } /* end if */ - - *_gathered_array = gathered_array; - *_gathered_array_num_entries = gathered_array_num_entries; - -done: - if (receive_counts_array) - H5MM_free(receive_counts_array); - if (displacements_array) - H5MM_free(displacements_array); - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__mpio_array_gatherv() */ - -/*------------------------------------------------------------------------- * Function: H5D__mpio_get_sum_chunk * * Purpose: Routine for obtaining total number of chunks to cover @@ -793,11 +1097,17 @@ static herr_t H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, H5D_chunk_map_t *fm) { H5FD_mpio_chunk_opt_t chunk_opt_mode; - int io_option = H5D_MULTI_CHUNK_IO_MORE_OPT; - int sum_chunk = -1; +#ifdef H5Dmpio_DEBUG + hbool_t log_file_flag = FALSE; + FILE * debug_log_file = NULL; +#endif #ifdef H5_HAVE_INSTRUMENTED_LIBRARY htri_t temp_not_link_io = FALSE; #endif + int io_option = H5D_MULTI_CHUNK_IO_MORE_OPT; + int sum_chunk = -1; + int mpi_rank; + int mpi_size; herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC @@ -808,6 +1118,36 @@ H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf HDassert(type_info); HDassert(fm); + /* Obtain the current rank of the process and the number of ranks */ + if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0) + HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain MPI rank") + if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0) + HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain MPI size") + +#ifdef H5Dmpio_DEBUG + /* Initialize file-level debugging if not initialized */ + if (!H5D_mpio_debug_inited && H5D__mpio_debug_init() < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize H5Dmpio debugging") + + /* Open file for debugging if necessary */ + log_file_flag = H5D_mpio_debug_flags_s[(int)'f']; + if (log_file_flag) { + char debug_log_filename[1024]; + time_t time_now; + + HDsnprintf(debug_log_filename, 1024, "H5Dmpio_debug.rank%d", mpi_rank); + + if (NULL == (debug_log_file = HDfopen(debug_log_filename, "a"))) + HGOTO_ERROR(H5E_IO, H5E_OPENERROR, FAIL, "couldn't open debugging log file") + + /* Print a short header for this I/O operation */ + time_now = time(NULL); + HDfprintf(debug_log_file, "##### %s", asctime(localtime(&time_now))); + + debug_stream = debug_log_file; + } +#endif + /* Check the optional property list for the collective chunk IO optimization option */ if (H5CX_get_mpio_chunk_opt_mode(&chunk_opt_mode) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "couldn't get chunk optimization option") @@ -820,13 +1160,10 @@ H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf /* via default path. branch by num threshold */ else { unsigned one_link_chunk_io_threshold; /* Threshold to use single collective I/O for all chunks */ - int mpi_size; /* Number of processes in MPI job */ if (H5D__mpio_get_sum_chunk(io_info, fm, &sum_chunk) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSWAP, FAIL, "unable to obtain the total chunk number of all processes"); - if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size") /* Get the chunk optimization option threshold */ if (H5CX_get_mpio_chunk_opt_num(&one_link_chunk_io_threshold) < 0) @@ -872,22 +1209,12 @@ H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf case H5D_ONE_LINK_CHUNK_IO_MORE_OPT: /* Check if there are any filters in the pipeline */ if (io_info->dset->shared->dcpl_cache.pline.nused > 0) { - /* For now, Multi-chunk IO must be forced for parallel filtered read, - * so that data can be unfiltered as it is received. There is significant - * complexity in unfiltering the data when it is read all at once into a - * single buffer. - */ - if (io_info->op_type == H5D_IO_OP_READ) { - if (H5D__multi_chunk_filtered_collective_io(io_info, type_info, fm) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, - "couldn't finish optimized multiple filtered chunk MPI-IO") - } /* end if */ - else if (H5D__link_chunk_filtered_collective_io(io_info, type_info, fm) < 0) + if (H5D__link_chunk_filtered_collective_io(io_info, type_info, fm, mpi_rank, mpi_size) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish filtered linked chunk MPI-IO") } /* end if */ else /* Perform unfiltered link chunk collective IO */ - if (H5D__link_chunk_collective_io(io_info, type_info, fm, sum_chunk) < 0) + if (H5D__link_chunk_collective_io(io_info, type_info, fm, sum_chunk, mpi_rank, mpi_size) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish linked chunk MPI-IO") break; @@ -895,18 +1222,28 @@ H5D__chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf default: /* multiple chunk IO via threshold */ /* Check if there are any filters in the pipeline */ if (io_info->dset->shared->dcpl_cache.pline.nused > 0) { - if (H5D__multi_chunk_filtered_collective_io(io_info, type_info, fm) < 0) + if (H5D__multi_chunk_filtered_collective_io(io_info, type_info, fm, mpi_rank, mpi_size) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple filtered chunk MPI-IO") } /* end if */ else /* Perform unfiltered multi chunk collective IO */ - if (H5D__multi_chunk_collective_io(io_info, type_info, fm) < 0) + if (H5D__multi_chunk_collective_io(io_info, type_info, fm, mpi_rank, mpi_size) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish optimized multiple chunk MPI-IO") break; } /* end switch */ done: +#ifdef H5Dmpio_DEBUG + /* Close debugging log file */ + if (debug_log_file) { + HDfprintf(debug_log_file, "##############\n\n"); + if (EOF == HDfclose(debug_log_file)) + HDONE_ERROR(H5E_IO, H5E_CLOSEERROR, FAIL, "couldn't close debugging log file") + debug_stream = H5DEBUG(D); + } +#endif + FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__chunk_collective_io */ @@ -989,7 +1326,7 @@ done: */ static herr_t H5D__link_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, H5D_chunk_map_t *fm, - int sum_chunk) + int sum_chunk, int mpi_rank, int mpi_size) { H5D_chunk_addr_info_t *chunk_addr_info_array = NULL; MPI_Datatype chunk_final_mtype; /* Final memory MPI datatype for all chunks with selection */ @@ -1070,9 +1407,8 @@ H5D__link_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *typ /* Set up the base storage address for this chunk */ io_info->store = &ctg_store; -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "before inter_collective_io for total chunk = 1 \n"); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG(mpi_rank, "before inter_collective_io for total chunk = 1"); #endif /* Perform I/O */ @@ -1088,9 +1424,8 @@ H5D__link_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *typ num_chunk = H5SL_count(fm->sel_chunks); H5_CHECK_OVERFLOW(num_chunk, size_t, int); -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "total_chunks = %zu, num_chunk = %zu\n", total_chunks, num_chunk); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG_VA(mpi_rank, "total_chunks = %zu, num_chunk = %zu", total_chunks, num_chunk); #endif /* Set up MPI datatype for chunks selected */ @@ -1121,18 +1456,17 @@ H5D__link_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *typ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk file is derived datatype flags buffer") -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "before sorting the chunk address \n"); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG(mpi_rank, "before sorting chunk addresses"); #endif + /* Sort the chunk address */ - if (H5D__sort_chunk(io_info, fm, chunk_addr_info_array, sum_chunk) < 0) + if (H5D__sort_chunk(io_info, fm, chunk_addr_info_array, sum_chunk, mpi_rank, mpi_size) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSWAP, FAIL, "unable to sort chunk address") ctg_store.contig.dset_addr = chunk_addr_info_array[0].chunk_addr; -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "after sorting the chunk address \n"); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG(mpi_rank, "after sorting chunk addresses"); #endif /* Obtain MPI derived datatype from all individual chunks */ @@ -1237,9 +1571,9 @@ H5D__link_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *typ /* No chunks selected for this process */ mpi_buf_count = (hsize_t)0; } /* end else */ -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "before coming to final collective IO\n"); + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG(mpi_rank, "before coming to final collective I/O"); #endif /* Set up the base storage address for this chunk */ @@ -1252,11 +1586,11 @@ H5D__link_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *typ } /* end else */ done: -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "before freeing memory inside H5D_link_collective_io ret_value = %d\n", - ret_value); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG_VA(mpi_rank, "before freeing memory inside H5D_link_collective_io ret_value = %d", + ret_value); #endif + /* Release resources */ if (chunk_addr_info_array) H5MM_xfree(chunk_addr_info_array); @@ -1289,68 +1623,89 @@ done: /*------------------------------------------------------------------------- * Function: H5D__link_chunk_filtered_collective_io * - * Purpose: Routine for one collective IO with one MPI derived datatype - * to link with all filtered chunks - * - * 1. Construct a list of selected chunks in the collective IO - * operation - * A. If any chunk is being written to by more than 1 - * process, the process writing to the chunk which - * currently has the least amount of chunks assigned - * to it becomes the new owner (in the case of ties, - * the lowest MPI rank becomes the new owner) - * 2. If the operation is a write operation - * A. Loop through each chunk in the operation - * I. If this is not a full overwrite of the chunk - * a) Read the chunk from file and pass the chunk - * through the filter pipeline in reverse order - * (Unfilter the chunk) + * Purpose: Performs collective I/O on filtered chunks by creating a + * single MPI derived datatype to link with all filtered + * chunks. The general algorithm is as follows: + * + * 1. Construct a list of selected chunks in the collective + * I/O operation + * 2. If the operation is a read operation + * A. Ensure that the list of chunks is sorted in + * monotonically non-decreasing order of chunk offset + * in the file + * B. Participate in a collective read of chunks from + * the file + * C. Loop through each selected chunk, unfiltering it and + * scattering the data to the application's read buffer + * 3. If the operation is a write operation + * A. Redistribute any chunks being written by more than 1 + * MPI rank, such that the chunk is only owned by 1 MPI + * rank. The rank writing to the chunk which currently + * has the least amount of chunks assigned to it becomes + * the new owner (in the case of ties, the lowest MPI + * rank becomes the new owner) + * B. Participate in a collective read of chunks from the + * file + * C. Loop through each chunk selected in the operation + * and for each chunk: + * I. If we actually read the chunk from the file (if + * a chunk is being fully overwritten, we skip + * reading it), pass the chunk through the filter + * pipeline in reverse order (unfilter the chunk) * II. Update the chunk data with the modifications from - * the owning process + * the owning MPI rank * III. Receive any modification data from other - * processes and update the chunk data with these + * ranks and update the chunk data with those * modifications * IV. Filter the chunk - * B. Contribute the modified chunks to an array gathered - * by all processes which contains the new sizes of - * every chunk modified in the collective IO operation - * C. All processes collectively re-allocate each chunk - * from the gathered array with their new sizes after - * the filter operation - * D. If this process has any chunks selected in the IO - * operation, create an MPI derived type for memory and - * file to write out the process' selected chunks to the - * file - * E. Perform the collective write - * F. All processes collectively re-insert each modified + * D. Contribute the modified chunks to an array gathered + * by all ranks which contains information for + * re-allocating space in the file for every chunk + * modified. Then, each rank collectively re-allocates + * each chunk from the gathered array with their new + * sizes after the filter operation + * E. Proceed with the collective write operation for all + * the modified chunks + * F. Contribute the modified chunks to an array gathered + * by all ranks which contains information for + * re-inserting every chunk modified into the chunk + * index. Then, each rank collectively re-inserts each * chunk from the gathered array into the chunk index * + * TODO: Note that steps D. and F. here are both collective + * operations that partially share data from the + * H5D_filtered_collective_io_info_t structure. To + * try to conserve on memory a bit, the distributed + * arrays these operations create are discarded after + * each operation is performed. If memory consumption + * here proves to not be an issue, the necessary data + * for both operations could be combined into a single + * structure so that only one collective MPI operation + * is needed to carry out both operations, rather than + * two. * * Return: Non-negative on success/Negative on failure * - * Programmer: Jordan Henderson - * Friday, Nov. 4th, 2016 - * *------------------------------------------------------------------------- */ static herr_t H5D__link_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - H5D_chunk_map_t *fm) + H5D_chunk_map_t *fm, int mpi_rank, int mpi_size) { - H5D_filtered_collective_io_info_t *chunk_list = NULL; /* The list of chunks being read/written */ - H5D_filtered_collective_io_info_t *collective_chunk_list = - NULL; /* The list of chunks used during collective operations */ - H5D_storage_t ctg_store; /* Chunk storage information as contiguous dataset */ - MPI_Datatype mem_type = MPI_BYTE; - MPI_Datatype file_type = MPI_BYTE; - hbool_t mem_type_is_derived = FALSE; - hbool_t file_type_is_derived = FALSE; - size_t chunk_list_num_entries; - size_t collective_chunk_list_num_entries; - size_t * num_chunks_selected_array = NULL; /* Array of number of chunks selected on each process */ - size_t i; /* Local index variable */ - int mpi_rank, mpi_size, mpi_code; - herr_t ret_value = SUCCEED; + H5D_filtered_collective_io_info_t *chunk_list = NULL; /* The list of chunks being read/written */ + H5D_filtered_collective_io_info_t *chunk_hash_table = NULL; + unsigned char ** chunk_msg_bufs = NULL; + H5D_storage_t ctg_store; /* Chunk storage information as contiguous dataset */ + MPI_Datatype mem_type = MPI_BYTE; + MPI_Datatype file_type = MPI_BYTE; + hbool_t mem_type_is_derived = FALSE; + hbool_t file_type_is_derived = FALSE; + size_t * rank_chunks_assigned_map = NULL; + size_t chunk_list_num_entries; + size_t i; + int chunk_msg_bufs_len = 0; + int mpi_code; + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC @@ -1358,11 +1713,12 @@ H5D__link_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_in HDassert(type_info); HDassert(fm); - /* Obtain the current rank of the process and the number of processes */ - if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank") - if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size") +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_DEBUG_VA(mpi_rank, "Performing Linked-chunk I/O (%s) with MPI Comm size of %d", + io_info->op_type == H5D_IO_OP_WRITE ? "write" : "read", mpi_size); + H5D_MPIO_TIME_START(mpi_rank, "Linked-chunk I/O"); +#endif /* Set the actual-chunk-opt-mode property. */ H5CX_set_mpio_actual_chunk_opt(H5D_MPIO_LINK_CHUNK); @@ -1373,123 +1729,127 @@ H5D__link_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_in H5CX_set_mpio_actual_io_mode(H5D_MPIO_CHUNK_COLLECTIVE); /* Build a list of selected chunks in the collective io operation */ - if (H5D__construct_filtered_io_info_list(io_info, type_info, fm, &chunk_list, &chunk_list_num_entries) < - 0) + if (H5D__mpio_collective_filtered_chunk_io_setup(io_info, type_info, fm, &chunk_list, + &chunk_list_num_entries, mpi_rank) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "couldn't construct filtered I/O info list") - if (io_info->op_type == H5D_IO_OP_WRITE) { /* Filtered collective write */ + if (io_info->op_type == H5D_IO_OP_READ) { /* Filtered collective read */ + if (H5D__mpio_collective_filtered_chunk_read(chunk_list, chunk_list_num_entries, io_info, type_info, + mpi_rank, mpi_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't read filtered chunks") + } + else { /* Filtered collective write */ H5D_chk_idx_info_t index_info; - H5D_chunk_ud_t udata; hsize_t mpi_buf_count; - /* Construct chunked index info */ - index_info.f = io_info->dset->oloc.file; - index_info.pline = &(io_info->dset->shared->dcpl_cache.pline); - index_info.layout = &(io_info->dset->shared->layout.u.chunk); - index_info.storage = &(io_info->dset->shared->layout.storage.u.chunk); - - /* Set up chunk information for insertion to chunk index */ - udata.common.layout = index_info.layout; - udata.common.storage = index_info.storage; - udata.filter_mask = 0; - - /* Iterate through all the chunks in the collective write operation, - * updating each chunk with the data modifications from other processes, - * then re-filtering the chunk. - */ - for (i = 0; i < chunk_list_num_entries; i++) - if (mpi_rank == chunk_list[i].owners.new_owner) - if (H5D__filtered_collective_chunk_entry_io(&chunk_list[i], io_info, type_info, fm) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't process chunk entry") + H5D_MPIO_INIT_CHUNK_IDX_INFO(index_info, io_info); - /* Gather the new chunk sizes to all processes for a collective reallocation - * of the chunks in the file. - */ - if (H5D__mpio_array_gatherv(chunk_list, chunk_list_num_entries, - sizeof(H5D_filtered_collective_io_info_t), - (void **)&collective_chunk_list, &collective_chunk_list_num_entries, true, - 0, io_info->comm, NULL) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "couldn't gather new chunk sizes") - - /* Collectively re-allocate the modified chunks (from each process) in the file */ - for (i = 0; i < collective_chunk_list_num_entries; i++) { - hbool_t insert; - - if (H5D__chunk_file_alloc(&index_info, &collective_chunk_list[i].chunk_states.chunk_current, - &collective_chunk_list[i].chunk_states.new_chunk, &insert, - collective_chunk_list[i].scaled) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate chunk") - } /* end for */ + if (mpi_size > 1) { + /* Redistribute shared chunks being written to */ + if (H5D__mpio_redistribute_shared_chunks(chunk_list, chunk_list_num_entries, io_info, fm, + mpi_rank, mpi_size, &rank_chunks_assigned_map) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to redistribute shared chunks") - if (NULL == (num_chunks_selected_array = (size_t *)H5MM_malloc((size_t)mpi_size * sizeof(size_t)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate num chunks selected array") + /* Send any chunk modification messages for chunks this rank no longer owns */ + if (H5D__mpio_share_chunk_modification_data(chunk_list, &chunk_list_num_entries, io_info, + type_info, mpi_rank, mpi_size, &chunk_hash_table, + &chunk_msg_bufs, &chunk_msg_bufs_len) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, + "unable to send chunk modification data between MPI ranks") - if (MPI_SUCCESS != - (mpi_code = MPI_Allgather(&chunk_list_num_entries, 1, MPI_UNSIGNED_LONG_LONG, - num_chunks_selected_array, 1, MPI_UNSIGNED_LONG_LONG, io_info->comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code) + /* Make sure the local chunk list was updated correctly */ + HDassert(chunk_list_num_entries == rank_chunks_assigned_map[mpi_rank]); + } - /* If this process has any chunks selected, create a MPI type for collectively - * writing out the chunks to file. Otherwise, the process contributes to the + /* Proceed to update all the chunks this rank owns with its own + * modification data and data from other ranks, before re-filtering + * the chunks. As chunk reads are done collectively here, all ranks + * must participate. + */ + if (H5D__mpio_collective_filtered_chunk_update(chunk_list, chunk_list_num_entries, chunk_hash_table, + chunk_msg_bufs, chunk_msg_bufs_len, io_info, type_info, + mpi_rank, mpi_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't update modified chunks") + + /* Free up resources used by chunk hash table now that we're done updating chunks */ + HASH_CLEAR(hh, chunk_hash_table); + + /* All ranks now collectively re-allocate file space for all chunks */ + if (H5D__mpio_collective_filtered_chunk_reallocate(chunk_list, chunk_list_num_entries, + rank_chunks_assigned_map, io_info, &index_info, + mpi_rank, mpi_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, + "couldn't collectively re-allocate file space for chunks") + + /* If this rank has any chunks selected, create a MPI type for collectively + * writing out the chunks to file. Otherwise, the rank contributes to the * collective write with a none type. */ - if (chunk_list_num_entries) { - size_t offset; - - /* During the collective re-allocation of chunks in the file, the record for each - * chunk is only updated in the collective array, not in the local copy of chunks on each - * process. However, each process needs the updated chunk records so that they can create - * a MPI type for the collective write that will write to the chunk's possible new locations - * in the file instead of the old ones. This ugly hack seems to be the best solution to - * copy the information back to the local array and avoid having to modify the collective - * write type function in an ugly way so that it will accept the collective array instead - * of the local array. This works correctly because the array gather function guarantees - * that the chunk data in the collective array is ordered in blocks by rank. - */ - for (i = 0, offset = 0; i < (size_t)mpi_rank; i++) - offset += num_chunks_selected_array[i]; - - H5MM_memcpy(chunk_list, &collective_chunk_list[offset], - num_chunks_selected_array[mpi_rank] * sizeof(H5D_filtered_collective_io_info_t)); + if (H5D__mpio_collective_filtered_io_type(chunk_list, chunk_list_num_entries, io_info->op_type, + &mem_type, &mem_type_is_derived, &file_type, + &file_type_is_derived) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "couldn't create MPI type for writing filtered chunks") - /* Create single MPI type encompassing each selection in the dataspace */ - if (H5D__mpio_filtered_collective_write_type(chunk_list, chunk_list_num_entries, &mem_type, - &mem_type_is_derived, &file_type, - &file_type_is_derived) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_BADTYPE, FAIL, "couldn't create MPI link chunk I/O type") + mpi_buf_count = (file_type_is_derived || mem_type_is_derived) ? 1 : 0; - /* Override the write buffer to point to the address of the first - * chunk data buffer + /* Setup contig storage info for I/O operation */ + if (chunk_list_num_entries) { + /* + * Override the write buffer to point to the first + * chunk's data buffer */ io_info->u.wbuf = chunk_list[0].buf; - } /* end if */ - /* We have a single, complicated MPI datatype for both memory & file */ - mpi_buf_count = (mem_type_is_derived && file_type_is_derived) ? (hsize_t)1 : (hsize_t)0; - - /* Set up the base storage address for this operation */ - ctg_store.contig.dset_addr = 0; /* Write address must be set to address 0 */ - io_info->store = &ctg_store; + /* + * Setup the base storage address for this operation + * to be the first chunk's file address + */ + ctg_store.contig.dset_addr = chunk_list[0].chunk_new.offset; + } + else + ctg_store.contig.dset_addr = 0; /* Perform I/O */ + io_info->store = &ctg_store; if (H5D__final_collective_io(io_info, type_info, mpi_buf_count, file_type, mem_type) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish MPI-IO") + /* Free up resources in anticipation of following collective operation */ + for (i = 0; i < chunk_list_num_entries; i++) { + if (chunk_list[i].buf) { + H5MM_free(chunk_list[i].buf); + chunk_list[i].buf = NULL; + } + } + /* Participate in the collective re-insertion of all chunks modified - * in this iteration into the chunk index + * into the chunk index */ - for (i = 0; i < collective_chunk_list_num_entries; i++) { - udata.chunk_block = collective_chunk_list[i].chunk_states.new_chunk; - udata.common.scaled = collective_chunk_list[i].scaled; - udata.chunk_idx = collective_chunk_list[i].index; - - if ((index_info.storage->ops->insert)(&index_info, &udata, io_info->dset) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk address into index") - } /* end for */ - } /* end if */ + if (H5D__mpio_collective_filtered_chunk_reinsert(chunk_list, chunk_list_num_entries, + rank_chunks_assigned_map, io_info, &index_info, + mpi_rank, mpi_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, + "couldn't collectively re-insert modified chunks into chunk index") + } done: - /* Free resources used by a process which had some selection */ + /* Free the MPI buf and file types, if they were derived */ + if (mem_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + if (file_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + + if (chunk_msg_bufs) { + for (i = 0; i < (size_t)chunk_msg_bufs_len; i++) + H5MM_free(chunk_msg_bufs[i]); + + H5MM_free(chunk_msg_bufs); + } + + HASH_CLEAR(hh, chunk_hash_table); + + /* Free resources used by a rank which had some selection */ if (chunk_list) { for (i = 0; i < chunk_list_num_entries; i++) if (chunk_list[i].buf) @@ -1498,16 +1858,13 @@ done: H5MM_free(chunk_list); } /* end if */ - if (num_chunks_selected_array) - H5MM_free(num_chunks_selected_array); - if (collective_chunk_list) - H5MM_free(collective_chunk_list); + if (rank_chunks_assigned_map) + H5MM_free(rank_chunks_assigned_map); - /* Free the MPI buf and file types, if they were derived */ - if (mem_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type))) - HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) - if (file_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type))) - HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__link_chunk_filtered_collective_io() */ @@ -1530,7 +1887,8 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, H5D_chunk_map_t *fm) +H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, H5D_chunk_map_t *fm, + int mpi_rank, int mpi_size) { H5D_io_info_t ctg_io_info; /* Contiguous I/O info object */ H5D_storage_t ctg_store; /* Chunk storage information as contiguous dataset */ @@ -1543,11 +1901,8 @@ H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *ty H5FD_mpio_collective_opt_t last_coll_opt_mode = H5FD_MPIO_COLLECTIVE_IO; /* Last parallel transfer with independent IO or collective IO with this mode */ - size_t total_chunk; /* Total # of chunks in dataset */ -#ifdef H5Dmpio_DEBUG - int mpi_rank; -#endif - size_t u; /* Local index variable */ + size_t total_chunk; /* Total # of chunks in dataset */ + size_t u; /* Local index variable */ H5D_mpio_actual_io_mode_t actual_io_mode = H5D_MPIO_NO_COLLECTIVE; /* Local variable for tracking the I/O mode used. */ herr_t ret_value = SUCCEED; @@ -1557,10 +1912,6 @@ H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *ty /* Set the actual chunk opt mode property */ H5CX_set_mpio_actual_chunk_opt(H5D_MPIO_MULTI_CHUNK); -#ifdef H5Dmpio_DEBUG - mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file); -#endif - /* Retrieve total # of chunks in dataset */ H5_CHECKED_ASSIGN(total_chunk, size_t, fm->layout->u.chunk.nchunks, hsize_t); HDassert(total_chunk != 0); @@ -1568,13 +1919,13 @@ H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *ty /* Allocate memories */ chunk_io_option = (uint8_t *)H5MM_calloc(total_chunk); chunk_addr = (haddr_t *)H5MM_calloc(total_chunk * sizeof(haddr_t)); -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "total_chunk %zu\n", total_chunk); + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG_VA(mpi_rank, "total_chunk %zu", total_chunk); #endif /* Obtain IO option for each chunk */ - if (H5D__obtain_mpio_mode(io_info, fm, chunk_io_option, chunk_addr) < 0) + if (H5D__obtain_mpio_mode(io_info, fm, chunk_io_option, chunk_addr, mpi_rank, mpi_size) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTRECV, FAIL, "unable to obtain MPIO mode") /* Set up contiguous I/O info object */ @@ -1602,9 +1953,8 @@ H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *ty H5S_t * fspace; /* Dataspace describing chunk & selection in it */ H5S_t * mspace; /* Dataspace describing selection in memory corresponding to this chunk */ -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "mpi_rank = %d, chunk index = %zu\n", mpi_rank, u); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG_VA(mpi_rank, "mpi_rank = %d, chunk index = %zu", mpi_rank, u); #endif /* Get the chunk info for this chunk, if there are elements selected */ chunk_info = fm->select_chunk[u]; @@ -1622,10 +1972,9 @@ H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *ty * needs to contribute MPI NONE TYPE. */ if (chunk_io_option[u] == H5D_CHUNK_IO_MODE_COL) { -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "inside collective chunk IO mpi_rank = %d, chunk index = %zu\n", - mpi_rank, u); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG_VA(mpi_rank, "inside collective chunk IO mpi_rank = %d, chunk index = %zu", + mpi_rank, u); #endif /* Set the file & memory dataspaces */ @@ -1661,10 +2010,9 @@ H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *ty HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish shared collective MPI-IO") } /* end if */ else { /* possible independent IO for this chunk */ -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "inside independent IO mpi_rank = %d, chunk index = %zu\n", mpi_rank, - u); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG_VA(mpi_rank, "inside independent IO mpi_rank = %d, chunk index = %zu", mpi_rank, + u); #endif HDassert(chunk_io_option[u] == 0); @@ -1694,9 +2042,8 @@ H5D__multi_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *ty /* Perform the I/O */ if (H5D__inter_collective_io(&ctg_io_info, type_info, fspace, mspace) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish shared collective MPI-IO") -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "after inter collective IO\n"); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG(mpi_rank, "after inter collective IO"); #endif } /* end else */ } /* end for */ @@ -1716,80 +2063,101 @@ done: /*------------------------------------------------------------------------- * Function: H5D__multi_chunk_filtered_collective_io * - * Purpose: To do filtered collective IO iteratively to save on memory. - * While link_chunk_filtered_collective_io will construct and - * work on a list of all of the chunks selected in the IO - * operation at once, this function works iteratively on a set - * of chunks at a time; at most one chunk per rank per - * iteration. - * - * 1. Construct a list of selected chunks in the collective IO - * operation - * A. If any chunk is being written to by more than 1 - * process, the process writing to the chunk which - * currently has the least amount of chunks assigned - * to it becomes the new owner (in the case of ties, - * the lowest MPI rank becomes the new owner) - * 2. If the operation is a read operation - * A. Loop through each chunk in the operation - * I. Read the chunk from the file - * II. Unfilter the chunk - * III. Scatter the read chunk data to the user's buffer - * 3. If the operation is a write operation - * A. Loop through each chunk in the operation - * I. If this is not a full overwrite of the chunk - * a) Read the chunk from file and pass the chunk - * through the filter pipeline in reverse order - * (Unfilter the chunk) - * II. Update the chunk data with the modifications from - * the owning process - * III. Receive any modification data from other - * processes and update the chunk data with these - * modifications - * IV. Filter the chunk - * V. Contribute the chunk to an array gathered by - * all processes which contains every chunk - * modified in this iteration (up to one chunk - * per process, some processes may not have a - * selection/may have less chunks to work on than - * other processes) - * VI. All processes collectively re-allocate each - * chunk from the gathered array with their new - * sizes after the filter operation - * VII. Proceed with the collective write operation - * for the chunks modified on this iteration - * VIII. All processes collectively re-insert each - * chunk from the gathered array into the chunk - * index + * Purpose: Performs collective I/O on filtered chunks iteratively to + * save on memory and potentially get better performance + * depending on the average number of chunks per rank. While + * linked-chunk I/O will construct and work on a list of all + * of the chunks selected in the I/O operation at once, this + * function works iteratively on a set of chunks at a time; at + * most one chunk per rank per iteration. The general + * algorithm is as follows: + * + * 1. Construct a list of selected chunks in the collective + * I/O operation + * 2. If the operation is a read operation, loop an amount of + * times equal to the maximum number of chunks selected on + * any particular rank and on each iteration: + * A. Participate in a collective read of chunks from + * the file (ranks that run out of chunks still need + * to participate) + * B. Unfilter the chunk that was read (if any) + * C. Scatter the read chunk's data to the application's + * read buffer + * 3. If the operation is a write operation, redistribute any + * chunks being written to by more than 1 MPI rank, such + * that the chunk is only owned by 1 MPI rank. The rank + * writing to the chunk which currently has the least + * amount of chunks assigned to it becomes the new owner + * (in the case of ties, the lowest MPI rank becomes the + * new owner). Then, loop an amount of times equal to the + * maximum number of chunks selected on any particular + * rank and on each iteration: + * A. Participate in a collective read of chunks from + * the file (ranks that run out of chunks still need + * to participate) + * I. If we actually read a chunk from the file (if + * a chunk is being fully overwritten, we skip + * reading it), pass the chunk through the filter + * pipeline in reverse order (unfilter the chunk) + * B. Update the chunk data with the modifications from + * the owning rank + * C. Receive any modification data from other ranks and + * update the chunk data with those modifications + * D. Filter the chunk + * E. Contribute the chunk to an array gathered by + * all ranks which contains information for + * re-allocating space in the file for every chunk + * modified in this iteration (up to one chunk per + * rank; some ranks may not have a selection/may have + * less chunks to work on than other ranks). Then, + * each rank collectively re-allocates each chunk + * from the gathered array with their new sizes + * after the filter operation + * F. Proceed with the collective write operation + * for the chunks modified on this iteration + * G. Contribute the chunk to an array gathered by + * all ranks which contains information for + * re-inserting every chunk modified on this + * iteration into the chunk index. Then, each rank + * collectively re-inserts each chunk from the + * gathered array into the chunk index + * + * TODO: Note that steps E. and G. here are both collective + * operations that partially share data from the + * H5D_filtered_collective_io_info_t structure. To + * try to conserve on memory a bit, the distributed + * arrays these operations create are discarded after + * each operation is performed. If memory consumption + * here proves to not be an issue, the necessary data + * for both operations could be combined into a single + * structure so that only one collective MPI operation + * is needed to carry out both operations, rather than + * two. * * Return: Non-negative on success/Negative on failure * - * Programmer: Jordan Henderson - * Friday, Dec. 2nd, 2016 - * *------------------------------------------------------------------------- */ static herr_t H5D__multi_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - H5D_chunk_map_t *fm) + H5D_chunk_map_t *fm, int mpi_rank, int mpi_size) { - H5D_filtered_collective_io_info_t *chunk_list = NULL; /* The list of chunks being read/written */ - H5D_filtered_collective_io_info_t *collective_chunk_list = - NULL; /* The list of chunks used during collective operations */ - H5D_storage_t store; /* union of EFL and chunk pointer in file space */ - H5D_io_info_t ctg_io_info; /* Contiguous I/O info object */ - H5D_storage_t ctg_store; /* Chunk storage information as contiguous dataset */ - MPI_Datatype *file_type_array = NULL; - MPI_Datatype *mem_type_array = NULL; - hbool_t * file_type_is_derived_array = NULL; - hbool_t * mem_type_is_derived_array = NULL; - hbool_t * has_chunk_selected_array = - NULL; /* Array of whether or not each process is contributing a chunk to each iteration */ - size_t chunk_list_num_entries; - size_t collective_chunk_list_num_entries; - size_t i, j; /* Local index variable */ - int mpi_rank, mpi_size, mpi_code; - herr_t ret_value = SUCCEED; + H5D_filtered_collective_io_info_t *chunk_list = NULL; /* The list of chunks being read/written */ + H5D_filtered_collective_io_info_t *chunk_hash_table = NULL; + unsigned char ** chunk_msg_bufs = NULL; + H5D_io_info_t ctg_io_info; /* Contiguous I/O info object */ + H5D_storage_t ctg_store; /* Chunk storage information as contiguous dataset */ + MPI_Datatype mem_type = MPI_BYTE; + MPI_Datatype file_type = MPI_BYTE; + hbool_t mem_type_is_derived = FALSE; + hbool_t file_type_is_derived = FALSE; + hbool_t have_chunk_to_process; + size_t chunk_list_num_entries; + size_t i; + size_t max_num_chunks; + int chunk_msg_bufs_len = 0; + int mpi_code; + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC @@ -1797,11 +2165,12 @@ H5D__multi_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_i HDassert(type_info); HDassert(fm); - /* Obtain the current rank of the process and the number of processes */ - if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank") - if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size") +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_DEBUG_VA(mpi_rank, "Performing Multi-chunk I/O (%s) with MPI Comm size of %d", + io_info->op_type == H5D_IO_OP_WRITE ? "write" : "read", mpi_size); + H5D_MPIO_TIME_START(mpi_rank, "Multi-chunk I/O"); +#endif /* Set the actual chunk opt mode property */ H5CX_set_mpio_actual_chunk_opt(H5D_MPIO_MULTI_CHUNK); @@ -1812,10 +2181,19 @@ H5D__multi_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_i H5CX_set_mpio_actual_io_mode(H5D_MPIO_CHUNK_COLLECTIVE); /* Build a list of selected chunks in the collective IO operation */ - if (H5D__construct_filtered_io_info_list(io_info, type_info, fm, &chunk_list, &chunk_list_num_entries) < - 0) + if (H5D__mpio_collective_filtered_chunk_io_setup(io_info, type_info, fm, &chunk_list, + &chunk_list_num_entries, mpi_rank) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "couldn't construct filtered I/O info list") + /* Retrieve the maximum number of chunks selected for any rank */ + if (MPI_SUCCESS != (mpi_code = MPI_Allreduce(&chunk_list_num_entries, &max_num_chunks, 1, + MPI_UNSIGNED_LONG_LONG, MPI_MAX, io_info->comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code) + + /* If no one has anything selected at all, end the operation */ + if (0 == max_num_chunks) + HGOTO_DONE(SUCCEED); + /* Set up contiguous I/O info object */ H5MM_memcpy(&ctg_io_info, io_info, sizeof(ctg_io_info)); ctg_io_info.store = &ctg_store; @@ -1823,190 +2201,147 @@ H5D__multi_chunk_filtered_collective_io(H5D_io_info_t *io_info, const H5D_type_i /* Initialize temporary contiguous storage info */ ctg_store.contig.dset_size = (hsize_t)io_info->dset->shared->layout.u.chunk.size; - ctg_store.contig.dset_addr = 0; - - /* Set dataset storage for I/O info */ - io_info->store = &store; if (io_info->op_type == H5D_IO_OP_READ) { /* Filtered collective read */ - for (i = 0; i < chunk_list_num_entries; i++) - if (H5D__filtered_collective_chunk_entry_io(&chunk_list[i], io_info, type_info, fm) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't process chunk entry") - } /* end if */ + for (i = 0; i < max_num_chunks; i++) { + /* Check if this rank has a chunk to work on for this iteration */ + have_chunk_to_process = (i < chunk_list_num_entries); + + if (H5D__mpio_collective_filtered_chunk_read(have_chunk_to_process ? &chunk_list[i] : NULL, + have_chunk_to_process ? 1 : 0, io_info, type_info, + mpi_rank, mpi_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't read filtered chunks") + + if (have_chunk_to_process && chunk_list[i].buf) { + H5MM_free(chunk_list[i].buf); + chunk_list[i].buf = NULL; + } + } + } else { /* Filtered collective write */ H5D_chk_idx_info_t index_info; - H5D_chunk_ud_t udata; - size_t max_num_chunks; hsize_t mpi_buf_count; /* Construct chunked index info */ - index_info.f = io_info->dset->oloc.file; - index_info.pline = &(io_info->dset->shared->dcpl_cache.pline); - index_info.layout = &(io_info->dset->shared->layout.u.chunk); - index_info.storage = &(io_info->dset->shared->layout.storage.u.chunk); - - /* Set up chunk information for insertion to chunk index */ - udata.common.layout = index_info.layout; - udata.common.storage = index_info.storage; - udata.filter_mask = 0; - - /* Retrieve the maximum number of chunks being written among all processes */ - if (MPI_SUCCESS != (mpi_code = MPI_Allreduce(&chunk_list_num_entries, &max_num_chunks, 1, - MPI_UNSIGNED_LONG_LONG, MPI_MAX, io_info->comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Allreduce failed", mpi_code) - - /* If no one is writing anything at all, end the operation */ - if (!(max_num_chunks > 0)) - HGOTO_DONE(SUCCEED); - - /* Allocate arrays for storing MPI file and mem types and whether or not the - * types were derived. - */ - if (NULL == (file_type_array = (MPI_Datatype *)H5MM_malloc(max_num_chunks * sizeof(MPI_Datatype)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate file type array") - - if (NULL == (file_type_is_derived_array = (hbool_t *)H5MM_calloc(max_num_chunks * sizeof(hbool_t)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate file type is derived array") - - if (NULL == (mem_type_array = (MPI_Datatype *)H5MM_malloc(max_num_chunks * sizeof(MPI_Datatype)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate mem type array") - - if (NULL == (mem_type_is_derived_array = (hbool_t *)H5MM_calloc(max_num_chunks * sizeof(hbool_t)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate mem type is derived array") + H5D_MPIO_INIT_CHUNK_IDX_INFO(index_info, io_info); + + if (mpi_size > 1) { + /* Redistribute shared chunks being written to */ + if (H5D__mpio_redistribute_shared_chunks(chunk_list, chunk_list_num_entries, io_info, fm, + mpi_rank, mpi_size, NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to redistribute shared chunks") + + /* Send any chunk modification messages for chunks this rank no longer owns */ + if (H5D__mpio_share_chunk_modification_data(chunk_list, &chunk_list_num_entries, io_info, + type_info, mpi_rank, mpi_size, &chunk_hash_table, + &chunk_msg_bufs, &chunk_msg_bufs_len) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, + "unable to send chunk modification data between MPI ranks") + } - /* Iterate over the max number of chunks among all processes, as this process could - * have no chunks left to work on, but it still needs to participate in the collective - * re-allocation and re-insertion of chunks modified by other processes. + /* Iterate over the max number of chunks among all ranks, as this rank could + * have no chunks left to work on, but it still needs to participate in the + * collective re-allocation and re-insertion of chunks modified by other ranks. */ for (i = 0; i < max_num_chunks; i++) { - /* Check if this process has a chunk to work on for this iteration */ - hbool_t have_chunk_to_process = - (i < chunk_list_num_entries) && (mpi_rank == chunk_list[i].owners.new_owner); + /* Check if this rank has a chunk to work on for this iteration */ + have_chunk_to_process = (i < chunk_list_num_entries) && (mpi_rank == chunk_list[i].new_owner); - if (have_chunk_to_process) - if (H5D__filtered_collective_chunk_entry_io(&chunk_list[i], io_info, type_info, fm) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't process chunk entry") - - /* Gather the new chunk sizes to all processes for a collective re-allocation - * of the chunks in the file - */ - if (H5D__mpio_array_gatherv(&chunk_list[i], have_chunk_to_process ? 1 : 0, - sizeof(H5D_filtered_collective_io_info_t), - (void **)&collective_chunk_list, &collective_chunk_list_num_entries, - true, 0, io_info->comm, NULL) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "couldn't gather new chunk sizes") - - /* Participate in the collective re-allocation of all chunks modified - * in this iteration. + /* Proceed to update the chunk this rank owns (if any left) with its + * own modification data and data from other ranks, before re-filtering + * the chunks. As chunk reads are done collectively here, all ranks + * must participate. */ - for (j = 0; j < collective_chunk_list_num_entries; j++) { - hbool_t insert = FALSE; - - if (H5D__chunk_file_alloc(&index_info, &collective_chunk_list[j].chunk_states.chunk_current, - &collective_chunk_list[j].chunk_states.new_chunk, &insert, - chunk_list[j].scaled) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate chunk") - } /* end for */ + if (H5D__mpio_collective_filtered_chunk_update(have_chunk_to_process ? &chunk_list[i] : NULL, + have_chunk_to_process ? 1 : 0, chunk_hash_table, + chunk_msg_bufs, chunk_msg_bufs_len, io_info, + type_info, mpi_rank, mpi_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't update modified chunks") + + /* All ranks now collectively re-allocate file space for all chunks */ + if (H5D__mpio_collective_filtered_chunk_reallocate(have_chunk_to_process ? &chunk_list[i] : NULL, + have_chunk_to_process ? 1 : 0, NULL, io_info, + &index_info, mpi_rank, mpi_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, + "couldn't collectively re-allocate file space for chunks") - if (NULL == - (has_chunk_selected_array = (hbool_t *)H5MM_malloc((size_t)mpi_size * sizeof(hbool_t)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate num chunks selected array") - - if (MPI_SUCCESS != - (mpi_code = MPI_Allgather(&have_chunk_to_process, 1, MPI_C_BOOL, has_chunk_selected_array, 1, - MPI_C_BOOL, io_info->comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code) - - /* If this process has a chunk to work on, create a MPI type for the - * memory and file for writing out the chunk + /* + * If this rank has a chunk to work on, create a MPI type + * for writing out the chunk. Otherwise, the rank will + * use MPI_BYTE for the file and memory type and specify + * a count of 0. */ - if (have_chunk_to_process) { - size_t offset; - int mpi_type_count; - - for (j = 0, offset = 0; j < (size_t)mpi_rank; j++) - offset += has_chunk_selected_array[j]; - - /* Collect the new chunk info back to the local copy, since only the record in the - * collective array gets updated by the chunk re-allocation */ - H5MM_memcpy(&chunk_list[i].chunk_states.new_chunk, - &collective_chunk_list[offset].chunk_states.new_chunk, - sizeof(chunk_list[i].chunk_states.new_chunk)); - - H5_CHECKED_ASSIGN(mpi_type_count, int, chunk_list[i].chunk_states.new_chunk.length, hsize_t); - - /* Create MPI memory type for writing to chunk */ - if (MPI_SUCCESS != - (mpi_code = MPI_Type_contiguous(mpi_type_count, MPI_BYTE, &mem_type_array[i]))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) - if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&mem_type_array[i]))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - mem_type_is_derived_array[i] = TRUE; - - /* Create MPI file type for writing to chunk */ - if (MPI_SUCCESS != - (mpi_code = MPI_Type_contiguous(mpi_type_count, MPI_BYTE, &file_type_array[i]))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) - if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(&file_type_array[i]))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - file_type_is_derived_array[i] = TRUE; - - mpi_buf_count = 1; + if (H5D__mpio_collective_filtered_io_type( + have_chunk_to_process ? &chunk_list[i] : NULL, have_chunk_to_process ? 1 : 0, + io_info->op_type, &mem_type, &mem_type_is_derived, &file_type, &file_type_is_derived) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "couldn't create MPI type for writing filtered chunks") - /* Set up the base storage address for this operation */ - ctg_store.contig.dset_addr = chunk_list[i].chunk_states.new_chunk.offset; + mpi_buf_count = (file_type_is_derived || mem_type_is_derived) ? 1 : 0; - /* Override the write buffer to point to the address of the - * chunk data buffer + /* Override the write buffer to point to the chunk data buffer */ + if (have_chunk_to_process) { + /* + * Override the write buffer to point to the + * chunk's data buffer */ ctg_io_info.u.wbuf = chunk_list[i].buf; - } /* end if */ - else { - mem_type_array[i] = file_type_array[i] = MPI_BYTE; - mpi_buf_count = 0; - } /* end else */ + + /* + * Setup the base storage address for this + * operation to be the chunk's file address + */ + ctg_store.contig.dset_addr = chunk_list[i].chunk_new.offset; + } + else + ctg_store.contig.dset_addr = 0; /* Perform the I/O */ - if (H5D__final_collective_io(&ctg_io_info, type_info, mpi_buf_count, file_type_array[i], - mem_type_array[i]) < 0) + if (H5D__final_collective_io(&ctg_io_info, type_info, mpi_buf_count, file_type, mem_type) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "couldn't finish MPI-IO") + /* Free up resources in anticipation of following collective operation */ + if (have_chunk_to_process && chunk_list[i].buf) { + H5MM_free(chunk_list[i].buf); + chunk_list[i].buf = NULL; + } + /* Participate in the collective re-insertion of all chunks modified * in this iteration into the chunk index */ - for (j = 0; j < collective_chunk_list_num_entries; j++) { - udata.chunk_block = collective_chunk_list[j].chunk_states.new_chunk; - udata.common.scaled = collective_chunk_list[j].scaled; - udata.chunk_idx = collective_chunk_list[j].index; - - if ((index_info.storage->ops->insert)(&index_info, &udata, io_info->dset) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, - "unable to insert chunk address into index") - } /* end for */ + if (H5D__mpio_collective_filtered_chunk_reinsert(have_chunk_to_process ? &chunk_list[i] : NULL, + have_chunk_to_process ? 1 : 0, NULL, io_info, + &index_info, mpi_rank, mpi_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, + "couldn't collectively re-insert modified chunks into chunk index") + + /* Free the MPI types, if they were derived */ + if (mem_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + mem_type_is_derived = FALSE; + if (file_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + file_type_is_derived = FALSE; + } /* end for */ + } - if (collective_chunk_list) { - H5MM_free(collective_chunk_list); - collective_chunk_list = NULL; - } /* end if */ - if (has_chunk_selected_array) { - H5MM_free(has_chunk_selected_array); - has_chunk_selected_array = NULL; - } /* end if */ - } /* end for */ +done: + /* Free the MPI buf and file types, if they were derived */ + if (mem_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + if (file_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) - /* Free the MPI file and memory types, if they were derived */ - for (i = 0; i < max_num_chunks; i++) { - if (file_type_is_derived_array[i]) - if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type_array[i]))) - HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + if (chunk_msg_bufs) { + for (i = 0; i < (size_t)chunk_msg_bufs_len; i++) + H5MM_free(chunk_msg_bufs[i]); - if (mem_type_is_derived_array[i]) - if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type_array[i]))) - HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) - } /* end for */ - } /* end else */ + H5MM_free(chunk_msg_bufs); + } -done: + HASH_CLEAR(hh, chunk_hash_table); + + /* Free resources used by a rank which had some selection */ if (chunk_list) { for (i = 0; i < chunk_list_num_entries; i++) if (chunk_list[i].buf) @@ -2015,16 +2350,10 @@ done: H5MM_free(chunk_list); } /* end if */ - if (collective_chunk_list) - H5MM_free(collective_chunk_list); - if (file_type_array) - H5MM_free(file_type_array); - if (mem_type_array) - H5MM_free(mem_type_array); - if (file_type_is_derived_array) - H5MM_free(file_type_is_derived_array); - if (mem_type_is_derived_array) - H5MM_free(mem_type_is_derived_array); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__multi_chunk_filtered_collective_io() */ @@ -2050,11 +2379,22 @@ H5D__inter_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf hbool_t mbt_is_derived = FALSE; hbool_t mft_is_derived = FALSE; MPI_Datatype mpi_file_type, mpi_buf_type; - int mpi_code; /* MPI return code */ - herr_t ret_value = SUCCEED; /* return value */ + int mpi_code; /* MPI return code */ +#ifdef H5Dmpio_DEBUG + int mpi_rank; +#endif + herr_t ret_value = SUCCEED; /* return value */ FUNC_ENTER_STATIC +#ifdef H5Dmpio_DEBUG + mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file); + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Inter collective I/O"); + if (mpi_rank < 0) + HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain MPI rank") +#endif + if ((file_space != NULL) && (mem_space != NULL)) { int mpi_file_count; /* Number of file "objects" to transfer */ hsize_t *permute_map = NULL; /* array that holds the mapping from the old, @@ -2113,9 +2453,8 @@ H5D__inter_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf mft_is_derived = FALSE; } /* end else */ -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "before final collective IO \n"); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG(mpi_rank, "before final collective I/O"); #endif /* Perform final collective I/O operation */ @@ -2129,9 +2468,10 @@ done: if (mft_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&mpi_file_type))) HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "before leaving inter_collective_io ret_value = %d\n", ret_value); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_DEBUG_VA(mpi_rank, "before leaving inter_collective_io ret_value = %d", ret_value); + H5D_MPIO_TRACE_EXIT(mpi_rank); #endif FUNC_LEAVE_NOAPI(ret_value) @@ -2153,10 +2493,21 @@ static herr_t H5D__final_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t mpi_buf_count, MPI_Datatype mpi_file_type, MPI_Datatype mpi_buf_type) { +#ifdef H5Dmpio_DEBUG + int mpi_rank; +#endif herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC +#ifdef H5Dmpio_DEBUG + mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file); + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Final collective I/O"); + if (mpi_rank < 0) + HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain MPI rank") +#endif + /* Pass buf type, file type to the file driver. */ if (H5CX_set_mpi_coll_datatypes(mpi_buf_type, mpi_file_type) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set MPI-I/O collective I/O datatypes") @@ -2171,10 +2522,12 @@ H5D__final_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *type_inf } /* end else */ done: -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "ret_value before leaving final_collective_io=%d\n", ret_value); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_DEBUG_VA(mpi_rank, "ret_value before leaving final_collective_io=%d", ret_value); + H5D_MPIO_TRACE_EXIT(mpi_rank); #endif + FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__final_collective_io */ @@ -2216,62 +2569,149 @@ H5D__cmp_chunk_addr(const void *chunk_addr_info1, const void *chunk_addr_info2) * * Return: -1, 0, 1 * - * Programmer: Jordan Henderson - * Wednesday, Nov. 30th, 2016 - * *------------------------------------------------------------------------- */ static int H5D__cmp_filtered_collective_io_info_entry(const void *filtered_collective_io_info_entry1, const void *filtered_collective_io_info_entry2) { - haddr_t addr1 = HADDR_UNDEF, addr2 = HADDR_UNDEF; + const H5D_filtered_collective_io_info_t *entry1; + const H5D_filtered_collective_io_info_t *entry2; + haddr_t addr1 = HADDR_UNDEF; + haddr_t addr2 = HADDR_UNDEF; + int ret_value; FUNC_ENTER_STATIC_NOERR - addr1 = ((const H5D_filtered_collective_io_info_t *)filtered_collective_io_info_entry1) - ->chunk_states.new_chunk.offset; - addr2 = ((const H5D_filtered_collective_io_info_t *)filtered_collective_io_info_entry2) - ->chunk_states.new_chunk.offset; + entry1 = (const H5D_filtered_collective_io_info_t *)filtered_collective_io_info_entry1; + entry2 = (const H5D_filtered_collective_io_info_t *)filtered_collective_io_info_entry2; - FUNC_LEAVE_NOAPI(H5F_addr_cmp(addr1, addr2)) -} /* end H5D__cmp_filtered_collective_io_info_entry() */ + addr1 = entry1->chunk_new.offset; + addr2 = entry2->chunk_new.offset; -#if MPI_VERSION >= 3 + /* + * If both chunk addresses are defined, H5F_addr_cmp is safe to use. + * Otherwise, if both addresses aren't defined, compared chunk + * entries based on their chunk index. Finally, if only one chunk + * address is defined, return the appropriate value based on which + * is defined. + */ + if (H5F_addr_defined(addr1) && H5F_addr_defined(addr2)) { + ret_value = H5F_addr_cmp(addr1, addr2); + } + else if (!H5F_addr_defined(addr1) && !H5F_addr_defined(addr2)) { + hsize_t chunk_idx1 = entry1->index_info.chunk_idx; + hsize_t chunk_idx2 = entry2->index_info.chunk_idx; + + ret_value = (chunk_idx1 > chunk_idx2) - (chunk_idx1 < chunk_idx2); + } + else + ret_value = H5F_addr_defined(addr1) ? 1 : -1; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__cmp_filtered_collective_io_info_entry() */ /*------------------------------------------------------------------------- - * Function: H5D__cmp_filtered_collective_io_info_entry_owner + * Function: H5D__cmp_chunk_redistribute_info * - * Purpose: Routine to compare filtered collective chunk io info - * entries's original owner fields + * Purpose: Routine to compare two H5D_chunk_redistribute_info_t + * structures * - * Description: Callback for qsort() to compare filtered collective chunk - * io info entries's original owner fields + * Description: Callback for qsort() to compare two + * H5D_chunk_redistribute_info_t structures + * + * Return: -1, 0, 1 + * + *------------------------------------------------------------------------- + */ +static int +H5D__cmp_chunk_redistribute_info(const void *_entry1, const void *_entry2) +{ + const H5D_chunk_redistribute_info_t *entry1; + const H5D_chunk_redistribute_info_t *entry2; + hsize_t chunk_index1; + hsize_t chunk_index2; + int ret_value; + + FUNC_ENTER_STATIC_NOERR + + entry1 = (const H5D_chunk_redistribute_info_t *)_entry1; + entry2 = (const H5D_chunk_redistribute_info_t *)_entry2; + + chunk_index1 = entry1->chunk_idx; + chunk_index2 = entry2->chunk_idx; + + if (chunk_index1 == chunk_index2) { + int orig_owner1 = entry1->orig_owner; + int orig_owner2 = entry2->orig_owner; + + ret_value = (orig_owner1 > orig_owner2) - (orig_owner1 < orig_owner2); + } + else + ret_value = (chunk_index1 > chunk_index2) - (chunk_index1 < chunk_index2); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__cmp_chunk_redistribute_info() */ + +/*------------------------------------------------------------------------- + * Function: H5D__cmp_chunk_redistribute_info_orig_owner * - * Return: The difference between the two - * H5D_filtered_collective_io_info_t's original owner fields + * Purpose: Routine to compare the original owning MPI rank for two + * H5D_chunk_redistribute_info_t structures * - * Programmer: Jordan Henderson - * Monday, Apr. 10th, 2017 + * Description: Callback for qsort() to compare the original owning MPI + * rank for two H5D_chunk_redistribute_info_t + * structures + * + * Return: -1, 0, 1 * *------------------------------------------------------------------------- */ static int -H5D__cmp_filtered_collective_io_info_entry_owner(const void *filtered_collective_io_info_entry1, - const void *filtered_collective_io_info_entry2) +H5D__cmp_chunk_redistribute_info_orig_owner(const void *_entry1, const void *_entry2) { - int owner1 = -1, owner2 = -1; + const H5D_chunk_redistribute_info_t *entry1; + const H5D_chunk_redistribute_info_t *entry2; + int owner1 = -1; + int owner2 = -1; + int ret_value; FUNC_ENTER_STATIC_NOERR - owner1 = ((const H5D_filtered_collective_io_info_t *)filtered_collective_io_info_entry1) - ->owners.original_owner; - owner2 = ((const H5D_filtered_collective_io_info_t *)filtered_collective_io_info_entry2) - ->owners.original_owner; + entry1 = (const H5D_chunk_redistribute_info_t *)_entry1; + entry2 = (const H5D_chunk_redistribute_info_t *)_entry2; - FUNC_LEAVE_NOAPI(owner1 - owner2) -} /* end H5D__cmp_filtered_collective_io_info_entry_owner() */ -#endif + owner1 = entry1->orig_owner; + owner2 = entry2->orig_owner; + + if (owner1 == owner2) { + haddr_t addr1 = entry1->chunk_block.offset; + haddr_t addr2 = entry2->chunk_block.offset; + + /* + * If both chunk addresses are defined, H5F_addr_cmp is safe to use. + * Otherwise, if both addresses aren't defined, compared chunk + * entries based on their chunk index. Finally, if only one chunk + * address is defined, return the appropriate value based on which + * is defined. + */ + if (H5F_addr_defined(addr1) && H5F_addr_defined(addr2)) { + ret_value = H5F_addr_cmp(addr1, addr2); + } + else if (!H5F_addr_defined(addr1) && !H5F_addr_defined(addr2)) { + hsize_t chunk_idx1 = entry1->chunk_idx; + hsize_t chunk_idx2 = entry2->chunk_idx; + + ret_value = (chunk_idx1 > chunk_idx2) - (chunk_idx1 < chunk_idx2); + } + else + ret_value = H5F_addr_defined(addr1) ? 1 : -1; + } + else + ret_value = (owner1 > owner2) - (owner1 < owner2); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__cmp_chunk_redistribute_info_orig_owner() */ /*------------------------------------------------------------------------- * Function: H5D__sort_chunk @@ -2300,7 +2740,7 @@ H5D__cmp_filtered_collective_io_info_entry_owner(const void *filtered_collective */ static herr_t H5D__sort_chunk(H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, - H5D_chunk_addr_info_t chunk_addr_info_array[], int sum_chunk) + H5D_chunk_addr_info_t chunk_addr_info_array[], int sum_chunk, int mpi_rank, int mpi_size) { H5SL_node_t * chunk_node; /* Current node in chunk skip list */ H5D_chunk_info_t *chunk_info; /* Current chunking info. of this node. */ @@ -2312,17 +2752,12 @@ H5D__sort_chunk(H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, hbool_t do_sort = FALSE; /* Whether the addresses need to be sorted */ int bsearch_coll_chunk_threshold; int many_chunk_opt = H5D_OBTAIN_ONE_CHUNK_ADDR_IND; - int mpi_size; /* Number of MPI processes */ int mpi_code; /* MPI return code */ int i; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC - /* Retrieve # of MPI processes */ - if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size") - /* Calculate the actual threshold to obtain all chunk addresses collectively * The bigger this number is, the more possible the use of obtaining chunk * address collectively. @@ -2336,28 +2771,20 @@ H5D__sort_chunk(H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, ((sum_chunk / mpi_size) >= H5D_ALL_CHUNK_ADDR_THRES_COL_NUM)) many_chunk_opt = H5D_OBTAIN_ALL_CHUNK_ADDR_COL; -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "many_chunk_opt= %d\n", many_chunk_opt); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG_VA(mpi_rank, "many_chunk_opt = %d", many_chunk_opt); #endif /* If we need to optimize the way to obtain the chunk address */ if (many_chunk_opt != H5D_OBTAIN_ONE_CHUNK_ADDR_IND) { - int mpi_rank; - -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "Coming inside H5D_OBTAIN_ALL_CHUNK_ADDR_COL\n"); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG(mpi_rank, "Coming inside H5D_OBTAIN_ALL_CHUNK_ADDR_COL"); #endif /* Allocate array for chunk addresses */ if (NULL == (total_chunk_addr_array = (haddr_t *)H5MM_malloc(sizeof(haddr_t) * (size_t)fm->layout->u.chunk.nchunks))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate memory chunk address array") - /* Retrieve all the chunk addresses with process 0 */ - if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank") - if (mpi_rank == 0) { herr_t result; @@ -2437,10 +2864,10 @@ H5D__sort_chunk(H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, chunk_node = H5SL_next(chunk_node); } /* end while */ -#ifdef H5D_DEBUG - if (H5DEBUG(D)) - HDfprintf(H5DEBUG(D), "before Qsort\n"); +#ifdef H5Dmpio_DEBUG + H5D_MPIO_DEBUG(mpi_rank, "before Qsort"); #endif + if (do_sort) { size_t num_chunks = H5SL_count(fm->sel_chunks); @@ -2497,7 +2924,7 @@ done: */ static herr_t H5D__obtain_mpio_mode(H5D_io_info_t *io_info, H5D_chunk_map_t *fm, uint8_t assign_io_mode[], - haddr_t chunk_addr[]) + haddr_t chunk_addr[], int mpi_rank, int mpi_size) { size_t total_chunks; unsigned percent_nproc_per_chunk, threshold_nproc_per_chunk; @@ -2510,7 +2937,6 @@ H5D__obtain_mpio_mode(H5D_io_info_t *io_info, H5D_chunk_map_t *fm, uint8_t assig H5P_coll_md_read_flag_t md_reads_file_flag; hbool_t md_reads_context_flag; hbool_t restore_md_reads_state = FALSE; - int mpi_size, mpi_rank; MPI_Comm comm; int root; size_t ic; @@ -2523,12 +2949,6 @@ H5D__obtain_mpio_mode(H5D_io_info_t *io_info, H5D_chunk_map_t *fm, uint8_t assig root = 0; comm = io_info->comm; - /* Obtain the number of process and the current rank of the process */ - if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank") - if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size") - /* Setup parameters */ H5_CHECKED_ASSIGN(total_chunks, size_t, fm->layout->u.chunk.nchunks, hsize_t); if (H5CX_get_mpio_chunk_opt_ratio(&percent_nproc_per_chunk) < 0) @@ -2672,34 +3092,32 @@ done: } /* end H5D__obtain_mpio_mode() */ /*------------------------------------------------------------------------- - * Function: H5D__construct_filtered_io_info_list + * Function: H5D__mpio_collective_filtered_chunk_io_setup * * Purpose: Constructs a list of entries which contain the necessary * information for inter-process communication when performing * collective io on filtered chunks. This list is used by - * each process when performing I/O on locally selected chunks - * and also in operations that must be collectively done - * on every chunk, such as chunk re-allocation, insertion of - * chunks into the chunk index, etc. + * each MPI rank when performing I/O on locally selected + * chunks and also in operations that must be collectively + * done on every chunk, such as chunk re-allocation, insertion + * of chunks into the chunk index, etc. * * Return: Non-negative on success/Negative on failure * - * Programmer: Jordan Henderson - * Tuesday, January 10th, 2017 - * *------------------------------------------------------------------------- */ static herr_t -H5D__construct_filtered_io_info_list(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - const H5D_chunk_map_t * fm, - H5D_filtered_collective_io_info_t **chunk_list, size_t *num_entries) +H5D__mpio_collective_filtered_chunk_io_setup(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, + const H5D_chunk_map_t * fm, + H5D_filtered_collective_io_info_t **chunk_list, + size_t *num_entries, int mpi_rank) { - H5D_filtered_collective_io_info_t *local_info_array = - NULL; /* The list of initially selected chunks for this process */ - size_t num_chunks_selected; - size_t i; - int mpi_rank; - herr_t ret_value = SUCCEED; + H5D_filtered_collective_io_info_t *local_info_array = NULL; + H5D_chunk_ud_t udata; + hbool_t filter_partial_edge_chunks; + size_t num_chunks_selected; + size_t i; + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC @@ -2709,19 +3127,23 @@ H5D__construct_filtered_io_info_list(const H5D_io_info_t *io_info, const H5D_typ HDassert(chunk_list); HDassert(num_entries); - if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank") +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Filtered Collective I/O Setup"); +#endif - /* Each process builds a local list of the chunks they have selected */ + /* Each rank builds a local list of the chunks they have selected */ if ((num_chunks_selected = H5SL_count(fm->sel_chunks))) { H5D_chunk_info_t *chunk_info; - H5D_chunk_ud_t udata; H5SL_node_t * chunk_node; hsize_t select_npoints; - hssize_t chunk_npoints; + hbool_t need_sort = FALSE; + + /* Determine whether partial edge chunks should be filtered */ + filter_partial_edge_chunks = !(io_info->dset->shared->layout.u.chunk.flags & + H5O_LAYOUT_CHUNK_DONT_FILTER_PARTIAL_BOUND_CHUNKS); - if (NULL == (local_info_array = (H5D_filtered_collective_io_info_t *)H5MM_malloc( - num_chunks_selected * sizeof(H5D_filtered_collective_io_info_t)))) + if (NULL == (local_info_array = H5MM_malloc(num_chunks_selected * sizeof(*local_info_array)))) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate local io info array buffer") chunk_node = H5SL_first(fm->sel_chunks); @@ -2732,743 +3154,2846 @@ H5D__construct_filtered_io_info_list(const H5D_io_info_t *io_info, const H5D_typ if (H5D__chunk_lookup(io_info->dset, chunk_info->scaled, &udata) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address") - local_info_array[i].index = chunk_info->index; - local_info_array[i].chunk_states.chunk_current = local_info_array[i].chunk_states.new_chunk = - udata.chunk_block; - local_info_array[i].num_writers = 0; - local_info_array[i].owners.original_owner = local_info_array[i].owners.new_owner = mpi_rank; - local_info_array[i].buf = NULL; - - local_info_array[i].async_info.num_receive_requests = 0; - local_info_array[i].async_info.receive_buffer_array = NULL; - local_info_array[i].async_info.receive_requests_array = NULL; - - H5MM_memcpy(local_info_array[i].scaled, chunk_info->scaled, sizeof(chunk_info->scaled)); - - select_npoints = H5S_GET_SELECT_NPOINTS(chunk_info->mspace); - local_info_array[i].io_size = (size_t)select_npoints * type_info->src_type_size; - - /* Currently the full overwrite status of a chunk is only obtained on a per-process - * basis. This means that if the total selection in the chunk, as determined by the combination - * of selections of all of the processes interested in the chunk, covers the entire chunk, - * the performance optimization of not reading the chunk from the file is still valid, but - * is not applied in the current implementation. Something like an appropriately placed - * MPI_Allreduce or a running total of the number of chunk points selected during chunk - * redistribution should suffice for implementing this case - JTH. + /* Initialize rank-local chunk info */ + local_info_array[i].chunk_info = chunk_info; + local_info_array[i].chunk_buf_size = 0; + local_info_array[i].num_writers = 0; + local_info_array[i].orig_owner = mpi_rank; + local_info_array[i].new_owner = mpi_rank; + local_info_array[i].buf = NULL; + + select_npoints = H5S_GET_SELECT_NPOINTS(chunk_info->fspace); + local_info_array[i].io_size = (size_t)select_npoints * type_info->dst_type_size; + + /* + * Determine whether this chunk will need to be read from the file. If this is + * a read operation, the chunk will be read. If this is a write operation, we + * generally need to read a filtered chunk from the file before modifying it, + * unless the chunk is being fully overwritten. + * + * TODO: Currently the full overwrite status of a chunk is only obtained on a + * per-rank basis. This means that if the total selection in the chunk, as + * determined by the combination of selections of all of the ranks interested in + * the chunk, covers the entire chunk, the performance optimization of not reading + * the chunk from the file is still valid, but is not applied in the current + * implementation. + * + * To implement this case, a few approaches were considered: + * + * - Keep a running total (distributed to each rank) of the number of chunk + * elements selected during chunk redistribution and compare that to the total + * number of elements in the chunk once redistribution is finished + * + * - Process all incoming chunk messages before doing I/O (these are currently + * processed AFTER doing I/O), combine the owning rank's selection in a chunk + * with the selections received from other ranks and check to see whether that + * combined selection covers the entire chunk + * + * The first approach will be dangerous if the application performs an overlapping + * write to a chunk, as the number of selected elements can equal or exceed the + * number of elements in the chunk without the whole chunk selection being covered. + * While it might be considered erroneous for an application to do an overlapping + * write, we don't explicitly disallow it. + * + * The second approach contains a bit of complexity in that part of the chunk + * messages will be needed before doing I/O and part will be needed after doing I/O. + * Since modification data from chunk messages can't be applied until after any I/O + * is performed (otherwise, we'll overwrite any applied modification data), chunk + * messages are currently entirely processed after I/O. However, in order to determine + * if a chunk is being fully overwritten, we need the dataspace portion of the chunk + * messages before doing I/O. The naive way to do this is to process chunk messages + * twice, using just the relevant information from the message before and after I/O. + * The better way would be to avoid processing chunk messages twice by extracting (and + * keeping around) the dataspace portion of the message before I/O and processing the + * rest of the chunk message after I/O. Note that the dataspace portion of each chunk + * message is used to correctly apply chunk modification data from the message, so + * must be kept around both before and after I/O in this case. */ - if ((chunk_npoints = H5S_GET_EXTENT_NPOINTS(chunk_info->fspace)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid") - local_info_array[i].full_overwrite = - (local_info_array[i].io_size >= (hsize_t)chunk_npoints * type_info->dst_type_size) ? TRUE - : FALSE; + if (io_info->op_type == H5D_IO_OP_READ) + local_info_array[i].need_read = TRUE; + else { + local_info_array[i].need_read = + local_info_array[i].io_size < (size_t)io_info->dset->shared->layout.u.chunk.size; + } - chunk_node = H5SL_next(chunk_node); - } /* end for */ - } /* end if */ + local_info_array[i].skip_filter_pline = FALSE; + if (!filter_partial_edge_chunks) { + /* + * If this is a partial edge chunk and the "don't filter partial edge + * chunks" flag is set, make sure not to apply filters to the chunk. + */ + if (H5D__chunk_is_partial_edge_chunk(io_info->dset->shared->ndims, + io_info->dset->shared->layout.u.chunk.dim, + chunk_info->scaled, io_info->dset->shared->curr_dims)) + local_info_array[i].skip_filter_pline = TRUE; + } - /* Redistribute shared chunks to new owners as necessary */ - if (io_info->op_type == H5D_IO_OP_WRITE) -#if MPI_VERSION >= 3 - if (H5D__chunk_redistribute_shared_chunks(io_info, type_info, fm, local_info_array, - &num_chunks_selected) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to redistribute shared chunks") -#else - HGOTO_ERROR( - H5E_DATASET, H5E_WRITEERROR, FAIL, - "unable to redistribute shared chunks - MPI version < 3 (MPI_Mprobe and MPI_Imrecv missing)") -#endif + /* Initialize the chunk's shared info */ + local_info_array[i].chunk_current = udata.chunk_block; + local_info_array[i].chunk_new = udata.chunk_block; - *chunk_list = local_info_array; - *num_entries = num_chunks_selected; + /* + * Check if the list is not in ascending order of offset in the file + * or has unallocated chunks. In either case, the list should get + * sorted. + */ + if (i) { + haddr_t curr_chunk_offset = local_info_array[i].chunk_current.offset; + haddr_t prev_chunk_offset = local_info_array[i - 1].chunk_current.offset; -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__construct_filtered_io_info_list() */ + if (!H5F_addr_defined(prev_chunk_offset) || !H5F_addr_defined(curr_chunk_offset) || + (curr_chunk_offset < prev_chunk_offset)) + need_sort = TRUE; + } -#if MPI_VERSION >= 3 + /* + * Extensible arrays may calculate a chunk's index a little differently + * than normal when the dataset's unlimited dimension is not the + * slowest-changing dimension, so set the index here based on what the + * extensible array code calculated instead of what was calculated + * in the chunk file mapping. + */ + if (io_info->dset->shared->layout.u.chunk.idx_type == H5D_CHUNK_IDX_EARRAY) + local_info_array[i].index_info.chunk_idx = udata.chunk_idx; + else + local_info_array[i].index_info.chunk_idx = chunk_info->index; -/*------------------------------------------------------------------------- - * Function: H5D__chunk_redistribute_shared_chunks - * - * Purpose: When performing a collective write on a Dataset with - * filters applied, this function is used to redistribute any - * chunks which are selected by more than one process, so as - * to preserve file integrity after the write by ensuring - * that any shared chunks are only modified by one process. - * - * The current implementation follows this 3-phase process: - * - * - Collect everyone's list of chunks into one large list, - * sort the list in increasing order of chunk offset in the - * file and hand the list off to rank 0 - * - * - Rank 0 scans the list looking for matching runs of chunk - * offset in the file (corresponding to a shared chunk which - * has been selected by more than one rank in the I/O - * operation) and for each shared chunk, it redistributes - * the chunk to the process writing to the chunk which - * currently has the least amount of chunks assigned to it - * by modifying the "new_owner" field in each of the list - * entries corresponding to that chunk - * - * - After the chunks have been redistributed, rank 0 re-sorts - * the list in order of previous owner so that each rank - * will get back exactly the array that they contributed to - * the redistribution operation, with the "new_owner" field - * of each chunk they are modifying having possibly been - * modified. Rank 0 then scatters each segment of the list - * back to its corresponding rank + local_info_array[i].index_info.filter_mask = udata.filter_mask; + local_info_array[i].index_info.need_insert = FALSE; + + chunk_node = H5SL_next(chunk_node); + } + + /* Ensure the chunk list is sorted in ascending order of offset in the file */ + if (need_sort) + HDqsort(local_info_array, num_chunks_selected, sizeof(H5D_filtered_collective_io_info_t), + H5D__cmp_filtered_collective_io_info_entry); + +#ifdef H5Dmpio_DEBUG + H5D__mpio_dump_collective_filtered_chunk_list(local_info_array, num_chunks_selected, mpi_rank); +#endif + } + else if (H5F_get_coll_metadata_reads(io_info->dset->oloc.file)) { + hsize_t scaled[H5O_LAYOUT_NDIMS] = {0}; + + /* + * If this rank has no selection in the dataset and collective + * metadata reads are enabled, do a fake lookup of a chunk to + * ensure that this rank has the chunk index opened. Otherwise, + * only the ranks that had a selection will have opened the + * chunk index and they will have done so independently. Therefore, + * when ranks with no selection participate in later collective + * metadata reads, they will try to open the chunk index collectively + * and issues will occur since other ranks won't participate. + * + * In the future, we should consider having a chunk index "open" + * callback that can be used to ensure collectivity between ranks + * in a more natural way, but this hack should suffice for now. + */ + if (H5D__chunk_lookup(io_info->dset, scaled, &udata) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address") + } + + *chunk_list = local_info_array; + *num_entries = num_chunks_selected; + +done: +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_collective_filtered_chunk_io_setup() */ + +/*------------------------------------------------------------------------- + * Function: H5D__mpio_redistribute_shared_chunks + * + * Purpose: When performing a parallel write on a chunked Dataset with + * filters applied, we must ensure that any particular chunk + * is only written to by a single MPI rank in order to avoid + * potential data races on the chunk. This function is used to + * redistribute (by assigning ownership to a single rank) any + * chunks which are selected by more than one MPI rank. + * + * An initial Allgather is performed to determine how many + * chunks each rank has selected in the write operation and + * then that number is compared against a threshold value to + * determine whether chunk redistribution should be done on + * MPI rank 0 only, or on all MPI ranks. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_redistribute_shared_chunks(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, const H5D_io_info_t *io_info, + const H5D_chunk_map_t *fm, int mpi_rank, int mpi_size, + size_t **rank_chunks_assigned_map) +{ + hbool_t redistribute_on_all_ranks; + size_t *num_chunks_map = NULL; + size_t coll_chunk_list_size = 0; + size_t i; + int mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(chunk_list || 0 == chunk_list_num_entries); + HDassert(io_info); + HDassert(fm); + HDassert(mpi_size > 1); /* No chunk sharing is possible for MPI Comm size of 1 */ + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Redistribute shared chunks"); +#endif + + /* + * Allocate an array for each rank to keep track of the number of + * chunks assigned to any other rank in order to cut down on future + * MPI communication. + */ + if (NULL == (num_chunks_map = H5MM_malloc((size_t)mpi_size * sizeof(*num_chunks_map)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't allocate assigned chunks array") + + /* Perform initial Allgather to determine the collective chunk list size */ + if (MPI_SUCCESS != (mpi_code = MPI_Allgather(&chunk_list_num_entries, 1, H5_SIZE_T_AS_MPI_TYPE, + num_chunks_map, 1, H5_SIZE_T_AS_MPI_TYPE, io_info->comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code) + + for (i = 0; i < (size_t)mpi_size; i++) + coll_chunk_list_size += num_chunks_map[i]; + + /* + * Determine whether we should perform chunk redistribution on all + * ranks or just rank 0. For a relatively small number of chunks, + * we redistribute on all ranks to cut down on MPI communication + * overhead. For a larger number of chunks, we redistribute on + * rank 0 only to cut down on memory usage. + */ + redistribute_on_all_ranks = coll_chunk_list_size < H5D_CHUNK_REDISTRIBUTE_THRES; + + if (H5D__mpio_redistribute_shared_chunks_int(chunk_list, num_chunks_map, redistribute_on_all_ranks, + io_info, fm, mpi_rank, mpi_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTREDISTRIBUTE, FAIL, "can't redistribute shared chunks") + + /* + * If the caller provided a pointer for the mapping from + * rank value -> number of chunks assigned, return that + * mapping here. + */ + if (rank_chunks_assigned_map) { + /* + * If we performed chunk redistribution on rank 0 only, distribute + * the rank value -> number of chunks assigned mapping back to all + * ranks. + */ + if (!redistribute_on_all_ranks) { + if (MPI_SUCCESS != + (mpi_code = MPI_Bcast(num_chunks_map, mpi_size, H5_SIZE_T_AS_MPI_TYPE, 0, io_info->comm))) + HMPI_GOTO_ERROR(FAIL, "couldn't broadcast chunk mapping to other ranks", mpi_code) + } + + *rank_chunks_assigned_map = num_chunks_map; + } + +done: + if (!rank_chunks_assigned_map || (ret_value < 0)) { + num_chunks_map = H5MM_xfree(num_chunks_map); + } + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_redistribute_shared_chunks() */ + +/*------------------------------------------------------------------------- + * Function: H5D__mpio_redistribute_shared_chunks_int + * + * Purpose: Routine to perform redistribution of shared chunks during + * parallel writes to datasets with filters applied. + * + * If `all_ranks_involved` is TRUE, chunk redistribution + * occurs on all MPI ranks. This is usually done when there + * is a relatively small number of chunks involved in order to + * cut down on MPI communication overhead while increasing + * total memory usage a bit. + * + * If `all_ranks_involved` is FALSE, only rank 0 will perform + * chunk redistribution. This is usually done when there is + * a relatively large number of chunks involved in order to + * cut down on total memory usage at the cost of increased + * overhead from MPI communication. + * + * This implementation is as follows: + * + * - All MPI ranks send their list of selected chunks to the + * ranks involved in chunk redistribution. Then, the + * involved ranks sort this new list in order of chunk + * index. + * + * - The involved ranks scan the list looking for matching + * runs of chunk index values (corresponding to a shared + * chunk which has been selected by more than one rank in + * the I/O operation) and for each shared chunk, + * redistribute the chunk to the MPI rank writing to the + * chunk which currently has the least amount of chunks + * assigned to it. This is done by modifying the "new_owner" + * field in each of the list entries corresponding to that + * chunk. The involved ranks then re-sort the list in order + * of original chunk owner so that each rank's section of + * contributed chunks is contiguous in the collective chunk + * list. + * + * - If chunk redistribution occurred on all ranks, each rank + * scans through the collective chunk list to find their + * contributed section of chunks and uses that to update + * their local chunk list with the newly-updated "new_owner" + * and "num_writers" fields. If chunk redistribution + * occurred only on rank 0, an MPI_Scatterv operation will + * be used to scatter the segments of the collective chunk + * list from rank 0 back to the corresponding ranks. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_redistribute_shared_chunks_int(H5D_filtered_collective_io_info_t *chunk_list, + size_t *num_chunks_assigned_map, hbool_t all_ranks_involved, + const H5D_io_info_t *io_info, const H5D_chunk_map_t *fm, + int mpi_rank, int mpi_size) +{ + MPI_Datatype struct_type; + MPI_Datatype packed_type; + hbool_t struct_type_derived = FALSE; + hbool_t packed_type_derived = FALSE; + size_t i; + size_t coll_chunk_list_num_entries = 0; + void * coll_chunk_list = NULL; + int * counts_disps_array = NULL; + int * counts_ptr = NULL; + int * displacements_ptr = NULL; + int num_chunks_int; + int mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(num_chunks_assigned_map); + HDassert(chunk_list || 0 == num_chunks_assigned_map[mpi_rank]); + HDassert(io_info); + HDassert(fm); + HDassert(mpi_size > 1); + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Redistribute shared chunks (internal)"); +#endif + + /* + * Make sure it's safe to cast this rank's number + * of chunks to be sent into an int for MPI + */ + H5_CHECKED_ASSIGN(num_chunks_int, int, num_chunks_assigned_map[mpi_rank], size_t); + + /* + * Phase 1 - Participate in collective gathering of every rank's + * list of chunks to the ranks which are performing the redistribution + * operation. + */ + + if (all_ranks_involved || (mpi_rank == 0)) { + /* + * Allocate array to store the receive counts of each rank, as well as + * the displacements into the final array where each rank will place + * their data. The first half of the array contains the receive counts + * (in rank order), while the latter half contains the displacements + * (also in rank order). + */ + if (NULL == (counts_disps_array = H5MM_malloc(2 * (size_t)mpi_size * sizeof(*counts_disps_array)))) { + /* Push an error, but still participate in collective gather operation */ + HDONE_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "couldn't allocate receive counts and displacements array") + } + else { + /* Set the receive counts from the assigned chunks map */ + counts_ptr = counts_disps_array; + + for (i = 0; i < (size_t)mpi_size; i++) + H5_CHECKED_ASSIGN(counts_ptr[i], int, num_chunks_assigned_map[i], size_t); + + /* Set the displacements into the receive buffer for the gather operation */ + displacements_ptr = &counts_disps_array[mpi_size]; + + *displacements_ptr = 0; + for (i = 1; i < (size_t)mpi_size; i++) + displacements_ptr[i] = displacements_ptr[i - 1] + counts_ptr[i - 1]; + } + } + + /* + * Construct MPI derived types for extracting information + * necessary for MPI communication + */ + if (H5D__mpio_get_chunk_redistribute_info_types(&packed_type, &packed_type_derived, &struct_type, + &struct_type_derived) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "can't create derived datatypes for chunk redistribution info") + + /* Perform gather operation */ + if (H5_mpio_gatherv_alloc(chunk_list, num_chunks_int, struct_type, counts_ptr, displacements_ptr, + packed_type, all_ranks_involved, 0, io_info->comm, mpi_rank, mpi_size, + &coll_chunk_list, &coll_chunk_list_num_entries) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, + "can't gather chunk redistribution info to involved ranks") + + /* + * If all ranks are redistributing shared chunks, we no + * longer need the receive counts and displacements array + */ + if (all_ranks_involved) { + counts_disps_array = H5MM_xfree(counts_disps_array); + } + + /* + * Phase 2 - Involved ranks now redistribute any shared chunks to new + * owners as necessary. + */ + + if (all_ranks_involved || (mpi_rank == 0)) { + H5D_chunk_redistribute_info_t *chunk_entry; + hsize_t curr_chunk_idx; + size_t set_begin_index; + int num_writers; + int new_chunk_owner; + + /* Clear the mapping from rank value -> number of assigned chunks */ + HDmemset(num_chunks_assigned_map, 0, (size_t)mpi_size * sizeof(*num_chunks_assigned_map)); + + /* Sort collective chunk list according to chunk index */ + HDqsort(coll_chunk_list, coll_chunk_list_num_entries, sizeof(H5D_chunk_redistribute_info_t), + H5D__cmp_chunk_redistribute_info); + + /* + * Process all chunks in the collective chunk list. + * Note that the loop counter is incremented by both + * the outer loop (while processing each entry in + * the collective chunk list) and the inner loop + * (while processing duplicate entries for shared + * chunks). + */ + chunk_entry = &((H5D_chunk_redistribute_info_t *)coll_chunk_list)[0]; + for (i = 0; i < coll_chunk_list_num_entries;) { + /* Set chunk's initial new owner to its original owner */ + new_chunk_owner = chunk_entry->orig_owner; + + /* + * Set the current chunk index so we know when we've processed + * all duplicate entries for a particular shared chunk + */ + curr_chunk_idx = chunk_entry->chunk_idx; + + /* Reset the initial number of writers to this chunk */ + num_writers = 0; + + /* Set index for the beginning of this section of duplicate chunk entries */ + set_begin_index = i; + + /* + * Process each chunk entry in the set for the current + * (possibly shared) chunk and increment the loop counter + * while doing so. + */ + do { + /* + * The new owner of the chunk is determined by the rank + * writing to the chunk which currently has the least amount + * of chunks assigned to it + */ + if (num_chunks_assigned_map[chunk_entry->orig_owner] < + num_chunks_assigned_map[new_chunk_owner]) + new_chunk_owner = chunk_entry->orig_owner; + + /* Update the number of writers to this particular chunk */ + num_writers++; + + chunk_entry++; + } while (++i < coll_chunk_list_num_entries && chunk_entry->chunk_idx == curr_chunk_idx); + + /* We should never have more writers to a chunk than the number of MPI ranks */ + HDassert(num_writers <= mpi_size); + + /* Set all processed chunk entries' "new_owner" and "num_writers" fields */ + for (; set_begin_index < i; set_begin_index++) { + H5D_chunk_redistribute_info_t *entry; + + entry = &((H5D_chunk_redistribute_info_t *)coll_chunk_list)[set_begin_index]; + + entry->new_owner = new_chunk_owner; + entry->num_writers = num_writers; + } + + /* Update the number of chunks assigned to the MPI rank that now owns this chunk */ + num_chunks_assigned_map[new_chunk_owner]++; + } + + /* + * Re-sort the collective chunk list in order of original chunk owner + * so that each rank's section of contributed chunks is contiguous in + * the collective chunk list. + * + * NOTE: this re-sort is frail in that it needs to sort the collective + * chunk list so that each rank's section of contributed chunks + * is in the exact order it was contributed in, or things will + * be scrambled when each rank's local chunk list is updated. + * Therefore, the sorting algorithm here is tied to the one + * used during the I/O setup operation. Specifically, chunks + * are first sorted by ascending order of offset in the file and + * then by chunk index. In the future, a better redistribution + * algorithm may be devised that doesn't rely on frail sorting, + * but the current implementation is a quick and naive approach. + */ + HDqsort(coll_chunk_list, coll_chunk_list_num_entries, sizeof(H5D_chunk_redistribute_info_t), + H5D__cmp_chunk_redistribute_info_orig_owner); + } + + if (all_ranks_involved) { + /* + * If redistribution occurred on all ranks, search for the section + * in the collective chunk list corresponding to this rank's locally + * selected chunks and update the local list after redistribution. + */ + for (i = 0; i < coll_chunk_list_num_entries; i++) + if (mpi_rank == ((H5D_chunk_redistribute_info_t *)coll_chunk_list)[i].orig_owner) + break; + + for (size_t j = 0; j < (size_t)num_chunks_int; j++) { + H5D_chunk_redistribute_info_t *coll_entry; + + coll_entry = &((H5D_chunk_redistribute_info_t *)coll_chunk_list)[i++]; + + chunk_list[j].new_owner = coll_entry->new_owner; + chunk_list[j].num_writers = coll_entry->num_writers; + } + } + else { + /* + * If redistribution occurred only on rank 0, scatter the segments + * of the collective chunk list back to each rank so that their + * local chunk lists get updated + */ + if (MPI_SUCCESS != + (mpi_code = MPI_Scatterv(coll_chunk_list, counts_ptr, displacements_ptr, packed_type, chunk_list, + num_chunks_int, struct_type, 0, io_info->comm))) + HMPI_GOTO_ERROR(FAIL, "unable to scatter shared chunks info buffer", mpi_code) + } + +#ifdef H5Dmpio_DEBUG + H5D__mpio_dump_collective_filtered_chunk_list(chunk_list, num_chunks_assigned_map[mpi_rank], mpi_rank); +#endif + +done: + H5MM_free(coll_chunk_list); + + if (struct_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&struct_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } + if (packed_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&packed_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } + + H5MM_free(counts_disps_array); + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_redistribute_shared_chunks_int() */ + +/*------------------------------------------------------------------------- + * Function: H5D__mpio_share_chunk_modification_data + * + * Purpose: When performing a parallel write on a chunked dataset with + * filters applied, we must first ensure that any particular + * chunk is only written to by a single MPI rank in order to + * avoid potential data races on the chunk. Once dataset + * chunks have been redistributed in a suitable manner, each + * MPI rank must send its chunk data to other ranks for each + * chunk it no longer owns. + * + * The current implementation here follows the Nonblocking + * Consensus algorithm described in: + * http://unixer.de/publications/img/hoefler-dsde-protocols.pdf + * + * First, each MPI rank scans through its list of selected + * chunks and does the following for each chunk: + * + * * If a chunk in the MPI rank's chunk list is still owned + * by that rank, the rank checks how many messages are + * incoming for that chunk and adds that to its running + * total. Then, the rank updates its local chunk list so + * that any previous chunk entries for chunks that are no + * longer owned by the rank get overwritten by chunk + * entries for chunks the rank still owns. Since the data + * for the chunks no longer owned will have already been + * sent, those chunks can effectively be discarded. + * * If a chunk in the MPI rank's chunk list is no longer + * owned by that rank, the rank sends the data it wishes to + * update the chunk with to the MPI rank that now has + * ownership of that chunk. To do this, it encodes the + * chunk's index, its selection in the chunk and its + * modification data into a buffer and then posts a + * non-blocking MPI_Issend to the owning rank. + * + * Once this step is complete, all MPI ranks allocate arrays + * to hold chunk message receive buffers and MPI request + * objects for each non-blocking receive they will post for + * incoming chunk modification messages. Then, all MPI ranks + * enter a loop that alternates between non-blocking + * MPI_Iprobe calls to probe for incoming messages and + * MPI_Testall calls to see if all send requests have + * completed. As chunk modification messages arrive, + * non-blocking MPI_Irecv calls will be posted for each + * message. + * + * Once all send requests have completed, an MPI_Ibarrier is + * posted and the loop then alternates between MPI_Iprobe + * calls and MPI_Test calls to check if all ranks have reached + * the non-blocking barrier. Once all ranks have reached the + * barrier, processing can move on to updating the selected + * chunks that are owned in the operation. + * + * Any chunk messages that were received from other ranks + * will be returned through the `chunk_msg_bufs` array and + * `chunk_msg_bufs_len` will be set appropriately. + * + * NOTE: The use of non-blocking sends and receives of chunk + * data here may contribute to large amounts of memory + * usage/MPI request overhead if the number of shared + * chunks is high. If this becomes a problem, it may be + * useful to split the message receiving loop away so + * that chunk modification messages can be received and + * processed immediately (MPI_Recv) using a single chunk + * message buffer. However, it's possible this may + * degrade performance since the chunk message sends + * are synchronous (MPI_Issend) in the Nonblocking + * Consensus algorithm. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_share_chunk_modification_data(H5D_filtered_collective_io_info_t *chunk_list, + size_t *chunk_list_num_entries, H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, int mpi_rank, int mpi_size, + H5D_filtered_collective_io_info_t **chunk_hash_table, + unsigned char ***chunk_msg_bufs, int *chunk_msg_bufs_len) +{ +#if MPI_VERSION >= 3 + H5D_filtered_collective_io_info_t *chunk_table = NULL; + H5S_sel_iter_t * mem_iter = NULL; + unsigned char ** msg_send_bufs = NULL; + unsigned char ** msg_recv_bufs = NULL; + MPI_Request * send_requests = NULL; + MPI_Request * recv_requests = NULL; + MPI_Request ibarrier = MPI_REQUEST_NULL; + hbool_t mem_iter_init = FALSE; + hbool_t ibarrier_posted = FALSE; + size_t send_bufs_nalloc = 0; + size_t num_send_requests = 0; + size_t num_recv_requests = 0; + size_t num_msgs_incoming = 0; + size_t last_assigned_idx; + size_t i; + int mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(chunk_list_num_entries); + HDassert(chunk_list || 0 == *chunk_list_num_entries); + HDassert(io_info); + HDassert(type_info); + HDassert(mpi_size > 1); + HDassert(chunk_msg_bufs); + HDassert(chunk_msg_bufs_len); + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Share chunk modification data"); +#endif + + /* Set to latest format for encoding dataspace */ + H5CX_set_libver_bounds(NULL); + + if (*chunk_list_num_entries) { + /* Allocate a selection iterator for iterating over chunk dataspaces */ + if (NULL == (mem_iter = H5FL_MALLOC(H5S_sel_iter_t))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate dataspace selection iterator") + + /* + * Allocate send buffer and MPI_Request arrays for non-blocking + * sends of outgoing chunk messages + */ + send_bufs_nalloc = H5D_CHUNK_NUM_SEND_MSGS_INIT; + if (NULL == (msg_send_bufs = H5MM_malloc(send_bufs_nalloc * sizeof(*msg_send_bufs)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, + "couldn't allocate chunk modification message buffer array") + + if (NULL == (send_requests = H5MM_malloc(send_bufs_nalloc * sizeof(*send_requests)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate send requests array") + } + + /* + * For each chunk this rank owns, add to the total number of + * incoming MPI messages, then update the local chunk list to + * overwrite any previous chunks no longer owned by this rank. + * Since the data for those chunks will have already been sent, + * this rank should no longer be interested in them and they + * can effectively be discarded. This bookkeeping also makes + * the code for the collective file space re-allocation and + * chunk re-insertion operations a bit simpler. + * + * For each chunk this rank doesn't own, use non-blocking + * synchronous sends to send the data this rank is writing to + * the rank that does own the chunk. + */ + for (i = 0, last_assigned_idx = 0; i < *chunk_list_num_entries; i++) { + H5D_filtered_collective_io_info_t *chunk_entry = &chunk_list[i]; + + if (mpi_rank == chunk_entry->new_owner) { + num_msgs_incoming += (size_t)(chunk_entry->num_writers - 1); + + /* + * Overwrite chunk entries this rank doesn't own with entries that it + * does own, since it has sent the necessary data and is no longer + * interested in the chunks it doesn't own. + */ + chunk_list[last_assigned_idx] = chunk_list[i]; + + /* + * Since, at large scale, a chunk's index value may be larger than + * the maximum value that can be stored in an int, we cannot rely + * on using a chunk's index value as the tag for the MPI messages + * sent/received for a chunk. Therefore, add this chunk to a hash + * table with the chunk's index as a key so that we can quickly find + * the chunk when processing chunk messages that were received. The + * message itself will contain the chunk's index so we can update + * the correct chunk with the received data. + */ + HASH_ADD(hh, chunk_table, index_info.chunk_idx, sizeof(hsize_t), &chunk_list[last_assigned_idx]); + + last_assigned_idx++; + } + else { + H5D_chunk_info_t *chunk_info = chunk_entry->chunk_info; + unsigned char * mod_data_p = NULL; + hsize_t iter_nelmts; + size_t mod_data_size = 0; + size_t space_size = 0; + + /* Add the size of the chunk index to the encoded size */ + mod_data_size += sizeof(hsize_t); + + /* Determine size of serialized chunk file dataspace */ + if (H5S_encode(chunk_info->fspace, &mod_data_p, &space_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get encoded dataspace size") + mod_data_size += space_size; + + /* Determine size of data being written */ + iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->mspace); + H5_CHECK_OVERFLOW(iter_nelmts, hsize_t, size_t); + + mod_data_size += (size_t)iter_nelmts * type_info->src_type_size; + + if (NULL == (msg_send_bufs[num_send_requests] = H5MM_malloc(mod_data_size))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, + "couldn't allocate chunk modification message buffer") + + mod_data_p = msg_send_bufs[num_send_requests]; + + /* Store the chunk's index into the buffer */ + HDmemcpy(mod_data_p, &chunk_entry->index_info.chunk_idx, sizeof(hsize_t)); + mod_data_p += sizeof(hsize_t); + + /* Serialize the chunk's file dataspace into the buffer */ + if (H5S_encode(chunk_info->fspace, &mod_data_p, &mod_data_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "unable to encode dataspace") + + /* Initialize iterator for memory selection */ + if (H5S_select_iter_init(mem_iter, chunk_info->mspace, type_info->src_type_size, + H5S_SEL_ITER_SHARE_WITH_DATASPACE) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to initialize memory selection information") + mem_iter_init = TRUE; + + /* Collect the modification data into the buffer */ + if (0 == H5D__gather_mem(io_info->u.wbuf, mem_iter, (size_t)iter_nelmts, mod_data_p)) + HGOTO_ERROR(H5E_IO, H5E_CANTGATHER, FAIL, "couldn't gather from write buffer") + + /* + * Ensure that the size of the chunk data being sent can be + * safely cast to an int for MPI. Note that this should + * generally be OK for now (unless a rank is sending a + * whole 32-bit-sized chunk of data + its encoded selection), + * but if we allow larger than 32-bit-sized chunks in the + * future, this may become a problem and derived datatypes + * will need to be used. + */ + H5_CHECK_OVERFLOW(mod_data_size, size_t, int) + + /* Send modification data to new owner */ + if (MPI_SUCCESS != + (mpi_code = MPI_Issend(msg_send_bufs[num_send_requests], (int)mod_data_size, MPI_BYTE, + chunk_entry->new_owner, H5D_CHUNK_MOD_DATA_TAG, io_info->comm, + &send_requests[num_send_requests]))) + HMPI_GOTO_ERROR(FAIL, "MPI_Issend failed", mpi_code) + + num_send_requests++; + + /* Resize send buffer and send request arrays if necessary */ + if (num_send_requests == send_bufs_nalloc) { + void *tmp_alloc; + + send_bufs_nalloc = (size_t)((double)send_bufs_nalloc * 1.5); + + if (NULL == + (tmp_alloc = H5MM_realloc(msg_send_bufs, send_bufs_nalloc * sizeof(*msg_send_bufs)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, + "couldn't resize chunk modification message buffer array") + msg_send_bufs = tmp_alloc; + + if (NULL == + (tmp_alloc = H5MM_realloc(send_requests, send_bufs_nalloc * sizeof(*send_requests)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't resize send requests array") + send_requests = tmp_alloc; + } + + if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release memory selection iterator") + mem_iter_init = FALSE; + } + } + + /* Check if the number of send or receive requests will overflow an int (MPI requirement) */ + if (num_send_requests > INT_MAX || num_msgs_incoming > INT_MAX) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, + "too many shared chunks in parallel filtered write operation") + + H5_CHECK_OVERFLOW(num_send_requests, size_t, int) + H5_CHECK_OVERFLOW(num_msgs_incoming, size_t, int) + + /* + * Allocate receive buffer and MPI_Request arrays for non-blocking + * receives of incoming chunk messages + */ + if (num_msgs_incoming) { + if (NULL == (msg_recv_bufs = H5MM_malloc(num_msgs_incoming * sizeof(*msg_recv_bufs)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, + "couldn't allocate chunk modification message buffer array") + + if (NULL == (recv_requests = H5MM_malloc(num_msgs_incoming * sizeof(*recv_requests)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate receive requests array") + } + + /* Process any incoming messages until everyone is done */ + do { + MPI_Status status; + int msg_flag; + + /* Probe for an incoming message from any rank */ + if (MPI_SUCCESS != (mpi_code = MPI_Iprobe(MPI_ANY_SOURCE, H5D_CHUNK_MOD_DATA_TAG, io_info->comm, + &msg_flag, &status))) + HMPI_GOTO_ERROR(FAIL, "MPI_Iprobe failed", mpi_code) + + /* + * If a message was found, allocate a buffer for the message and + * post a non-blocking receive to receive it + */ + if (msg_flag) { +#if MPI_VERSION >= 3 + MPI_Count msg_size = 0; + + if (MPI_SUCCESS != (mpi_code = MPI_Get_elements_x(&status, MPI_BYTE, &msg_size))) + HMPI_GOTO_ERROR(FAIL, "MPI_Get_elements_x failed", mpi_code) + + H5_CHECK_OVERFLOW(msg_size, MPI_Count, int) +#else + int msg_size = 0; + + if (MPI_SUCCESS != (mpi_code = MPI_Get_elements(&status, MPI_BYTE, &msg_size))) + HMPI_GOTO_ERROR(FAIL, "MPI_Get_elements failed", mpi_code) +#endif + + if (msg_size <= 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "invalid chunk modification message size") + + HDassert((num_recv_requests + 1) <= num_msgs_incoming); + if (NULL == + (msg_recv_bufs[num_recv_requests] = H5MM_malloc((size_t)msg_size * sizeof(unsigned char)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, + "couldn't allocate chunk modification message receive buffer") + + if (MPI_SUCCESS != (mpi_code = MPI_Irecv(msg_recv_bufs[num_recv_requests], (int)msg_size, + MPI_BYTE, status.MPI_SOURCE, H5D_CHUNK_MOD_DATA_TAG, + io_info->comm, &recv_requests[num_recv_requests]))) + HMPI_GOTO_ERROR(FAIL, "MPI_Irecv failed", mpi_code) + + num_recv_requests++; + } + + if (ibarrier_posted) { + int ibarrier_completed; + + if (MPI_SUCCESS != (mpi_code = MPI_Test(&ibarrier, &ibarrier_completed, MPI_STATUS_IGNORE))) + HMPI_GOTO_ERROR(FAIL, "MPI_Test failed", mpi_code) + + if (ibarrier_completed) + break; + } + else { + int all_sends_completed; + + /* Determine if all send requests have completed */ + if (MPI_SUCCESS != (mpi_code = MPI_Testall((int)num_send_requests, send_requests, + &all_sends_completed, MPI_STATUSES_IGNORE))) + HMPI_GOTO_ERROR(FAIL, "MPI_Testall failed", mpi_code) + + if (all_sends_completed) { + /* Post non-blocking barrier */ + if (MPI_SUCCESS != (mpi_code = MPI_Ibarrier(io_info->comm, &ibarrier))) + HMPI_GOTO_ERROR(FAIL, "MPI_Ibarrier failed", mpi_code) + ibarrier_posted = TRUE; + + /* + * Now that all send requests have completed, free up the + * send buffers used in the non-blocking operations + */ + if (msg_send_bufs) { + for (i = 0; i < num_send_requests; i++) { + if (msg_send_bufs[i]) + H5MM_free(msg_send_bufs[i]); + } + + msg_send_bufs = H5MM_xfree(msg_send_bufs); + } + } + } + } while (1); + + /* + * Ensure all receive requests have completed before moving on. + * For linked-chunk I/O, more overlap with computation could + * theoretically be achieved by returning the receive requests + * array and postponing this wait until during chunk updating + * when the data is really needed. However, multi-chunk I/O + * only updates a chunk at a time and the messages may not come + * in the order that chunks are processed. So, the safest way to + * support both I/O modes is to simply make sure all messages + * are available. + */ + if (MPI_SUCCESS != (mpi_code = MPI_Waitall((int)num_recv_requests, recv_requests, MPI_STATUSES_IGNORE))) + HMPI_GOTO_ERROR(FAIL, "MPI_Waitall failed", mpi_code) + + /* Set the new number of locally-selected chunks */ + *chunk_list_num_entries = last_assigned_idx; + + /* Return chunk message buffers if any were received */ + *chunk_hash_table = chunk_table; + *chunk_msg_bufs = msg_recv_bufs; + *chunk_msg_bufs_len = (int)num_recv_requests; + +done: + if (ret_value < 0) { + /* If this rank failed, make sure to participate in collective barrier */ + if (!ibarrier_posted) { + if (MPI_SUCCESS != (mpi_code = MPI_Ibarrier(io_info->comm, &ibarrier))) + HMPI_GOTO_ERROR(FAIL, "MPI_Ibarrier failed", mpi_code) + } + + if (num_send_requests) { + for (i = 0; i < num_send_requests; i++) { + MPI_Cancel(&send_requests[i]); + } + } + + if (recv_requests) { + for (i = 0; i < num_recv_requests; i++) { + MPI_Cancel(&recv_requests[i]); + } + } + + if (msg_recv_bufs) { + for (i = 0; i < num_recv_requests; i++) { + H5MM_free(msg_recv_bufs[i]); + } + + H5MM_free(msg_recv_bufs); + } + + HASH_CLEAR(hh, chunk_table); + } + + if (recv_requests) + H5MM_free(recv_requests); + if (send_requests) + H5MM_free(send_requests); + + if (msg_send_bufs) { + for (i = 0; i < num_send_requests; i++) { + if (msg_send_bufs[i]) + H5MM_free(msg_send_bufs[i]); + } + + H5MM_free(msg_send_bufs); + } + + if (mem_iter) { + if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release dataspace selection iterator") + mem_iter = H5FL_FREE(H5S_sel_iter_t, mem_iter); + } + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif + + FUNC_LEAVE_NOAPI(ret_value) +#else + FUNC_ENTER_STATIC + HERROR( + H5E_DATASET, H5E_WRITEERROR, + "unable to send chunk modification data between MPI ranks - MPI version < 3 (MPI_Ibarrier missing)") + FUNC_LEAVE_NOAPI(FAIL) +#endif +} /* end H5D__mpio_share_chunk_modification_data() */ + +/*------------------------------------------------------------------------- + * Function: H5D__mpio_collective_filtered_chunk_common_io + * + * Purpose: This routine performs the common part of collective I/O + * when reading or writing filtered chunks collectively. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_collective_filtered_chunk_common_io(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, const H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, int mpi_size) +{ + H5D_io_info_t coll_io_info; + H5D_storage_t ctg_store; + MPI_Datatype file_type = MPI_DATATYPE_NULL; + MPI_Datatype mem_type = MPI_DATATYPE_NULL; + hbool_t mem_type_is_derived = FALSE; + hbool_t file_type_is_derived = FALSE; + hsize_t mpi_buf_count; + haddr_t base_read_offset = HADDR_UNDEF; + size_t num_chunks; + size_t i; + char fake_buf; /* Used as a fake buffer for ranks with no chunks, thus a NULL buf pointer */ + int mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(chunk_list || 0 == chunk_list_num_entries); + HDassert(io_info); + HDassert(type_info); + + /* Initialize temporary I/O info */ + coll_io_info = *io_info; + + /* + * Construct MPI derived datatype for collective I/O on chunks + */ + if (H5D__mpio_collective_filtered_io_type(chunk_list, chunk_list_num_entries, io_info->op_type, &mem_type, + &mem_type_is_derived, &file_type, &file_type_is_derived) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADTYPE, FAIL, "couldn't create MPI I/O type for chunk I/O") + + /* + * For reads, determine how many chunks are actually being read. + * Note that if this is a read during a write operation + * (read chunk -> unfilter -> modify -> write back), some + * chunks may not need to be read if they're being fully + * overwritten during a write operation. + */ + if (io_info->op_type == H5D_IO_OP_READ) { + for (i = 0, num_chunks = 0; i < chunk_list_num_entries; i++) { + HDassert(chunk_list[i].buf); + + if (chunk_list[i].need_read) { + if (!H5F_addr_defined(base_read_offset)) + base_read_offset = chunk_list[i].chunk_current.offset; + + num_chunks++; + } + } + } + else + num_chunks = chunk_list_num_entries; + + /* + * If this rank doesn't have a selection, it can + * skip I/O if independent I/O was requested at + * the low level, or if the MPI communicator size + * is 1. + * + * Otherwise, this rank has to participate in + * collective I/O, but probably has a NULL buf + * pointer, so override to a fake buffer since our + * write/read function expects one. + */ + if (num_chunks == 0) { + H5FD_mpio_collective_opt_t coll_opt_mode; + + /* Get the collective_opt property to check whether the application wants to do IO individually. */ + if (H5CX_get_mpio_coll_opt(&coll_opt_mode) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get MPI-I/O collective_opt property") + + if ((mpi_size == 1) || (H5FD_MPIO_INDIVIDUAL_IO == coll_opt_mode)) { + HGOTO_DONE(SUCCEED) + } + else { + if (io_info->op_type == H5D_IO_OP_WRITE) + coll_io_info.u.wbuf = &fake_buf; + else + coll_io_info.u.rbuf = &fake_buf; + } + } + + /* + * Setup for I/O operation + */ + + mpi_buf_count = (num_chunks) ? 1 : 0; + + if (num_chunks) { + /* + * Setup the base storage address for this operation + * to be the first chunk's file address + */ + if (io_info->op_type == H5D_IO_OP_WRITE) + ctg_store.contig.dset_addr = chunk_list[0].chunk_new.offset; + else + ctg_store.contig.dset_addr = base_read_offset; + } + else + ctg_store.contig.dset_addr = 0; + + ctg_store.contig.dset_size = (hsize_t)io_info->dset->shared->layout.u.chunk.size; + coll_io_info.store = &ctg_store; + + /* Perform I/O */ + if (H5D__final_collective_io(&coll_io_info, type_info, mpi_buf_count, file_type, mem_type) < 0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "couldn't finish MPI I/O") + +done: + /* Free the MPI buf and file types, if they were derived */ + if (mem_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&mem_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + if (file_type_is_derived && MPI_SUCCESS != (mpi_code = MPI_Type_free(&file_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_collective_filtered_chunk_common_io() */ + +/*------------------------------------------------------------------------- + * Function: H5D__mpio_collective_filtered_chunk_read + * + * Purpose: This routine coordinates a collective read across all ranks + * of the chunks they have selected. Each rank will then go + * and + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_collective_filtered_chunk_read(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, const H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, int mpi_rank, int mpi_size) +{ + H5D_fill_buf_info_t fb_info; + H5D_chunk_info_t * chunk_info = NULL; + H5D_io_info_t coll_io_info; + H5Z_EDC_t err_detect; /* Error detection info */ + H5Z_cb_t filter_cb; /* I/O filter callback function */ + hsize_t file_chunk_size = 0; + hsize_t iter_nelmts; /* Number of points to iterate over for the chunk IO operation */ + hbool_t should_fill = FALSE; + hbool_t fb_info_init = FALSE; + hbool_t index_empty = FALSE; + size_t i; + H5S_t * fill_space = NULL; + void * base_read_buf = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(chunk_list || 0 == chunk_list_num_entries); + HDassert(io_info); + HDassert(type_info); + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Filtered collective chunk read"); +#else + (void)mpi_rank; +#endif + + /* Initialize temporary I/O info */ + coll_io_info = *io_info; + coll_io_info.u.rbuf = NULL; + + if (chunk_list_num_entries) { + /* Retrieve filter settings from API context */ + if (H5CX_get_err_detect(&err_detect) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get error detection info") + if (H5CX_get_filter_cb(&filter_cb) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get I/O filter callback function") + + /* Set size of full chunks in dataset */ + file_chunk_size = io_info->dset->shared->layout.u.chunk.size; + + /* Determine if fill values should be "read" for unallocated chunks */ + should_fill = (io_info->dset->shared->dcpl_cache.fill.fill_time == H5D_FILL_TIME_ALLOC) || + ((io_info->dset->shared->dcpl_cache.fill.fill_time == H5D_FILL_TIME_IFSET) && + io_info->dset->shared->dcpl_cache.fill.fill_defined); + } + + /* + * Allocate memory buffers for all chunks being read. Chunk data buffers are of + * the largest size between the chunk's current filtered size and the chunk's true + * size, as calculated by the number of elements in the chunk's file space extent + * multiplied by the datatype size. This tries to ensure that: + * + * * If we're reading the chunk and the filter normally reduces the chunk size, + * the unfiltering operation won't need to grow the buffer. + * * If we're reading the chunk and the filter normally grows the chunk size, + * we make sure to read into a buffer of size equal to the filtered chunk's + * size; reading into a (smaller) buffer of size equal to the unfiltered + * chunk size would of course be bad. + */ + for (i = 0; i < chunk_list_num_entries; i++) { + HDassert(chunk_list[i].need_read); + + chunk_list[i].chunk_buf_size = MAX(chunk_list[i].chunk_current.length, file_chunk_size); + + if (NULL == (chunk_list[i].buf = H5MM_malloc(chunk_list[i].chunk_buf_size))) { + /* Push an error, but participate in collective read */ + HDONE_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk data buffer") + break; + } + + /* + * Check if chunk is currently allocated. If not, don't try to + * read it from the file. Instead, just fill the chunk buffer + * with the fill value if necessary. + */ + if (H5F_addr_defined(chunk_list[i].chunk_current.offset)) { + /* Set first read buffer */ + if (!base_read_buf) + base_read_buf = chunk_list[i].buf; + + /* Set chunk's new length for eventual filter pipeline calls */ + if (chunk_list[i].skip_filter_pline) + chunk_list[i].chunk_new.length = file_chunk_size; + else + chunk_list[i].chunk_new.length = chunk_list[i].chunk_current.length; + } + else { + chunk_list[i].need_read = FALSE; + + /* Set chunk's new length for eventual filter pipeline calls */ + chunk_list[i].chunk_new.length = file_chunk_size; + + if (should_fill) { + /* Initialize fill value buffer if not already initialized */ + if (!fb_info_init) { + hsize_t chunk_dims[H5S_MAX_RANK]; + + HDassert(io_info->dset->shared->ndims == io_info->dset->shared->layout.u.chunk.ndims - 1); + for (size_t j = 0; j < io_info->dset->shared->layout.u.chunk.ndims - 1; j++) + chunk_dims[j] = (hsize_t)io_info->dset->shared->layout.u.chunk.dim[j]; + + /* Get a dataspace for filling chunk memory buffers */ + if (NULL == (fill_space = H5S_create_simple( + io_info->dset->shared->layout.u.chunk.ndims - 1, chunk_dims, NULL))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to create chunk fill dataspace") + + /* Initialize fill value buffer */ + if (H5D__fill_init(&fb_info, NULL, (H5MM_allocate_t)H5D__chunk_mem_alloc, + (void *)&io_info->dset->shared->dcpl_cache.pline, + (H5MM_free_t)H5D__chunk_mem_free, + (void *)&io_info->dset->shared->dcpl_cache.pline, + &io_info->dset->shared->dcpl_cache.fill, io_info->dset->shared->type, + io_info->dset->shared->type_id, 0, file_chunk_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize fill value buffer") + + fb_info_init = TRUE; + } + + /* Write fill value to memory buffer */ + HDassert(fb_info.fill_buf); + if (H5D__fill(fb_info.fill_buf, io_info->dset->shared->type, chunk_list[i].buf, + type_info->mem_type, fill_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "couldn't fill chunk buffer with fill value") + } + } + } + + /* + * If dataset is incrementally allocated and hasn't been written to + * yet, the chunk index should be empty. In this case, a collective + * read of chunks is essentially a no-op, so avoid it here. + */ + index_empty = FALSE; + if (io_info->dset->shared->dcpl_cache.fill.alloc_time == H5D_ALLOC_TIME_INCR) + if (H5D__chunk_index_empty(io_info->dset, &index_empty) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "couldn't determine if chunk index is empty") + + if (!index_empty) { + /* + * Override the read buffer to point to the address of + * the first chunk data buffer being read into + */ + if (base_read_buf) + coll_io_info.u.rbuf = base_read_buf; + + /* Perform collective chunk read */ + if (H5D__mpio_collective_filtered_chunk_common_io(chunk_list, chunk_list_num_entries, &coll_io_info, + type_info, mpi_size) < 0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "couldn't finish collective filtered chunk read") + } + + /* + * Iterate through all the read chunks, unfiltering them and scattering their + * data out to the application's read buffer. + */ + for (i = 0; i < chunk_list_num_entries; i++) { + chunk_info = chunk_list[i].chunk_info; + + /* Unfilter the chunk, unless we didn't read it from the file */ + if (chunk_list[i].need_read && !chunk_list[i].skip_filter_pline) { + if (H5Z_pipeline(&io_info->dset->shared->dcpl_cache.pline, H5Z_FLAG_REVERSE, + &(chunk_list[i].index_info.filter_mask), err_detect, filter_cb, + (size_t *)&chunk_list[i].chunk_new.length, &chunk_list[i].chunk_buf_size, + &chunk_list[i].buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFILTER, FAIL, "couldn't unfilter chunk for modifying") + } + + /* Scatter the chunk data to the read buffer */ + iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->fspace); + + if (H5D_select_io_mem(io_info->u.rbuf, chunk_info->mspace, chunk_list[i].buf, chunk_info->fspace, + type_info->src_type_size, (size_t)iter_nelmts) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't copy chunk data to read buffer") + } + +done: + /* Free all resources used by entries in the chunk list */ + for (i = 0; i < chunk_list_num_entries; i++) { + if (chunk_list[i].buf) { + H5MM_free(chunk_list[i].buf); + chunk_list[i].buf = NULL; + } + } + + /* Release the fill buffer info, if it's been initialized */ + if (fb_info_init && H5D__fill_term(&fb_info) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release fill buffer info") + if (fill_space && (H5S_close(fill_space) < 0)) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close fill space") + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_collective_filtered_chunk_read() */ + +/*------------------------------------------------------------------------- + * Function: H5D__mpio_collective_filtered_chunk_update + * + * Purpose: When performing a parallel write on a chunked dataset with + * filters applied, all ranks must update their owned chunks + * with their own modification data and data from other ranks. + * This routine is responsible for coordinating that process. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_collective_filtered_chunk_update(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, + H5D_filtered_collective_io_info_t *chunk_hash_table, + unsigned char **chunk_msg_bufs, int chunk_msg_bufs_len, + const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, + int mpi_rank, int mpi_size) +{ + H5D_fill_buf_info_t fb_info; + H5D_chunk_info_t * chunk_info = NULL; + H5S_sel_iter_t * sel_iter = NULL; /* Dataspace selection iterator for H5D__scatter_mem */ + H5D_io_info_t coll_io_info; + H5Z_EDC_t err_detect; /* Error detection info */ + H5Z_cb_t filter_cb; /* I/O filter callback function */ + hsize_t file_chunk_size = 0; + hsize_t iter_nelmts; /* Number of points to iterate over for the chunk IO operation */ + hbool_t should_fill = FALSE; + hbool_t fb_info_init = FALSE; + hbool_t sel_iter_init = FALSE; + hbool_t index_empty = FALSE; + size_t i; + H5S_t * dataspace = NULL; + H5S_t * fill_space = NULL; + void * base_read_buf = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(chunk_list || 0 == chunk_list_num_entries); + HDassert((chunk_msg_bufs && chunk_hash_table) || 0 == chunk_msg_bufs_len); + HDassert(io_info); + HDassert(type_info); + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Filtered collective chunk update"); +#endif + + if (chunk_list_num_entries) { + /* Retrieve filter settings from API context */ + if (H5CX_get_err_detect(&err_detect) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get error detection info") + if (H5CX_get_filter_cb(&filter_cb) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get I/O filter callback function") + + /* Set size of full chunks in dataset */ + file_chunk_size = io_info->dset->shared->layout.u.chunk.size; + + /* Determine if fill values should be written to chunks */ + should_fill = (io_info->dset->shared->dcpl_cache.fill.fill_time == H5D_FILL_TIME_ALLOC) || + ((io_info->dset->shared->dcpl_cache.fill.fill_time == H5D_FILL_TIME_IFSET) && + io_info->dset->shared->dcpl_cache.fill.fill_defined); + } + + /* + * Allocate memory buffers for all owned chunks. Chunk data buffers are of the + * largest size between the chunk's current filtered size and the chunk's true + * size, as calculated by the number of elements in the chunk's file space extent + * multiplied by the datatype size. This tries to ensure that: + * + * * If we're fully overwriting the chunk and the filter normally reduces the + * chunk size, we simply have the exact buffer size required to hold the + * unfiltered chunk data. + * * If we're fully overwriting the chunk and the filter normally grows the + * chunk size (e.g., fletcher32 filter), the final filtering operation + * (hopefully) won't need to grow the buffer. + * * If we're reading the chunk and the filter normally reduces the chunk size, + * the unfiltering operation won't need to grow the buffer. + * * If we're reading the chunk and the filter normally grows the chunk size, + * we make sure to read into a buffer of size equal to the filtered chunk's + * size; reading into a (smaller) buffer of size equal to the unfiltered + * chunk size would of course be bad. + */ + for (i = 0; i < chunk_list_num_entries; i++) { + HDassert(mpi_rank == chunk_list[i].new_owner); + + chunk_list[i].chunk_buf_size = MAX(chunk_list[i].chunk_current.length, file_chunk_size); + + /* + * If this chunk hasn't been allocated yet and we aren't writing + * out fill values to it, make sure to 0-fill its memory buffer + * so we don't use uninitialized memory. + */ + if (!H5F_addr_defined(chunk_list[i].chunk_current.offset) && !should_fill) + chunk_list[i].buf = H5MM_calloc(chunk_list[i].chunk_buf_size); + else + chunk_list[i].buf = H5MM_malloc(chunk_list[i].chunk_buf_size); + + if (NULL == chunk_list[i].buf) { + /* Push an error, but participate in collective read */ + HDONE_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk data buffer") + break; + } + + /* Set chunk's new length for eventual filter pipeline calls */ + if (chunk_list[i].need_read) { + /* + * Check if chunk is currently allocated. If not, don't try to + * read it from the file. Instead, just fill the chunk buffer + * with the fill value if fill values are to be written. + */ + if (H5F_addr_defined(chunk_list[i].chunk_current.offset)) { + /* Set first read buffer */ + if (!base_read_buf) + base_read_buf = chunk_list[i].buf; + + /* Set chunk's new length for eventual filter pipeline calls */ + if (chunk_list[i].skip_filter_pline) + chunk_list[i].chunk_new.length = file_chunk_size; + else + chunk_list[i].chunk_new.length = chunk_list[i].chunk_current.length; + } + else { + chunk_list[i].need_read = FALSE; + + /* Set chunk's new length for eventual filter pipeline calls */ + chunk_list[i].chunk_new.length = file_chunk_size; + + if (should_fill) { + /* Initialize fill value buffer if not already initialized */ + if (!fb_info_init) { + hsize_t chunk_dims[H5S_MAX_RANK]; + + HDassert(io_info->dset->shared->ndims == + io_info->dset->shared->layout.u.chunk.ndims - 1); + for (size_t j = 0; j < io_info->dset->shared->layout.u.chunk.ndims - 1; j++) + chunk_dims[j] = (hsize_t)io_info->dset->shared->layout.u.chunk.dim[j]; + + /* Get a dataspace for filling chunk memory buffers */ + if (NULL == (fill_space = H5S_create_simple( + io_info->dset->shared->layout.u.chunk.ndims - 1, chunk_dims, NULL))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to create chunk fill dataspace") + + /* Initialize fill value buffer */ + if (H5D__fill_init(&fb_info, NULL, (H5MM_allocate_t)H5D__chunk_mem_alloc, + (void *)&io_info->dset->shared->dcpl_cache.pline, + (H5MM_free_t)H5D__chunk_mem_free, + (void *)&io_info->dset->shared->dcpl_cache.pline, + &io_info->dset->shared->dcpl_cache.fill, + io_info->dset->shared->type, io_info->dset->shared->type_id, 0, + file_chunk_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize fill value buffer") + + fb_info_init = TRUE; + } + + /* Write fill value to memory buffer */ + HDassert(fb_info.fill_buf); + if (H5D__fill(fb_info.fill_buf, io_info->dset->shared->type, chunk_list[i].buf, + type_info->mem_type, fill_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "couldn't fill chunk buffer with fill value") + } + } + } + else + chunk_list[i].chunk_new.length = file_chunk_size; + } + + /* + * If dataset is incrementally allocated and hasn't been written to + * yet, the chunk index should be empty. In this case, a collective + * read of chunks is essentially a no-op, so avoid it here. + */ + index_empty = FALSE; + if (io_info->dset->shared->dcpl_cache.fill.alloc_time == H5D_ALLOC_TIME_INCR) + if (H5D__chunk_index_empty(io_info->dset, &index_empty) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "couldn't determine if chunk index is empty") + + if (!index_empty) { + /* + * Setup for I/O operation + */ + + /* Initialize temporary I/O info */ + coll_io_info = *io_info; + coll_io_info.op_type = H5D_IO_OP_READ; + + /* Override the read buffer to point to the address of the first + * chunk data buffer being read into + */ + if (base_read_buf) + coll_io_info.u.rbuf = base_read_buf; + + /* Read all chunks that need to be read from the file */ + if (H5D__mpio_collective_filtered_chunk_common_io(chunk_list, chunk_list_num_entries, &coll_io_info, + type_info, mpi_size) < 0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "couldn't finish collective filtered chunk read") + } + + /* + * Now that all owned chunks have been read, update the chunks + * with modification data from the owning rank and other ranks. + */ + + /* Process all chunks with data from the owning rank first */ + for (i = 0; i < chunk_list_num_entries; i++) { + HDassert(mpi_rank == chunk_list[i].new_owner); + + chunk_info = chunk_list[i].chunk_info; + + /* + * If this chunk wasn't being fully overwritten, we read it from + * the file, so we need to unfilter it + */ + if (chunk_list[i].need_read && !chunk_list[i].skip_filter_pline) { + if (H5Z_pipeline(&io_info->dset->shared->dcpl_cache.pline, H5Z_FLAG_REVERSE, + &(chunk_list[i].index_info.filter_mask), err_detect, filter_cb, + (size_t *)&chunk_list[i].chunk_new.length, &chunk_list[i].chunk_buf_size, + &chunk_list[i].buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFILTER, FAIL, "couldn't unfilter chunk for modifying") + } + + iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->mspace); + + if (H5D_select_io_mem(chunk_list[i].buf, chunk_info->fspace, io_info->u.wbuf, chunk_info->mspace, + type_info->dst_type_size, (size_t)iter_nelmts) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't copy chunk data to write buffer") + } + + /* Allocate iterator for memory selection */ + if (NULL == (sel_iter = H5FL_MALLOC(H5S_sel_iter_t))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate memory iterator") + + /* Now process all received chunk message buffers */ + for (i = 0; i < (size_t)chunk_msg_bufs_len; i++) { + H5D_filtered_collective_io_info_t *chunk_entry = NULL; + const unsigned char * msg_ptr = chunk_msg_bufs[i]; + hsize_t chunk_idx; + + if (msg_ptr) { + /* Retrieve the chunk's index value */ + HDmemcpy(&chunk_idx, msg_ptr, sizeof(hsize_t)); + msg_ptr += sizeof(hsize_t); + + /* Find the chunk entry according to its chunk index */ + HASH_FIND(hh, chunk_hash_table, &chunk_idx, sizeof(hsize_t), chunk_entry); + HDassert(chunk_entry); + HDassert(mpi_rank == chunk_entry->new_owner); + + /* + * Only process the chunk if its data buffer is allocated. + * In the case of multi-chunk I/O, we're only working on + * a chunk at a time, so we need to skip over messages + * that aren't for the chunk we're currently working on. + */ + if (!chunk_entry->buf) + continue; + else { + /* Decode the chunk file dataspace from the message */ + if (NULL == (dataspace = H5S_decode(&msg_ptr))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDECODE, FAIL, "unable to decode dataspace") + + if (H5S_select_iter_init(sel_iter, dataspace, type_info->dst_type_size, + H5S_SEL_ITER_SHARE_WITH_DATASPACE) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to initialize memory selection information") + sel_iter_init = TRUE; + + iter_nelmts = H5S_GET_SELECT_NPOINTS(dataspace); + + /* Update the chunk data with the received modification data */ + if (H5D__scatter_mem(msg_ptr, sel_iter, (size_t)iter_nelmts, chunk_entry->buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't scatter to write buffer") + + if (H5S_SELECT_ITER_RELEASE(sel_iter) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") + sel_iter_init = FALSE; + + if (dataspace) { + if (H5S_close(dataspace) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTFREE, FAIL, "can't close dataspace") + dataspace = NULL; + } + + H5MM_free(chunk_msg_bufs[i]); + chunk_msg_bufs[i] = NULL; + } + } + } + + /* Finally, filter all the chunks */ + for (i = 0; i < chunk_list_num_entries; i++) { + if (!chunk_list[i].skip_filter_pline) { + if (H5Z_pipeline(&io_info->dset->shared->dcpl_cache.pline, 0, + &(chunk_list[i].index_info.filter_mask), err_detect, filter_cb, + (size_t *)&chunk_list[i].chunk_new.length, &chunk_list[i].chunk_buf_size, + &chunk_list[i].buf) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "output pipeline failed") + } + +#if H5_SIZEOF_SIZE_T > 4 + /* Check for the chunk expanding too much to encode in a 32-bit value */ + if (chunk_list[i].chunk_new.length > ((size_t)0xffffffff)) + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk too large for 32-bit length") +#endif + } + +done: + if (sel_iter) { + if (sel_iter_init && H5S_SELECT_ITER_RELEASE(sel_iter) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") + sel_iter = H5FL_FREE(H5S_sel_iter_t, sel_iter); + } + if (dataspace && (H5S_close(dataspace) < 0)) + HDONE_ERROR(H5E_DATASPACE, H5E_CANTFREE, FAIL, "can't close dataspace") + if (fill_space && (H5S_close(fill_space) < 0)) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close fill space") + + /* Release the fill buffer info, if it's been initialized */ + if (fb_info_init && H5D__fill_term(&fb_info) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release fill buffer info") + + /* On failure, try to free all resources used by entries in the chunk list */ + if (ret_value < 0) { + for (i = 0; i < chunk_list_num_entries; i++) { + if (chunk_list[i].buf) { + H5MM_free(chunk_list[i].buf); + chunk_list[i].buf = NULL; + } + } + } + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_collective_filtered_chunk_update() */ + +/*------------------------------------------------------------------------- + * Function: H5D__mpio_collective_filtered_chunk_reallocate + * + * Purpose: When performing a parallel write on a chunked dataset with + * filters applied, all ranks must eventually get together and + * perform a collective reallocation of space in the file for + * all chunks that were modified on all ranks. This routine is + * responsible for coordinating that process. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_collective_filtered_chunk_reallocate(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, size_t *num_chunks_assigned_map, + H5D_io_info_t *io_info, H5D_chk_idx_info_t *idx_info, + int mpi_rank, int mpi_size) +{ + H5D_chunk_alloc_info_t *collective_list = NULL; + MPI_Datatype send_type; + MPI_Datatype recv_type; + hbool_t send_type_derived = FALSE; + hbool_t recv_type_derived = FALSE; + hbool_t need_sort = FALSE; + size_t collective_num_entries = 0; + size_t num_local_chunks_processed = 0; + size_t i; + void * gathered_array = NULL; + int * counts_disps_array = NULL; + int * counts_ptr = NULL; + int * displacements_ptr = NULL; + int mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(chunk_list || 0 == chunk_list_num_entries); + HDassert(io_info); + HDassert(idx_info); + HDassert(idx_info->storage->idx_type != H5D_CHUNK_IDX_NONE); + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Reallocation of chunk file space"); +#endif + + /* + * Make sure it's safe to cast this rank's number + * of chunks to be sent into an int for MPI + */ + H5_CHECK_OVERFLOW(chunk_list_num_entries, size_t, int); + + /* Create derived datatypes for the chunk file space info needed */ + if (H5D__mpio_get_chunk_alloc_info_types(&recv_type, &recv_type_derived, &send_type, &send_type_derived) < + 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "can't create derived datatypes for chunk file space info") + + /* + * Gather the new chunk sizes to all ranks for a collective reallocation + * of the chunks in the file. + */ + if (num_chunks_assigned_map) { + /* + * If a mapping between rank value -> number of assigned chunks has + * been provided (usually during linked-chunk I/O), we can use this + * to optimize MPI overhead a bit since MPI ranks won't need to + * first inform each other about how many chunks they're contributing. + */ + if (NULL == (counts_disps_array = H5MM_malloc(2 * (size_t)mpi_size * sizeof(*counts_disps_array)))) { + /* Push an error, but still participate in collective gather operation */ + HDONE_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "couldn't allocate receive counts and displacements array") + } + else { + /* Set the receive counts from the assigned chunks map */ + counts_ptr = counts_disps_array; + + for (i = 0; i < (size_t)mpi_size; i++) + H5_CHECKED_ASSIGN(counts_ptr[i], int, num_chunks_assigned_map[i], size_t); + + /* Set the displacements into the receive buffer for the gather operation */ + displacements_ptr = &counts_disps_array[mpi_size]; + + *displacements_ptr = 0; + for (i = 1; i < (size_t)mpi_size; i++) + displacements_ptr[i] = displacements_ptr[i - 1] + counts_ptr[i - 1]; + } + + /* Perform gather operation */ + if (H5_mpio_gatherv_alloc(chunk_list, (int)chunk_list_num_entries, send_type, counts_ptr, + displacements_ptr, recv_type, TRUE, 0, io_info->comm, mpi_rank, mpi_size, + &gathered_array, &collective_num_entries) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "can't gather chunk file space info to/from ranks") + } + else { + /* + * If no mapping between rank value -> number of assigned chunks has + * been provided (usually during multi-chunk I/O), all MPI ranks will + * need to first inform other ranks about how many chunks they're + * contributing before performing the actual gather operation. Use + * the 'simple' MPI_Allgatherv wrapper for this. + */ + if (H5_mpio_gatherv_alloc_simple(chunk_list, (int)chunk_list_num_entries, send_type, recv_type, TRUE, + 0, io_info->comm, mpi_rank, mpi_size, &gathered_array, + &collective_num_entries) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "can't gather chunk file space info to/from ranks") + } + + /* Collectively re-allocate the modified chunks (from each rank) in the file */ + collective_list = (H5D_chunk_alloc_info_t *)gathered_array; + for (i = 0, num_local_chunks_processed = 0; i < collective_num_entries; i++) { + H5D_chunk_alloc_info_t *coll_entry = &collective_list[i]; + hbool_t need_insert; + hbool_t update_local_chunk; + + if (H5D__chunk_file_alloc(idx_info, &coll_entry->chunk_current, &coll_entry->chunk_new, &need_insert, + NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate chunk") + + /* + * If we just re-allocated a chunk that is local to this + * rank, make sure to update the chunk entry in the local + * chunk list + */ + update_local_chunk = + (num_local_chunks_processed < chunk_list_num_entries) && + (coll_entry->chunk_idx == chunk_list[num_local_chunks_processed].index_info.chunk_idx); + + if (update_local_chunk) { + H5D_filtered_collective_io_info_t *local_chunk; + + local_chunk = &chunk_list[num_local_chunks_processed]; + + /* Sanity check that this chunk is actually local */ + HDassert(mpi_rank == local_chunk->orig_owner); + HDassert(mpi_rank == local_chunk->new_owner); + + local_chunk->chunk_new = coll_entry->chunk_new; + local_chunk->index_info.need_insert = need_insert; + + /* + * Since chunk reallocation can move chunks around, check if + * the local chunk list is still in ascending offset of order + * in the file + */ + if (num_local_chunks_processed) { + haddr_t curr_chunk_offset = local_chunk->chunk_new.offset; + haddr_t prev_chunk_offset = chunk_list[num_local_chunks_processed - 1].chunk_new.offset; + + HDassert(H5F_addr_defined(prev_chunk_offset) && H5F_addr_defined(curr_chunk_offset)); + if (curr_chunk_offset < prev_chunk_offset) + need_sort = TRUE; + } + + num_local_chunks_processed++; + } + } + + HDassert(chunk_list_num_entries == num_local_chunks_processed); + + /* + * Ensure this rank's local chunk list is sorted in + * ascending order of offset in the file + */ + if (need_sort) + HDqsort(chunk_list, chunk_list_num_entries, sizeof(H5D_filtered_collective_io_info_t), + H5D__cmp_filtered_collective_io_info_entry); + +done: + H5MM_free(gathered_array); + H5MM_free(counts_disps_array); + + if (send_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&send_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } + if (recv_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&recv_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D__mpio_collective_filtered_chunk_reallocate() */ + +/*------------------------------------------------------------------------- + * Function: H5D__mpio_collective_filtered_chunk_reinsert + * + * Purpose: When performing a parallel write on a chunked dataset with + * filters applied, all ranks must eventually get together and + * perform a collective reinsertion into the dataset's chunk + * index of chunks that were modified. This routine is + * responsible for coordinating that process. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_collective_filtered_chunk_reinsert(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, size_t *num_chunks_assigned_map, + H5D_io_info_t *io_info, H5D_chk_idx_info_t *idx_info, + int mpi_rank, int mpi_size) +{ + H5D_chunk_ud_t chunk_ud; + MPI_Datatype send_type; + MPI_Datatype recv_type; + hbool_t send_type_derived = FALSE; + hbool_t recv_type_derived = FALSE; + hsize_t scaled_coords[H5O_LAYOUT_NDIMS]; + size_t collective_num_entries = 0; + size_t i; + void * gathered_array = NULL; + int * counts_disps_array = NULL; + int * counts_ptr = NULL; + int * displacements_ptr = NULL; + int mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(chunk_list || 0 == chunk_list_num_entries); + HDassert(io_info); + HDassert(idx_info); + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TRACE_ENTER(mpi_rank); + H5D_MPIO_TIME_START(mpi_rank, "Reinsertion of modified chunks into chunk index"); +#endif + + /* Only re-insert chunks if index has an insert method */ + if (!idx_info->storage->ops->insert) + HGOTO_DONE(SUCCEED); + + /* + * Make sure it's safe to cast this rank's number + * of chunks to be sent into an int for MPI + */ + H5_CHECK_OVERFLOW(chunk_list_num_entries, size_t, int); + + /* Create derived datatypes for the chunk re-insertion info needed */ + if (H5D__mpio_get_chunk_insert_info_types(&recv_type, &recv_type_derived, &send_type, + &send_type_derived) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, + "can't create derived datatypes for chunk re-insertion info") + + /* + * Gather information to all ranks for a collective re-insertion + * of the modified chunks into the chunk index + */ + if (num_chunks_assigned_map) { + /* + * If a mapping between rank value -> number of assigned chunks has + * been provided (usually during linked-chunk I/O), we can use this + * to optimize MPI overhead a bit since MPI ranks won't need to + * first inform each other about how many chunks they're contributing. + */ + if (NULL == (counts_disps_array = H5MM_malloc(2 * (size_t)mpi_size * sizeof(*counts_disps_array)))) { + /* Push an error, but still participate in collective gather operation */ + HDONE_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "couldn't allocate receive counts and displacements array") + } + else { + /* Set the receive counts from the assigned chunks map */ + counts_ptr = counts_disps_array; + + for (i = 0; i < (size_t)mpi_size; i++) + H5_CHECKED_ASSIGN(counts_ptr[i], int, num_chunks_assigned_map[i], size_t); + + /* Set the displacements into the receive buffer for the gather operation */ + displacements_ptr = &counts_disps_array[mpi_size]; + + *displacements_ptr = 0; + for (i = 1; i < (size_t)mpi_size; i++) + displacements_ptr[i] = displacements_ptr[i - 1] + counts_ptr[i - 1]; + } + + /* Perform gather operation */ + if (H5_mpio_gatherv_alloc(chunk_list, (int)chunk_list_num_entries, send_type, counts_ptr, + displacements_ptr, recv_type, TRUE, 0, io_info->comm, mpi_rank, mpi_size, + &gathered_array, &collective_num_entries) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, + "can't gather chunk index re-insertion info to/from ranks") + } + else { + /* + * If no mapping between rank value -> number of assigned chunks has + * been provided (usually during multi-chunk I/O), all MPI ranks will + * need to first inform other ranks about how many chunks they're + * contributing before performing the actual gather operation. Use + * the 'simple' MPI_Allgatherv wrapper for this. + */ + if (H5_mpio_gatherv_alloc_simple(chunk_list, (int)chunk_list_num_entries, send_type, recv_type, TRUE, + 0, io_info->comm, mpi_rank, mpi_size, &gathered_array, + &collective_num_entries) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, + "can't gather chunk index re-insertion info to/from ranks") + } + + /* Initialize static chunk udata fields from chunk index info */ + H5D_MPIO_INIT_CHUNK_UD_INFO(chunk_ud, idx_info); + + for (i = 0; i < collective_num_entries; i++) { + H5D_chunk_insert_info_t *coll_entry = &((H5D_chunk_insert_info_t *)gathered_array)[i]; + + /* + * We only need to reinsert this chunk if we had to actually + * allocate or reallocate space in the file for it + */ + if (!coll_entry->index_info.need_insert) + continue; + + chunk_ud.chunk_block = coll_entry->chunk_block; + chunk_ud.chunk_idx = coll_entry->index_info.chunk_idx; + chunk_ud.filter_mask = coll_entry->index_info.filter_mask; + chunk_ud.common.scaled = scaled_coords; + + /* Calculate scaled coordinates for the chunk */ + if (idx_info->layout->idx_type == H5D_CHUNK_IDX_EARRAY && idx_info->layout->u.earray.unlim_dim > 0) { + /* + * Extensible arrays where the unlimited dimension is not + * the slowest-changing dimension "swizzle" the coordinates + * to move the unlimited dimension value to offset 0. Therefore, + * we use the "swizzled" down chunks to calculate the "swizzled" + * scaled coordinates and then we undo the "swizzle" operation. + * + * TODO: In the future, this is something that should be handled + * by the particular chunk index rather than manually + * here. Likely, the chunk index ops should get a new + * callback that accepts a chunk index and provides the + * caller with the scaled coordinates for that chunk. + */ + H5VM_array_calc_pre(chunk_ud.chunk_idx, io_info->dset->shared->ndims, + idx_info->layout->u.earray.swizzled_down_chunks, scaled_coords); + + H5VM_unswizzle_coords(hsize_t, scaled_coords, idx_info->layout->u.earray.unlim_dim); + } + else { + H5VM_array_calc_pre(chunk_ud.chunk_idx, io_info->dset->shared->ndims, + io_info->dset->shared->layout.u.chunk.down_chunks, scaled_coords); + } + + scaled_coords[io_info->dset->shared->ndims] = 0; + +#ifndef NDEBUG + /* + * If a matching local chunk entry is found, the + * `chunk_info` structure (which contains the chunk's + * pre-computed scaled coordinates) will be valid + * for this rank. Compare those coordinates against + * the calculated coordinates above to make sure + * they match. + */ + for (size_t dbg_idx = 0; dbg_idx < chunk_list_num_entries; dbg_idx++) { + if (coll_entry->index_info.chunk_idx == chunk_list[dbg_idx].index_info.chunk_idx) { + hbool_t coords_match = !HDmemcmp(scaled_coords, chunk_list[dbg_idx].chunk_info->scaled, + io_info->dset->shared->ndims * sizeof(hsize_t)); + + HDassert(coords_match && "Calculated scaled coordinates for chunk didn't match " + "chunk's actual scaled coordinates!"); + break; + } + } +#endif + + if ((idx_info->storage->ops->insert)(idx_info, &chunk_ud, io_info->dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk address into index") + } + +done: + H5MM_free(gathered_array); + H5MM_free(counts_disps_array); + + if (send_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&send_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } + if (recv_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&recv_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } + +#ifdef H5Dmpio_DEBUG + H5D_MPIO_TIME_STOP(mpi_rank); + H5D_MPIO_TRACE_EXIT(mpi_rank); +#endif + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_collective_filtered_chunk_reinsert() */ + +/*------------------------------------------------------------------------- + * Function: H5D__mpio_get_chunk_redistribute_info_types * - * Return: Non-negative on success/Negative on failure + * Purpose: Constructs MPI derived datatypes for communicating the + * info from a H5D_filtered_collective_io_info_t structure + * that is necessary for redistributing shared chunks during a + * collective write of filtered chunks. + * + * The datatype returned through `contig_type` has an extent + * equal to the size of an H5D_chunk_redistribute_info_t + * structure and is suitable for communicating that structure + * type. * - * Programmer: Jordan Henderson - * Monday, May 1, 2017 + * The datatype returned through `resized_type` has an extent + * equal to the size of an H5D_filtered_collective_io_info_t + * structure. This makes it suitable for sending an array of + * those structures, while extracting out just the info + * necessary for the chunk redistribution operation during + * communication. + * + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ static herr_t -H5D__chunk_redistribute_shared_chunks(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - const H5D_chunk_map_t * fm, - H5D_filtered_collective_io_info_t *local_chunk_array, - size_t * local_chunk_array_num_entries) +H5D__mpio_get_chunk_redistribute_info_types(MPI_Datatype *contig_type, hbool_t *contig_type_derived, + MPI_Datatype *resized_type, hbool_t *resized_type_derived) { - H5D_filtered_collective_io_info_t *shared_chunks_info_array = - NULL; /* The list of all chunks selected in the operation by all processes */ - H5S_sel_iter_t *mem_iter = NULL; /* Memory iterator for H5D__gather_mem */ - unsigned char **mod_data = - NULL; /* Array of chunk modification data buffers sent by a process to new chunk owners */ - MPI_Request *send_requests = NULL; /* Array of MPI_Isend chunk modification data send requests */ - MPI_Status * send_statuses = NULL; /* Array of MPI_Isend chunk modification send statuses */ - hbool_t mem_iter_init = FALSE; - size_t shared_chunks_info_array_num_entries = 0; - size_t num_send_requests = 0; - size_t * num_assigned_chunks_array = NULL; - size_t i, last_assigned_idx; - int * send_counts = NULL; - int * send_displacements = NULL; - int scatter_recvcount_int; - int mpi_rank, mpi_size, mpi_code; + MPI_Datatype struct_type = MPI_DATATYPE_NULL; + hbool_t struct_type_derived = FALSE; + MPI_Datatype chunk_block_type = MPI_DATATYPE_NULL; + hbool_t chunk_block_type_derived = FALSE; + MPI_Datatype types[5]; + MPI_Aint displacements[5]; + int block_lengths[5]; + int field_count; + int mpi_code; herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC - HDassert(io_info); - HDassert(type_info); - HDassert(fm); - HDassert(local_chunk_array_num_entries); + HDassert(contig_type); + HDassert(contig_type_derived); + HDassert(resized_type); + HDassert(resized_type_derived); - if ((mpi_rank = H5F_mpi_get_rank(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi rank") - if ((mpi_size = H5F_mpi_get_size(io_info->dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_IO, H5E_MPI, FAIL, "unable to obtain mpi size") + *contig_type_derived = FALSE; + *resized_type_derived = FALSE; - /* Set to latest format for encoding dataspace */ - H5CX_set_libver_bounds(NULL); + /* Create struct type for the inner H5F_block_t structure */ + if (H5F_mpi_get_file_block_type(FALSE, &chunk_block_type, &chunk_block_type_derived) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't create derived type for chunk file description") - if (*local_chunk_array_num_entries) - if (NULL == (send_requests = - (MPI_Request *)H5MM_malloc(*local_chunk_array_num_entries * sizeof(MPI_Request)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate send requests buffer") + field_count = 5; + HDassert(field_count == (sizeof(types) / sizeof(MPI_Datatype))); - if (NULL == (mem_iter = (H5S_sel_iter_t *)H5MM_malloc(sizeof(H5S_sel_iter_t)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate memory iterator") + /* + * Create structure type to pack chunk H5F_block_t structure + * next to chunk_idx, orig_owner, new_owner and num_writers + * fields + */ + block_lengths[0] = 1; + block_lengths[1] = 1; + block_lengths[2] = 1; + block_lengths[3] = 1; + block_lengths[4] = 1; + displacements[0] = offsetof(H5D_chunk_redistribute_info_t, chunk_block); + displacements[1] = offsetof(H5D_chunk_redistribute_info_t, chunk_idx); + displacements[2] = offsetof(H5D_chunk_redistribute_info_t, orig_owner); + displacements[3] = offsetof(H5D_chunk_redistribute_info_t, new_owner); + displacements[4] = offsetof(H5D_chunk_redistribute_info_t, num_writers); + types[0] = chunk_block_type; + types[1] = HSIZE_AS_MPI_TYPE; + types[2] = MPI_INT; + types[3] = MPI_INT; + types[4] = MPI_INT; + if (MPI_SUCCESS != + (mpi_code = MPI_Type_create_struct(field_count, block_lengths, displacements, types, contig_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) + *contig_type_derived = TRUE; + + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(contig_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - /* Gather every rank's list of chunks to rank 0 to allow it to perform the redistribution operation. After - * this call, the gathered list will initially be sorted in increasing order of chunk offset in the file. + /* Create struct type to extract the chunk_current, chunk_idx, orig_owner, + * new_owner and num_writers fields from a H5D_filtered_collective_io_info_t + * structure */ - if (H5D__mpio_array_gatherv(local_chunk_array, *local_chunk_array_num_entries, - sizeof(H5D_filtered_collective_io_info_t), (void **)&shared_chunks_info_array, - &shared_chunks_info_array_num_entries, false, 0, io_info->comm, - H5D__cmp_filtered_collective_io_info_entry) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGATHER, FAIL, "couldn't gather array") + block_lengths[0] = 1; + block_lengths[1] = 1; + block_lengths[2] = 1; + block_lengths[3] = 1; + block_lengths[4] = 1; + displacements[0] = offsetof(H5D_filtered_collective_io_info_t, chunk_current); + displacements[1] = offsetof(H5D_filtered_collective_io_info_t, index_info.chunk_idx); + displacements[2] = offsetof(H5D_filtered_collective_io_info_t, orig_owner); + displacements[3] = offsetof(H5D_filtered_collective_io_info_t, new_owner); + displacements[4] = offsetof(H5D_filtered_collective_io_info_t, num_writers); + types[0] = chunk_block_type; + types[1] = HSIZE_AS_MPI_TYPE; + types[2] = MPI_INT; + types[3] = MPI_INT; + types[4] = MPI_INT; + if (MPI_SUCCESS != + (mpi_code = MPI_Type_create_struct(field_count, block_lengths, displacements, types, &struct_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) + struct_type_derived = TRUE; - /* Rank 0 redistributes any shared chunks to new owners as necessary */ - if (mpi_rank == 0) { - if (NULL == (send_counts = (int *)H5MM_calloc((size_t)mpi_size * sizeof(int)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate send counts buffer") + if (MPI_SUCCESS != (mpi_code = MPI_Type_create_resized( + struct_type, 0, sizeof(H5D_filtered_collective_io_info_t), resized_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_resized failed", mpi_code) + *resized_type_derived = TRUE; - if (NULL == (send_displacements = (int *)H5MM_malloc((size_t)mpi_size * sizeof(int)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate send displacements buffer") + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(resized_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - if (NULL == (num_assigned_chunks_array = (size_t *)H5MM_calloc((size_t)mpi_size * sizeof(size_t)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, - "unable to allocate number of assigned chunks array") +done: + if (struct_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&struct_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } + if (chunk_block_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&chunk_block_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } - for (i = 0; i < shared_chunks_info_array_num_entries;) { - H5D_filtered_collective_io_info_t *chunk_entry; - haddr_t last_seen_addr = shared_chunks_info_array[i].chunk_states.chunk_current.offset; - size_t set_begin_index = i; - size_t num_writers = 0; - int new_chunk_owner = shared_chunks_info_array[i].owners.original_owner; + if (ret_value < 0) { + if (*resized_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(resized_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + *resized_type_derived = FALSE; + } + if (*contig_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(contig_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + *contig_type_derived = FALSE; + } + } - /* Process each set of duplicate entries caused by another process writing to the same chunk */ - do { - chunk_entry = &shared_chunks_info_array[i]; + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_get_chunk_redistribute_info_types() */ - send_counts[chunk_entry->owners.original_owner] += (int)sizeof(*chunk_entry); +/*------------------------------------------------------------------------- + * Function: H5D__mpio_get_chunk_alloc_info_types + * + * Purpose: Constructs MPI derived datatypes for communicating the info + * from a H5D_filtered_collective_io_info_t structure that is + * necessary for re-allocating file space during a collective + * write of filtered chunks. + * + * The datatype returned through `contig_type` has an extent + * equal to the size of an H5D_chunk_alloc_info_t structure + * and is suitable for communicating that structure type. + * + * The datatype returned through `resized_type` has an extent + * equal to the size of an H5D_filtered_collective_io_info_t + * structure. This makes it suitable for sending an array of + * those structures, while extracting out just the info + * necessary for the chunk file space reallocation operation + * during communication. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_get_chunk_alloc_info_types(MPI_Datatype *contig_type, hbool_t *contig_type_derived, + MPI_Datatype *resized_type, hbool_t *resized_type_derived) +{ + MPI_Datatype struct_type = MPI_DATATYPE_NULL; + hbool_t struct_type_derived = FALSE; + MPI_Datatype chunk_block_type = MPI_DATATYPE_NULL; + hbool_t chunk_block_type_derived = FALSE; + MPI_Datatype types[3]; + MPI_Aint displacements[3]; + int block_lengths[3]; + int field_count; + int mpi_code; + herr_t ret_value = SUCCEED; - /* The new owner of the chunk is determined by the process - * writing to the chunk which currently has the least amount - * of chunks assigned to it - */ - if (num_assigned_chunks_array[chunk_entry->owners.original_owner] < - num_assigned_chunks_array[new_chunk_owner]) - new_chunk_owner = chunk_entry->owners.original_owner; + FUNC_ENTER_STATIC - num_writers++; - } while (++i < shared_chunks_info_array_num_entries && - shared_chunks_info_array[i].chunk_states.chunk_current.offset == last_seen_addr); + HDassert(contig_type); + HDassert(contig_type_derived); + HDassert(resized_type); + HDassert(resized_type_derived); - /* Set all of the chunk entries' "new_owner" fields */ - for (; set_begin_index < i; set_begin_index++) { - shared_chunks_info_array[set_begin_index].owners.new_owner = new_chunk_owner; - shared_chunks_info_array[set_begin_index].num_writers = num_writers; - } /* end for */ + *contig_type_derived = FALSE; + *resized_type_derived = FALSE; - num_assigned_chunks_array[new_chunk_owner]++; - } /* end for */ + /* Create struct type for the inner H5F_block_t structure */ + if (H5F_mpi_get_file_block_type(FALSE, &chunk_block_type, &chunk_block_type_derived) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't create derived type for chunk file description") - /* Sort the new list in order of previous owner so that each original owner of a chunk - * entry gets that entry back, with the possibly newly-modified "new_owner" field - */ - if (shared_chunks_info_array_num_entries > 1) - HDqsort(shared_chunks_info_array, shared_chunks_info_array_num_entries, - sizeof(H5D_filtered_collective_io_info_t), - H5D__cmp_filtered_collective_io_info_entry_owner); - - send_displacements[0] = 0; - for (i = 1; i < (size_t)mpi_size; i++) - send_displacements[i] = send_displacements[i - 1] + send_counts[i - 1]; - } /* end if */ + field_count = 3; + HDassert(field_count == (sizeof(types) / sizeof(MPI_Datatype))); - /* Scatter the segments of the list back to each process */ - H5_CHECKED_ASSIGN(scatter_recvcount_int, int, - *local_chunk_array_num_entries * sizeof(H5D_filtered_collective_io_info_t), size_t); + /* + * Create structure type to pack both chunk H5F_block_t structures + * next to chunk_idx field + */ + block_lengths[0] = 1; + block_lengths[1] = 1; + block_lengths[2] = 1; + displacements[0] = offsetof(H5D_chunk_alloc_info_t, chunk_current); + displacements[1] = offsetof(H5D_chunk_alloc_info_t, chunk_new); + displacements[2] = offsetof(H5D_chunk_alloc_info_t, chunk_idx); + types[0] = chunk_block_type; + types[1] = chunk_block_type; + types[2] = HSIZE_AS_MPI_TYPE; if (MPI_SUCCESS != - (mpi_code = MPI_Scatterv(shared_chunks_info_array, send_counts, send_displacements, MPI_BYTE, - local_chunk_array, scatter_recvcount_int, MPI_BYTE, 0, io_info->comm))) - HMPI_GOTO_ERROR(FAIL, "unable to scatter shared chunks info buffer", mpi_code) + (mpi_code = MPI_Type_create_struct(field_count, block_lengths, displacements, types, contig_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) + *contig_type_derived = TRUE; - if (shared_chunks_info_array) { - H5MM_free(shared_chunks_info_array); - shared_chunks_info_array = NULL; - } /* end if */ + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(contig_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - /* Now that the chunks have been redistributed, each process must send its modification data - * to the new owners of any of the chunks it previously possessed. Accordingly, each process - * must also issue asynchronous receives for any messages it may receive for each of the - * chunks it is assigned, in order to avoid potential deadlocking issues. + /* + * Create struct type to extract the chunk_current, chunk_new and chunk_idx + * fields from a H5D_filtered_collective_io_info_t structure */ - if (*local_chunk_array_num_entries) - if (NULL == (mod_data = (unsigned char **)H5MM_malloc(*local_chunk_array_num_entries * - sizeof(unsigned char *)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate modification data buffer array") - - /* Perform all the sends on the chunks that this rank doesn't own */ - /* (Sends and recvs must be two separate loops, to avoid deadlock) */ - for (i = 0, last_assigned_idx = 0; i < *local_chunk_array_num_entries; i++) { - H5D_filtered_collective_io_info_t *chunk_entry = &local_chunk_array[i]; - - if (mpi_rank != chunk_entry->owners.new_owner) { - H5D_chunk_info_t *chunk_info = NULL; - unsigned char * mod_data_p = NULL; - hsize_t iter_nelmts; - size_t mod_data_size; - - /* Look up the chunk and get its file and memory dataspaces */ - if (NULL == (chunk_info = (H5D_chunk_info_t *)H5SL_search(fm->sel_chunks, &chunk_entry->index))) - HGOTO_ERROR(H5E_DATASPACE, H5E_NOTFOUND, FAIL, "can't locate chunk in skip list") + block_lengths[0] = 1; + block_lengths[1] = 1; + block_lengths[2] = 1; + displacements[0] = offsetof(H5D_filtered_collective_io_info_t, chunk_current); + displacements[1] = offsetof(H5D_filtered_collective_io_info_t, chunk_new); + displacements[2] = offsetof(H5D_filtered_collective_io_info_t, index_info.chunk_idx); + types[0] = chunk_block_type; + types[1] = chunk_block_type; + types[2] = HSIZE_AS_MPI_TYPE; + if (MPI_SUCCESS != + (mpi_code = MPI_Type_create_struct(field_count, block_lengths, displacements, types, &struct_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) + struct_type_derived = TRUE; - /* Determine size of serialized chunk file dataspace, plus the size of - * the data being written - */ - if (H5S_encode(chunk_info->fspace, &mod_data_p, &mod_data_size) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "unable to get encoded dataspace size") + if (MPI_SUCCESS != (mpi_code = MPI_Type_create_resized( + struct_type, 0, sizeof(H5D_filtered_collective_io_info_t), resized_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_resized failed", mpi_code) + *resized_type_derived = TRUE; - iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->mspace); + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(resized_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - H5_CHECK_OVERFLOW(iter_nelmts, hsize_t, size_t); - mod_data_size += (size_t)iter_nelmts * type_info->src_type_size; +done: + if (struct_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&struct_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } + if (chunk_block_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&chunk_block_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } - if (NULL == (mod_data[num_send_requests] = (unsigned char *)H5MM_malloc(mod_data_size))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, - "couldn't allocate chunk modification send buffer") + if (ret_value < 0) { + if (*resized_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(resized_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + *resized_type_derived = FALSE; + } + if (*contig_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(contig_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + *contig_type_derived = FALSE; + } + } - /* Serialize the chunk's file dataspace into the buffer */ - mod_data_p = mod_data[num_send_requests]; - if (H5S_encode(chunk_info->fspace, &mod_data_p, &mod_data_size) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "unable to encode dataspace") + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_get_chunk_alloc_info_types() */ - /* Initialize iterator for memory selection */ - if (H5S_select_iter_init(mem_iter, chunk_info->mspace, type_info->src_type_size, 0) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to initialize memory selection information") - mem_iter_init = TRUE; +/*------------------------------------------------------------------------- + * Function: H5D__mpio_get_chunk_insert_info_types + * + * Purpose: Constructs MPI derived datatypes for communicating the + * information necessary when reinserting chunks into a + * dataset's chunk index. This includes the chunk's new offset + * and size (H5F_block_t) and the inner `index_info` structure + * of a H5D_filtered_collective_io_info_t structure. + * + * The datatype returned through `contig_type` has an extent + * equal to the size of an H5D_chunk_insert_info_t structure + * and is suitable for communicating that structure type. + * + * The datatype returned through `resized_type` has an extent + * equal to the size of the encompassing + * H5D_filtered_collective_io_info_t structure. This makes it + * suitable for sending an array of + * H5D_filtered_collective_io_info_t structures, while + * extracting out just the information needed during + * communication. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__mpio_get_chunk_insert_info_types(MPI_Datatype *contig_type, hbool_t *contig_type_derived, + MPI_Datatype *resized_type, hbool_t *resized_type_derived) +{ + MPI_Datatype struct_type = MPI_DATATYPE_NULL; + hbool_t struct_type_derived = FALSE; + MPI_Datatype chunk_block_type = MPI_DATATYPE_NULL; + hbool_t chunk_block_type_derived = FALSE; + MPI_Aint contig_type_extent; + MPI_Datatype types[4]; + MPI_Aint displacements[4]; + int block_lengths[4]; + int field_count; + int mpi_code; + herr_t ret_value = SUCCEED; - /* Collect the modification data into the buffer */ - if (0 == H5D__gather_mem(io_info->u.wbuf, mem_iter, (size_t)iter_nelmts, mod_data_p)) - HGOTO_ERROR(H5E_IO, H5E_CANTGATHER, FAIL, "couldn't gather from write buffer") + FUNC_ENTER_STATIC - /* Send modification data to new owner */ - H5_CHECK_OVERFLOW(mod_data_size, size_t, int) - H5_CHECK_OVERFLOW(chunk_entry->index, hsize_t, int) - if (MPI_SUCCESS != - (mpi_code = MPI_Isend(mod_data[num_send_requests], (int)mod_data_size, MPI_BYTE, - chunk_entry->owners.new_owner, (int)chunk_entry->index, io_info->comm, - &send_requests[num_send_requests]))) - HMPI_GOTO_ERROR(FAIL, "MPI_Isend failed", mpi_code) + HDassert(contig_type); + HDassert(contig_type_derived); + HDassert(resized_type); + HDassert(resized_type_derived); - if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release memory selection iterator") - mem_iter_init = FALSE; + *contig_type_derived = FALSE; + *resized_type_derived = FALSE; - num_send_requests++; - } /* end if */ - } /* end for */ + /* Create struct type for an H5F_block_t structure */ + if (H5F_mpi_get_file_block_type(FALSE, &chunk_block_type, &chunk_block_type_derived) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't create derived type for chunk file description") - /* Perform all the recvs on the chunks this rank owns */ - for (i = 0, last_assigned_idx = 0; i < *local_chunk_array_num_entries; i++) { - H5D_filtered_collective_io_info_t *chunk_entry = &local_chunk_array[i]; + field_count = 4; + HDassert(field_count == (sizeof(types) / sizeof(MPI_Datatype))); - if (mpi_rank == chunk_entry->owners.new_owner) { - /* Allocate all necessary buffers for an asynchronous receive operation */ - if (chunk_entry->num_writers > 1) { - MPI_Message message; - MPI_Status status; - size_t j; + /* + * Create struct type to pack information into memory as follows: + * + * Chunk's new Offset/Size (H5F_block_t) -> + * Chunk Index Info (H5D_chunk_index_info_t) + */ + block_lengths[0] = 1; + block_lengths[1] = 1; + block_lengths[2] = 1; + block_lengths[3] = 1; + displacements[0] = offsetof(H5D_chunk_insert_info_t, chunk_block); + displacements[1] = offsetof(H5D_chunk_insert_info_t, index_info.chunk_idx); + displacements[2] = offsetof(H5D_chunk_insert_info_t, index_info.filter_mask); + displacements[3] = offsetof(H5D_chunk_insert_info_t, index_info.need_insert); + types[0] = chunk_block_type; + types[1] = HSIZE_AS_MPI_TYPE; + types[2] = MPI_UNSIGNED; + types[3] = MPI_C_BOOL; + if (MPI_SUCCESS != + (mpi_code = MPI_Type_create_struct(field_count, block_lengths, displacements, types, &struct_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) + struct_type_derived = TRUE; - chunk_entry->async_info.num_receive_requests = (int)chunk_entry->num_writers - 1; - if (NULL == (chunk_entry->async_info.receive_requests_array = (MPI_Request *)H5MM_malloc( - (size_t)chunk_entry->async_info.num_receive_requests * sizeof(MPI_Request)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate async requests array") + contig_type_extent = (MPI_Aint)(sizeof(H5F_block_t) + sizeof(H5D_chunk_index_info_t)); - if (NULL == - (chunk_entry->async_info.receive_buffer_array = (unsigned char **)H5MM_malloc( - (size_t)chunk_entry->async_info.num_receive_requests * sizeof(unsigned char *)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate async receive buffers") + if (MPI_SUCCESS != (mpi_code = MPI_Type_create_resized(struct_type, 0, contig_type_extent, contig_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_resized failed", mpi_code) + *contig_type_derived = TRUE; - for (j = 0; j < chunk_entry->num_writers - 1; j++) { - int count = 0; + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(contig_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - /* Probe for a particular message from any process, removing that message - * from the receive queue in the process and allocating that much memory - * for the asynchronous receive - */ - if (MPI_SUCCESS != (mpi_code = MPI_Mprobe(MPI_ANY_SOURCE, (int)chunk_entry->index, - io_info->comm, &message, &status))) - HMPI_GOTO_ERROR(FAIL, "MPI_Mprobe failed", mpi_code) - - if (MPI_SUCCESS != (mpi_code = MPI_Get_count(&status, MPI_BYTE, &count))) - HMPI_GOTO_ERROR(FAIL, "MPI_Get_count failed", mpi_code) - - HDassert(count >= 0); - if (NULL == (chunk_entry->async_info.receive_buffer_array[j] = - (unsigned char *)H5MM_malloc((size_t)count * sizeof(char *)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, - "unable to allocate modification data receive buffer") - - if (MPI_SUCCESS != (mpi_code = MPI_Imrecv( - chunk_entry->async_info.receive_buffer_array[j], count, MPI_BYTE, - &message, &chunk_entry->async_info.receive_requests_array[j]))) - HMPI_GOTO_ERROR(FAIL, "MPI_Imrecv failed", mpi_code) - } /* end for */ - } /* end if */ - - local_chunk_array[last_assigned_idx++] = local_chunk_array[i]; - } /* end else */ - } /* end for */ + struct_type_derived = FALSE; + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&struct_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_free failed", mpi_code) - *local_chunk_array_num_entries = last_assigned_idx; + /* + * Create struct type to correctly extract all needed + * information from a H5D_filtered_collective_io_info_t + * structure. + */ + displacements[0] = offsetof(H5D_filtered_collective_io_info_t, chunk_new); + displacements[1] = offsetof(H5D_filtered_collective_io_info_t, index_info.chunk_idx); + displacements[2] = offsetof(H5D_filtered_collective_io_info_t, index_info.filter_mask); + displacements[3] = offsetof(H5D_filtered_collective_io_info_t, index_info.need_insert); + if (MPI_SUCCESS != + (mpi_code = MPI_Type_create_struct(field_count, block_lengths, displacements, types, &struct_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) + struct_type_derived = TRUE; - /* Wait for all async send requests to complete before returning */ - if (num_send_requests) { - if (NULL == (send_statuses = (MPI_Status *)H5MM_malloc(num_send_requests * sizeof(MPI_Status)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate send statuses buffer") + if (MPI_SUCCESS != (mpi_code = MPI_Type_create_resized( + struct_type, 0, sizeof(H5D_filtered_collective_io_info_t), resized_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_resized failed", mpi_code) + *resized_type_derived = TRUE; - H5_CHECK_OVERFLOW(num_send_requests, size_t, int); - if (MPI_SUCCESS != (mpi_code = MPI_Waitall((int)num_send_requests, send_requests, send_statuses))) - HMPI_GOTO_ERROR(FAIL, "MPI_Waitall failed", mpi_code) - } /* end if */ + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(resized_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) done: - /* Now that all async send requests have completed, free up the send - * buffers used in the async operations - */ - for (i = 0; i < num_send_requests; i++) { - if (mod_data[i]) - H5MM_free(mod_data[i]); - } /* end for */ + if (struct_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&struct_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } + if (chunk_block_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&chunk_block_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + } - if (send_requests) - H5MM_free(send_requests); - if (send_statuses) - H5MM_free(send_statuses); - if (send_counts) - H5MM_free(send_counts); - if (send_displacements) - H5MM_free(send_displacements); - if (mod_data) - H5MM_free(mod_data); - if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") - if (mem_iter) - H5MM_free(mem_iter); - if (num_assigned_chunks_array) - H5MM_free(num_assigned_chunks_array); - if (shared_chunks_info_array) - H5MM_free(shared_chunks_info_array); + if (ret_value < 0) { + if (*resized_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(resized_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + *resized_type_derived = FALSE; + } + if (*contig_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(contig_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + *contig_type_derived = FALSE; + } + } FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__chunk_redistribute_shared_chunks() */ -#endif +} /* end H5D__mpio_get_chunk_insert_info_types() */ /*------------------------------------------------------------------------- - * Function: H5D__mpio_filtered_collective_write_type + * Function: H5D__mpio_collective_filtered_io_type * * Purpose: Constructs a MPI derived datatype for both the memory and - * the file for a collective write of filtered chunks. The - * datatype contains the offsets in the file and the locations - * of the filtered chunk data buffers. + * the file for a collective I/O operation on filtered chunks. + * The datatype contains the chunk offsets and lengths in the + * file and the locations of the chunk data buffers to read + * into/write from. * * Return: Non-negative on success/Negative on failure * - * Programmer: Jordan Henderson - * Tuesday, November 22, 2016 - * *------------------------------------------------------------------------- */ static herr_t -H5D__mpio_filtered_collective_write_type(H5D_filtered_collective_io_info_t *chunk_list, size_t num_entries, - MPI_Datatype *new_mem_type, hbool_t *mem_type_derived, - MPI_Datatype *new_file_type, hbool_t *file_type_derived) +H5D__mpio_collective_filtered_io_type(H5D_filtered_collective_io_info_t *chunk_list, size_t num_entries, + H5D_io_op_type_t op_type, MPI_Datatype *new_mem_type, + hbool_t *mem_type_derived, MPI_Datatype *new_file_type, + hbool_t *file_type_derived) { - MPI_Aint *write_buf_array = NULL; /* Relative displacements of filtered chunk data buffers */ + MPI_Aint *io_buf_array = NULL; /* Relative displacements of filtered chunk data buffers */ MPI_Aint *file_offset_array = NULL; /* Chunk offsets in the file */ int * length_array = NULL; /* Filtered Chunk lengths */ - herr_t ret_value = SUCCEED; + int mpi_code; + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC - HDassert(chunk_list); + HDassert(chunk_list || 0 == num_entries); HDassert(new_mem_type); HDassert(mem_type_derived); HDassert(new_file_type); HDassert(file_type_derived); - if (num_entries > 0) { - size_t i; - int mpi_code; - void * base_buf; - - H5_CHECK_OVERFLOW(num_entries, size_t, int); - - /* Allocate arrays */ - if (NULL == (length_array = (int *)H5MM_malloc((size_t)num_entries * sizeof(int)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "memory allocation failed for filtered collective write length array") - if (NULL == (write_buf_array = (MPI_Aint *)H5MM_malloc((size_t)num_entries * sizeof(MPI_Aint)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "memory allocation failed for filtered collective write buf length array") - if (NULL == (file_offset_array = (MPI_Aint *)H5MM_malloc((size_t)num_entries * sizeof(MPI_Aint)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, - "memory allocation failed for collective write offset array") - - /* Ensure the list is sorted in ascending order of offset in the file */ - HDqsort(chunk_list, num_entries, sizeof(H5D_filtered_collective_io_info_t), - H5D__cmp_filtered_collective_io_info_entry); - - base_buf = chunk_list[0].buf; - for (i = 0; i < num_entries; i++) { - /* Set up the offset in the file, the length of the chunk data, and the relative - * displacement of the chunk data write buffer - */ - file_offset_array[i] = (MPI_Aint)chunk_list[i].chunk_states.new_chunk.offset; - length_array[i] = (int)chunk_list[i].chunk_states.new_chunk.length; - write_buf_array[i] = (MPI_Aint)chunk_list[i].buf - (MPI_Aint)base_buf; - } /* end for */ - - /* Create memory MPI type */ - if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)num_entries, length_array, - write_buf_array, MPI_BYTE, new_mem_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) - *mem_type_derived = TRUE; - if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_mem_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - - /* Create file MPI type */ - if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)num_entries, length_array, - file_offset_array, MPI_BYTE, new_file_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) - *file_type_derived = TRUE; - if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_file_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - } /* end if */ - -done: - if (write_buf_array) - H5MM_free(write_buf_array); - if (file_offset_array) - H5MM_free(file_offset_array); - if (length_array) - H5MM_free(length_array); - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__mpio_filtered_collective_write_type() */ - -/*------------------------------------------------------------------------- - * Function: H5D__filtered_collective_chunk_entry_io - * - * Purpose: Given an entry for a filtered chunk, performs the necessary - * steps for updating the chunk data during a collective - * write, or for reading the chunk from file during a - * collective read. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Jordan Henderson - * Wednesday, January 18, 2017 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5D__filtered_collective_chunk_entry_io(H5D_filtered_collective_io_info_t *chunk_entry, - const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - const H5D_chunk_map_t *fm) -{ - H5D_chunk_info_t *chunk_info = NULL; - H5S_sel_iter_t * mem_iter = NULL; /* Memory iterator for H5D__scatter_mem/H5D__gather_mem */ - H5S_sel_iter_t * file_iter = NULL; - H5Z_EDC_t err_detect; /* Error detection info */ - H5Z_cb_t filter_cb; /* I/O filter callback function */ - unsigned filter_mask = 0; - hsize_t iter_nelmts; /* Number of points to iterate over for the chunk IO operation */ - hssize_t extent_npoints; - hsize_t true_chunk_size; - hbool_t mem_iter_init = FALSE; - hbool_t file_iter_init = FALSE; - size_t buf_size; - size_t i; - H5S_t * dataspace = NULL; /* Other process' dataspace for the chunk */ - void * tmp_gath_buf = NULL; /* Temporary gather buffer to gather into from application buffer - before scattering out to the chunk data buffer (when writing data), - or vice versa (when reading data) */ - int mpi_code; - herr_t ret_value = SUCCEED; - - FUNC_ENTER_STATIC - - HDassert(chunk_entry); - HDassert(io_info); - HDassert(type_info); - HDassert(fm); - - /* Retrieve filter settings from API context */ - if (H5CX_get_err_detect(&err_detect) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get error detection info") - if (H5CX_get_filter_cb(&filter_cb) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get I/O filter callback function") - - /* Look up the chunk and get its file and memory dataspaces */ - if (NULL == (chunk_info = (H5D_chunk_info_t *)H5SL_search(fm->sel_chunks, &chunk_entry->index))) - HGOTO_ERROR(H5E_DATASPACE, H5E_NOTFOUND, FAIL, "can't locate chunk in skip list") - - if ((extent_npoints = H5S_GET_EXTENT_NPOINTS(chunk_info->fspace)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "dataspace is invalid") - true_chunk_size = (hsize_t)extent_npoints * type_info->src_type_size; - - /* If the size of the filtered chunk is larger than the number of points in the - * chunk file space extent times the datatype size, allocate enough space to hold the - * whole filtered chunk. Otherwise, allocate a buffer equal to the size of the - * chunk so that the unfiltering operation doesn't have to grow the buffer. - */ - buf_size = MAX(chunk_entry->chunk_states.chunk_current.length, true_chunk_size); + *mem_type_derived = FALSE; + *file_type_derived = FALSE; + *new_mem_type = MPI_BYTE; + *new_file_type = MPI_BYTE; - if (NULL == (chunk_entry->buf = H5MM_malloc(buf_size))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate chunk data buffer") + if (num_entries > 0) { + H5F_block_t *chunk_block; + size_t last_valid_idx = 0; + size_t i; + int chunk_count; - /* If this is not a full chunk overwrite or this is a read operation, the chunk must be - * read from the file and unfiltered. - */ - if (!chunk_entry->full_overwrite || io_info->op_type == H5D_IO_OP_READ) { - H5FD_mpio_xfer_t xfer_mode; /* Parallel transfer for this request */ + /* + * Determine number of chunks for I/O operation and + * setup for derived datatype creation if I/O operation + * includes multiple chunks + */ + if (num_entries == 1) { + /* Set last valid index to 0 for contiguous datatype creation */ + last_valid_idx = 0; - chunk_entry->chunk_states.new_chunk.length = chunk_entry->chunk_states.chunk_current.length; + if (op_type == H5D_IO_OP_WRITE) + chunk_count = 1; + else + chunk_count = chunk_list[0].need_read ? 1 : 0; + } + else { + MPI_Aint chunk_buf; + MPI_Aint base_buf; + haddr_t base_offset = HADDR_UNDEF; + + H5_CHECK_OVERFLOW(num_entries, size_t, int); + + /* Allocate arrays */ + if (NULL == (length_array = H5MM_malloc((size_t)num_entries * sizeof(int)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for filtered collective I/O length array") + if (NULL == (io_buf_array = H5MM_malloc((size_t)num_entries * sizeof(MPI_Aint)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for filtered collective I/O buf length array") + if (NULL == (file_offset_array = H5MM_malloc((size_t)num_entries * sizeof(MPI_Aint)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "memory allocation failed for filtered collective I/O offset array") - /* Currently, these chunk reads are done independently and will likely - * cause issues with collective metadata reads enabled. In the future, - * this should be refactored to use collective chunk reads - JTH */ + /* + * If doing a write, we can set the base chunk offset + * and base chunk data buffer right away. + * + * If doing a read, some chunks may be skipped over + * for reading if they aren't yet allocated in the + * file. Therefore, we have to find the first chunk + * actually being read in order to set the base chunk + * offset and base chunk data buffer. + */ + if (op_type == H5D_IO_OP_WRITE) { +#if MPI_VERSION >= 3 + if (MPI_SUCCESS != (mpi_code = MPI_Get_address(chunk_list[0].buf, &base_buf))) + HMPI_GOTO_ERROR(FAIL, "MPI_Get_address failed", mpi_code) +#else + base_buf = (MPI_Aint)chunk_list[0].buf; +#endif - /* Get the original state of parallel I/O transfer mode */ - if (H5CX_get_io_xfer_mode(&xfer_mode) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get MPI-I/O transfer mode") + base_offset = chunk_list[0].chunk_new.offset; + } - /* Change the xfer_mode to independent for handling the I/O */ - if (H5CX_set_io_xfer_mode(H5FD_MPIO_INDEPENDENT) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set MPI-I/O transfer mode") + for (i = 0, chunk_count = 0; i < num_entries; i++) { + if (op_type == H5D_IO_OP_READ) { + /* + * If this chunk isn't being read, don't add it + * to the MPI type we're building up for I/O + */ + if (!chunk_list[i].need_read) + continue; - if (H5F_shared_block_read(io_info->f_sh, H5FD_MEM_DRAW, - chunk_entry->chunk_states.chunk_current.offset, - chunk_entry->chunk_states.new_chunk.length, chunk_entry->buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read raw data chunk") + /* + * If this chunk is being read, go ahead and + * set the base chunk offset and base chunk + * data buffer if we haven't already + */ + if (!H5F_addr_defined(base_offset)) { +#if MPI_VERSION >= 3 + if (MPI_SUCCESS != (mpi_code = MPI_Get_address(chunk_list[i].buf, &base_buf))) + HMPI_GOTO_ERROR(FAIL, "MPI_Get_address failed", mpi_code) +#else + base_buf = (MPI_Aint)chunk_list[i].buf; +#endif - /* Return to the original I/O transfer mode setting */ - if (H5CX_set_io_xfer_mode(xfer_mode) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set MPI-I/O transfer mode") + base_offset = chunk_list[i].chunk_current.offset; + } + } - if (H5Z_pipeline(&io_info->dset->shared->dcpl_cache.pline, H5Z_FLAG_REVERSE, &filter_mask, err_detect, - filter_cb, (size_t *)&chunk_entry->chunk_states.new_chunk.length, &buf_size, - &chunk_entry->buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFILTER, FAIL, "couldn't unfilter chunk for modifying") - } /* end if */ - else { - chunk_entry->chunk_states.new_chunk.length = true_chunk_size; - } /* end else */ + /* Set convenience pointer for current chunk block */ + chunk_block = + (op_type == H5D_IO_OP_READ) ? &chunk_list[i].chunk_current : &chunk_list[i].chunk_new; - /* Initialize iterator for memory selection */ - if (NULL == (mem_iter = (H5S_sel_iter_t *)H5MM_malloc(sizeof(H5S_sel_iter_t)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate memory iterator") + /* + * Set the current chunk entry's offset in the file, relative to + * the first chunk entry + */ + HDassert(H5F_addr_defined(chunk_block->offset)); + file_offset_array[chunk_count] = (MPI_Aint)(chunk_block->offset - base_offset); - if (H5S_select_iter_init(mem_iter, chunk_info->mspace, type_info->src_type_size, 0) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize memory selection information") - mem_iter_init = TRUE; + /* + * Ensure the chunk list is sorted in ascending ordering of + * offset in the file + */ + if (chunk_count) + HDassert(file_offset_array[chunk_count] > file_offset_array[chunk_count - 1]); - /* If this is a read operation, scatter the read chunk data to the user's buffer. - * - * If this is a write operation, update the chunk data buffer with the modifications - * from the current process, then apply any modifications from other processes. Finally, - * filter the newly-updated chunk. - */ - switch (io_info->op_type) { - case H5D_IO_OP_READ: - if (NULL == (file_iter = (H5S_sel_iter_t *)H5MM_malloc(sizeof(H5S_sel_iter_t)))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate file iterator") + /* Set the current chunk entry's size for the I/O operation */ + H5_CHECK_OVERFLOW(chunk_block->length, hsize_t, int); + length_array[chunk_count] = (int)chunk_block->length; - if (H5S_select_iter_init(file_iter, chunk_info->fspace, type_info->src_type_size, 0) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to initialize memory selection information") - file_iter_init = TRUE; + /* + * Set the displacement of the chunk entry's chunk data buffer, + * relative to the first entry's data buffer + */ +#if MPI_VERSION >= 3 && MPI_SUBVERSION >= 1 + if (MPI_SUCCESS != (mpi_code = MPI_Get_address(chunk_list[i].buf, &chunk_buf))) + HMPI_GOTO_ERROR(FAIL, "MPI_Get_address failed", mpi_code) - iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->fspace); + io_buf_array[chunk_count] = MPI_Aint_diff(chunk_buf, base_buf); +#else + chunk_buf = (MPI_Aint)chunk_list[i].buf; + io_buf_array[chunk_count] = chunk_buf - base_buf; +#endif - if (NULL == (tmp_gath_buf = H5MM_malloc(iter_nelmts * type_info->src_type_size))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate temporary gather buffer") + /* + * Set last valid index in case only a single chunk will + * be involved in the I/O operation + */ + last_valid_idx = i; - if (!H5D__gather_mem(chunk_entry->buf, file_iter, (size_t)iter_nelmts, tmp_gath_buf)) - HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "couldn't gather from chunk buffer") + chunk_count++; + } /* end for */ + } - iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->mspace); + /* + * Create derived datatypes for the chunk list if this + * rank has any chunks to work on + */ + if (chunk_count > 0) { + if (chunk_count == 1) { + int chunk_len; - if (H5D__scatter_mem(tmp_gath_buf, mem_iter, (size_t)iter_nelmts, io_info->u.rbuf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't scatter to read buffer") + /* Single chunk - use a contiguous type for both memory and file */ - break; + /* Ensure that we can cast chunk size to an int for MPI */ + chunk_block = (op_type == H5D_IO_OP_READ) ? &chunk_list[last_valid_idx].chunk_current + : &chunk_list[last_valid_idx].chunk_new; + H5_CHECKED_ASSIGN(chunk_len, int, chunk_block->length, hsize_t); - case H5D_IO_OP_WRITE: - iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->mspace); + if (MPI_SUCCESS != (mpi_code = MPI_Type_contiguous(chunk_len, MPI_BYTE, new_file_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + *new_mem_type = *new_file_type; - if (NULL == (tmp_gath_buf = H5MM_malloc(iter_nelmts * type_info->src_type_size))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "couldn't allocate temporary gather buffer") + /* + * Since we use the same datatype for both memory and file, only + * mark the file type as derived so the caller doesn't try to + * free the same type twice + */ + *mem_type_derived = FALSE; + *file_type_derived = TRUE; - /* Gather modification data from the application write buffer into a temporary buffer */ - if (0 == H5D__gather_mem(io_info->u.wbuf, mem_iter, (size_t)iter_nelmts, tmp_gath_buf)) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "couldn't gather from write buffer") + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_file_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + } + else { + HDassert(file_offset_array); + HDassert(length_array); + HDassert(io_buf_array); - if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") - mem_iter_init = FALSE; + /* Multiple chunks - use an hindexed type for both memory and file */ - /* Initialize iterator for file selection */ - if (H5S_select_iter_init(mem_iter, chunk_info->fspace, type_info->dst_type_size, 0) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to initialize file selection information") - mem_iter_init = TRUE; + /* Create memory MPI type */ + if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed( + chunk_count, length_array, io_buf_array, MPI_BYTE, new_mem_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) + *mem_type_derived = TRUE; - iter_nelmts = H5S_GET_SELECT_NPOINTS(chunk_info->fspace); + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_mem_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - /* Scatter the owner's modification data into the chunk data buffer according to - * the file space. - */ - if (H5D__scatter_mem(tmp_gath_buf, mem_iter, (size_t)iter_nelmts, chunk_entry->buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "couldn't scatter to chunk data buffer") + /* Create file MPI type */ + if (MPI_SUCCESS != + (mpi_code = MPI_Type_create_hindexed(chunk_count, length_array, file_offset_array, + MPI_BYTE, new_file_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) + *file_type_derived = TRUE; - if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") - mem_iter_init = FALSE; + if (MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_file_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + } + } + } /* end if */ - if (MPI_SUCCESS != - (mpi_code = MPI_Waitall(chunk_entry->async_info.num_receive_requests, - chunk_entry->async_info.receive_requests_array, MPI_STATUSES_IGNORE))) - HMPI_GOTO_ERROR(FAIL, "MPI_Waitall failed", mpi_code) +done: + if (file_offset_array) + H5MM_free(file_offset_array); + if (io_buf_array) + H5MM_free(io_buf_array); + if (length_array) + H5MM_free(length_array); - /* For each asynchronous receive call previously posted, receive the chunk modification - * buffer from another rank and update the chunk data - */ - for (i = 0; i < (size_t)chunk_entry->async_info.num_receive_requests; i++) { - const unsigned char *mod_data_p; + if (ret_value < 0) { + if (*file_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(new_file_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + *file_type_derived = FALSE; + } + if (*mem_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(new_mem_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + *mem_type_derived = FALSE; + } + } - /* Decode the process' chunk file dataspace */ - mod_data_p = chunk_entry->async_info.receive_buffer_array[i]; - if (NULL == (dataspace = H5S_decode(&mod_data_p))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTDECODE, FAIL, "unable to decode dataspace") + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_collective_filtered_io_type() */ - if (H5S_select_iter_init(mem_iter, dataspace, type_info->dst_type_size, 0) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to initialize memory selection information") - mem_iter_init = TRUE; +#ifdef H5Dmpio_DEBUG - iter_nelmts = H5S_GET_SELECT_NPOINTS(dataspace); +static herr_t +H5D__mpio_dump_collective_filtered_chunk_list(H5D_filtered_collective_io_info_t *chunk_list, + size_t chunk_list_num_entries, int mpi_rank) +{ + H5D_filtered_collective_io_info_t *chunk_entry; + size_t i; + herr_t ret_value = SUCCEED; - /* Update the chunk data with the received modification data */ - if (H5D__scatter_mem(mod_data_p, mem_iter, (size_t)iter_nelmts, chunk_entry->buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "couldn't scatter to write buffer") + FUNC_ENTER_STATIC_NOERR - if (H5S_SELECT_ITER_RELEASE(mem_iter) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") - mem_iter_init = FALSE; - if (dataspace) { - if (H5S_close(dataspace) < 0) - HGOTO_ERROR(H5E_DATASPACE, H5E_CANTFREE, FAIL, "can't close dataspace") - dataspace = NULL; - } - H5MM_free(chunk_entry->async_info.receive_buffer_array[i]); - } /* end for */ + H5D_MPIO_DEBUG(mpi_rank, "CHUNK LIST: ["); + for (i = 0; i < chunk_list_num_entries; i++) { + unsigned chunk_rank; + + chunk_entry = &chunk_list[i]; + + HDassert(chunk_entry->chunk_info); + chunk_rank = (unsigned)H5S_GET_EXTENT_NDIMS(chunk_entry->chunk_info->fspace); + + H5D_MPIO_DEBUG(mpi_rank, " {"); + H5D_MPIO_DEBUG_VA(mpi_rank, " - Entry %zu -", i); + + H5D_MPIO_DEBUG(mpi_rank, " - Chunk Fspace Info -"); + H5D_MPIO_DEBUG_VA(mpi_rank, + " Chunk Current Info: { Offset: %" PRIuHADDR ", Length: %" PRIuHADDR " }", + chunk_entry->chunk_current.offset, chunk_entry->chunk_current.length); + H5D_MPIO_DEBUG_VA(mpi_rank, " Chunk New Info: { Offset: %" PRIuHADDR ", Length: %" PRIuHADDR " }", + chunk_entry->chunk_new.offset, chunk_entry->chunk_new.length); + + H5D_MPIO_DEBUG(mpi_rank, " - Chunk Insert Info -"); + H5D_MPIO_DEBUG_VA(mpi_rank, + " Chunk Scaled Coords (4-d): { %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE + ", %" PRIuHSIZE " }", + chunk_rank < 1 ? 0 : chunk_entry->chunk_info->scaled[0], + chunk_rank < 2 ? 0 : chunk_entry->chunk_info->scaled[1], + chunk_rank < 3 ? 0 : chunk_entry->chunk_info->scaled[2], + chunk_rank < 4 ? 0 : chunk_entry->chunk_info->scaled[3]); + H5D_MPIO_DEBUG_VA(mpi_rank, " Chunk Index: %" PRIuHSIZE, chunk_entry->index_info.chunk_idx); + H5D_MPIO_DEBUG_VA(mpi_rank, " Filter Mask: %u", chunk_entry->index_info.filter_mask); + H5D_MPIO_DEBUG_VA(mpi_rank, " Need Insert: %s", + chunk_entry->index_info.need_insert ? "YES" : "NO"); + + H5D_MPIO_DEBUG(mpi_rank, " - Other Info -"); + H5D_MPIO_DEBUG_VA(mpi_rank, " Chunk Info Ptr: %p", (void *)chunk_entry->chunk_info); + H5D_MPIO_DEBUG_VA(mpi_rank, " Need Read: %s", chunk_entry->need_read ? "YES" : "NO"); + H5D_MPIO_DEBUG_VA(mpi_rank, " Chunk I/O Size: %zu", chunk_entry->io_size); + H5D_MPIO_DEBUG_VA(mpi_rank, " Chunk Buffer Size: %zu", chunk_entry->chunk_buf_size); + H5D_MPIO_DEBUG_VA(mpi_rank, " Original Owner: %d", chunk_entry->orig_owner); + H5D_MPIO_DEBUG_VA(mpi_rank, " New Owner: %d", chunk_entry->new_owner); + H5D_MPIO_DEBUG_VA(mpi_rank, " # of Writers: %d", chunk_entry->num_writers); + H5D_MPIO_DEBUG_VA(mpi_rank, " Chunk Data Buffer Ptr: %p", (void *)chunk_entry->buf); + + H5D_MPIO_DEBUG(mpi_rank, " }"); + } + H5D_MPIO_DEBUG(mpi_rank, "]"); - /* Filter the chunk */ - if (H5Z_pipeline(&io_info->dset->shared->dcpl_cache.pline, 0, &filter_mask, err_detect, filter_cb, - (size_t *)&chunk_entry->chunk_states.new_chunk.length, &buf_size, - &chunk_entry->buf) < 0) - HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "output pipeline failed") + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__mpio_dump_collective_filtered_chunk_list() */ -#if H5_SIZEOF_SIZE_T > 4 - /* Check for the chunk expanding too much to encode in a 32-bit value */ - if (chunk_entry->chunk_states.new_chunk.length > ((size_t)0xffffffff)) - HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk too large for 32-bit length") #endif - break; - - default: - HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "invalid I/O operation") - } /* end switch */ - -done: - if (chunk_entry->async_info.receive_buffer_array) - H5MM_free(chunk_entry->async_info.receive_buffer_array); - if (chunk_entry->async_info.receive_requests_array) - H5MM_free(chunk_entry->async_info.receive_requests_array); - if (tmp_gath_buf) - H5MM_free(tmp_gath_buf); - if (file_iter_init && H5S_SELECT_ITER_RELEASE(file_iter) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") - if (file_iter) - H5MM_free(file_iter); - if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "couldn't release selection iterator") - if (mem_iter) - H5MM_free(mem_iter); - if (dataspace) - if (H5S_close(dataspace) < 0) - HDONE_ERROR(H5E_DATASPACE, H5E_CANTFREE, FAIL, "can't close dataspace") - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__filtered_collective_chunk_entry_io() */ #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 49c95a5..a424929 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -559,6 +559,7 @@ H5_DLL herr_t H5D__alloc_storage(const H5D_io_info_t *io_info, H5D_time_alloc_t hbool_t full_overwrite, hsize_t old_dim[]); H5_DLL herr_t H5D__get_storage_size(const H5D_t *dset, hsize_t *storage_size); H5_DLL herr_t H5D__get_chunk_storage_size(H5D_t *dset, const hsize_t *offset, hsize_t *storage_size); +H5_DLL herr_t H5D__chunk_index_empty(const H5D_t *dset, hbool_t *empty); H5_DLL herr_t H5D__get_num_chunks(const H5D_t *dset, const H5S_t *space, hsize_t *nchunks); H5_DLL herr_t H5D__get_chunk_info(const H5D_t *dset, const H5S_t *space, hsize_t chk_idx, hsize_t *coord, unsigned *filter_mask, haddr_t *offset, hsize_t *size); @@ -591,6 +592,10 @@ H5_DLL herr_t H5D__select_read(const H5D_io_info_t *io_info, const H5D_type_info H5_DLL herr_t H5D__select_write(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space); +/* Functions that perform direct copying between memory buffers */ +H5_DLL herr_t H5D_select_io_mem(void *dst_buf, const H5S_t *dst_space, const void *src_buf, + const H5S_t *src_space, size_t elmt_size, size_t nelmts); + /* Functions that perform scatter-gather serial I/O operations */ H5_DLL herr_t H5D__scatter_mem(const void *_tscat_buf, H5S_sel_iter_t *iter, size_t nelmts, void *_buf); H5_DLL size_t H5D__gather_mem(const void *_buf, H5S_sel_iter_t *iter, size_t nelmts, @@ -635,7 +640,13 @@ H5_DLL herr_t H5D__chunk_allocate(const H5D_io_info_t *io_info, hbool_t full_ov const hsize_t old_dim[]); H5_DLL herr_t H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info, const H5F_block_t *old_chunk, H5F_block_t *new_chunk, hbool_t *need_insert, const hsize_t *scaled); +H5_DLL void * H5D__chunk_mem_alloc(size_t size, const H5O_pline_t *pline); +H5_DLL void H5D__chunk_mem_free(void *chk, const void *_pline); +H5_DLL void * H5D__chunk_mem_xfree(void *chk, const void *pline); +H5_DLL void * H5D__chunk_mem_realloc(void *chk, size_t size, const H5O_pline_t *pline); H5_DLL herr_t H5D__chunk_update_old_edge_chunks(H5D_t *dset, hsize_t old_dim[]); +H5_DLL hbool_t H5D__chunk_is_partial_edge_chunk(unsigned dset_ndims, const uint32_t *chunk_dims, + const hsize_t *chunk_scaled, const hsize_t *dset_dims); H5_DLL herr_t H5D__chunk_prune_by_extent(H5D_t *dset, const hsize_t *old_dim); H5_DLL herr_t H5D__chunk_set_sizes(H5D_t *dset); #ifdef H5_HAVE_PARALLEL @@ -694,11 +705,11 @@ H5_DLL herr_t H5D__fill_term(H5D_fill_buf_info_t *fb_info); #ifdef H5_HAVE_PARALLEL -#ifdef H5S_DEBUG +#ifdef H5D_DEBUG #ifndef H5Dmpio_DEBUG #define H5Dmpio_DEBUG #endif /*H5Dmpio_DEBUG*/ -#endif /*H5S_DEBUG*/ +#endif /*H5D_DEBUG*/ /* MPI-IO function to read, it will select either regular or irregular read */ H5_DLL herr_t H5D__mpio_select_read(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space); @@ -727,6 +738,8 @@ H5_DLL herr_t H5D__chunk_collective_write(H5D_io_info_t *io_info, const H5D_type * memory and the file */ H5_DLL htri_t H5D__mpio_opt_possible(const H5D_io_info_t *io_info, const H5S_t *file_space, const H5S_t *mem_space, const H5D_type_info_t *type_info); +H5_DLL herr_t H5D__mpio_get_no_coll_cause_strings(char *local_cause, size_t local_cause_len, + char *global_cause, size_t global_cause_len); #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Dselect.c b/src/H5Dselect.c index e64d657..f464ca5 100644 --- a/src/H5Dselect.c +++ b/src/H5Dselect.c @@ -105,6 +105,9 @@ H5D__select_io(const H5D_io_info_t *io_info, size_t elmt_size, size_t nelmts, H5 HDassert(io_info->store); HDassert(io_info->u.rbuf); + if (elmt_size == 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid elmt_size of 0") + /* Check for only one element in selection */ if (nelmts == 1) { hsize_t single_mem_off; /* Offset in memory */ @@ -226,8 +229,6 @@ H5D__select_io(const H5D_io_info_t *io_info, size_t elmt_size, size_t nelmts, H5 /* Decrement number of elements left to process */ HDassert(((size_t)tmp_file_len % elmt_size) == 0); - if (elmt_size == 0) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "Resulted in division by zero") nelmts -= ((size_t)tmp_file_len / elmt_size); } /* end while */ } /* end else */ @@ -257,6 +258,188 @@ done: } /* end H5D__select_io() */ /*------------------------------------------------------------------------- + * Function: H5D_select_io_mem + * + * Purpose: Perform memory copies directly between two memory buffers + * according to the selections in the `dst_space` and + * `src_space` dataspaces. + * + * Note: This routine is [basically] the same as H5D__select_io, + * with the only difference being that the readvv/writevv + * calls are exchanged for H5VM_memcpyvv calls. Changes should + * be made to both routines. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_select_io_mem(void *dst_buf, const H5S_t *dst_space, const void *src_buf, const H5S_t *src_space, + size_t elmt_size, size_t nelmts) +{ + H5S_sel_iter_t *dst_sel_iter = NULL; /* Destination dataspace iteration info */ + H5S_sel_iter_t *src_sel_iter = NULL; /* Source dataspace iteration info */ + hbool_t dst_sel_iter_init = FALSE; /* Destination dataspace selection iterator initialized? */ + hbool_t src_sel_iter_init = FALSE; /* Source dataspace selection iterator initialized? */ + hsize_t * dst_off = NULL; /* Pointer to sequence offsets in destination buffer */ + hsize_t * src_off = NULL; /* Pointer to sequence offsets in source buffer */ + size_t * dst_len = NULL; /* Pointer to sequence lengths in destination buffer */ + size_t * src_len = NULL; /* Pointer to sequence lengths in source buffer */ + size_t curr_dst_seq; /* Current destination buffer sequence to operate on */ + size_t curr_src_seq; /* Current source buffer sequence to operate on */ + size_t dst_nseq; /* Number of sequences generated for destination buffer */ + size_t src_nseq; /* Number of sequences generated for source buffer */ + size_t dxpl_vec_size; /* Vector length from API context's DXPL */ + size_t vec_size; /* Vector length */ + ssize_t bytes_copied; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + HDassert(dst_buf); + HDassert(dst_space); + HDassert(src_buf); + HDassert(src_space); + + if (elmt_size == 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid elmt_size of 0") + + /* Check for only one element in selection */ + if (nelmts == 1) { + hsize_t single_dst_off; /* Offset in dst_space */ + hsize_t single_src_off; /* Offset in src_space */ + size_t single_dst_len; /* Length in dst_space */ + size_t single_src_len; /* Length in src_space */ + + /* Get offset of first element in selections */ + if (H5S_SELECT_OFFSET(dst_space, &single_dst_off) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't retrieve destination selection offset") + if (H5S_SELECT_OFFSET(src_space, &single_src_off) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't retrieve source selection offset") + + /* Set up necessary information for I/O operation */ + dst_nseq = src_nseq = 1; + curr_dst_seq = curr_src_seq = 0; + single_dst_off *= elmt_size; + single_src_off *= elmt_size; + single_dst_len = single_src_len = elmt_size; + + /* Perform vectorized memcpy from src_buf to dst_buf */ + if ((bytes_copied = + H5VM_memcpyvv(dst_buf, dst_nseq, &curr_dst_seq, &single_dst_len, &single_dst_off, src_buf, + src_nseq, &curr_src_seq, &single_src_len, &single_src_off)) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "vectorized memcpy failed") + + HDassert(((size_t)bytes_copied % elmt_size) == 0); + } + else { + unsigned sel_iter_flags = H5S_SEL_ITER_GET_SEQ_LIST_SORTED | H5S_SEL_ITER_SHARE_WITH_DATASPACE; + size_t dst_nelem; /* Number of elements used in destination buffer sequences */ + size_t src_nelem; /* Number of elements used in source buffer sequences */ + + /* Get info from API context */ + if (H5CX_get_vec_size(&dxpl_vec_size) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTGET, FAIL, "can't retrieve I/O vector size") + + /* Allocate the vector I/O arrays */ + if (dxpl_vec_size > H5D_IO_VECTOR_SIZE) + vec_size = dxpl_vec_size; + else + vec_size = H5D_IO_VECTOR_SIZE; + + if (NULL == (dst_len = H5FL_SEQ_MALLOC(size_t, vec_size))) + HGOTO_ERROR(H5E_IO, H5E_CANTALLOC, FAIL, "can't allocate I/O length vector array") + if (NULL == (dst_off = H5FL_SEQ_MALLOC(hsize_t, vec_size))) + HGOTO_ERROR(H5E_IO, H5E_CANTALLOC, FAIL, "can't allocate I/O offset vector array") + if (NULL == (src_len = H5FL_SEQ_MALLOC(size_t, vec_size))) + HGOTO_ERROR(H5E_IO, H5E_CANTALLOC, FAIL, "can't allocate I/O length vector array") + if (NULL == (src_off = H5FL_SEQ_MALLOC(hsize_t, vec_size))) + HGOTO_ERROR(H5E_IO, H5E_CANTALLOC, FAIL, "can't allocate I/O offset vector array") + + /* Allocate the dataspace selection iterators */ + if (NULL == (dst_sel_iter = H5FL_MALLOC(H5S_sel_iter_t))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate destination selection iterator") + if (NULL == (src_sel_iter = H5FL_MALLOC(H5S_sel_iter_t))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate source selection iterator") + + /* Initialize destination selection iterator */ + if (H5S_select_iter_init(dst_sel_iter, dst_space, elmt_size, sel_iter_flags) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize selection iterator") + dst_sel_iter_init = TRUE; /* Destination selection iteration info has been initialized */ + + /* Initialize source selection iterator */ + if (H5S_select_iter_init(src_sel_iter, src_space, elmt_size, H5S_SEL_ITER_SHARE_WITH_DATASPACE) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize selection iterator") + src_sel_iter_init = TRUE; /* Source selection iteration info has been initialized */ + + /* Initialize sequence counts */ + curr_dst_seq = curr_src_seq = 0; + dst_nseq = src_nseq = 0; + + /* Loop, until all bytes are processed */ + while (nelmts > 0) { + /* Check if more destination buffer sequences are needed */ + if (curr_dst_seq >= dst_nseq) { + /* Get sequences for destination selection */ + if (H5S_SELECT_ITER_GET_SEQ_LIST(dst_sel_iter, vec_size, nelmts, &dst_nseq, &dst_nelem, + dst_off, dst_len) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "sequence length generation failed") + + /* Start at the beginning of the sequences again */ + curr_dst_seq = 0; + } + + /* Check if more source buffer sequences are needed */ + if (curr_src_seq >= src_nseq) { + /* Get sequences for source selection */ + if (H5S_SELECT_ITER_GET_SEQ_LIST(src_sel_iter, vec_size, nelmts, &src_nseq, &src_nelem, + src_off, src_len) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "sequence length generation failed") + + /* Start at the beginning of the sequences again */ + curr_src_seq = 0; + } /* end if */ + + /* Perform vectorized memcpy from src_buf to dst_buf */ + if ((bytes_copied = H5VM_memcpyvv(dst_buf, dst_nseq, &curr_dst_seq, dst_len, dst_off, src_buf, + src_nseq, &curr_src_seq, src_len, src_off)) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "vectorized memcpy failed") + + /* Decrement number of elements left to process */ + HDassert(((size_t)bytes_copied % elmt_size) == 0); + nelmts -= ((size_t)bytes_copied / elmt_size); + } + } + +done: + /* Release selection iterators */ + if (src_sel_iter) { + if (src_sel_iter_init && H5S_SELECT_ITER_RELEASE(src_sel_iter) < 0) + HDONE_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection iterator") + + src_sel_iter = H5FL_FREE(H5S_sel_iter_t, src_sel_iter); + } + if (dst_sel_iter) { + if (dst_sel_iter_init && H5S_SELECT_ITER_RELEASE(dst_sel_iter) < 0) + HDONE_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection iterator") + + dst_sel_iter = H5FL_FREE(H5S_sel_iter_t, dst_sel_iter); + } + + /* Release vector arrays, if allocated */ + if (src_off) + src_off = H5FL_SEQ_FREE(hsize_t, src_off); + if (src_len) + src_len = H5FL_SEQ_FREE(size_t, src_len); + if (dst_off) + dst_off = H5FL_SEQ_FREE(hsize_t, dst_off); + if (dst_len) + dst_len = H5FL_SEQ_FREE(size_t, dst_len); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_select_io_mem() */ + +/*------------------------------------------------------------------------- * Function: H5D__select_read * * Purpose: Reads directly from file into application memory. diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index 1969899..4aa8a96 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -188,6 +188,41 @@ H5FD__mpio_parse_debug_str(const char *s) FUNC_LEAVE_NOAPI_VOID } /* end H5FD__mpio_parse_debug_str() */ + +/*--------------------------------------------------------------------------- + * Function: H5FD__mem_t_to_str + * + * Purpose: Returns a string representing the enum value in an H5FD_mem_t + * enum + * + * Returns: H5FD_mem_t enum value string + * + *--------------------------------------------------------------------------- + */ +static const char * +H5FD__mem_t_to_str(H5FD_mem_t mem_type) +{ + switch (mem_type) { + case H5FD_MEM_NOLIST: + return "H5FD_MEM_NOLIST"; + case H5FD_MEM_DEFAULT: + return "H5FD_MEM_DEFAULT"; + case H5FD_MEM_SUPER: + return "H5FD_MEM_SUPER"; + case H5FD_MEM_BTREE: + return "H5FD_MEM_BTREE"; + case H5FD_MEM_DRAW: + return "H5FD_MEM_DRAW"; + case H5FD_MEM_GHEAP: + return "H5FD_MEM_GHEAP"; + case H5FD_MEM_LHEAP: + return "H5FD_MEM_LHEAP"; + case H5FD_MEM_OHDR: + return "H5FD_MEM_OHDR"; + default: + return "(Unknown)"; + } +} #endif /* H5FDmpio_DEBUG */ /*------------------------------------------------------------------------- @@ -994,7 +1029,6 @@ H5FD__mpio_query(const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags /* out *flags |= H5FD_FEAT_AGGREGATE_METADATA; /* OK to aggregate metadata allocations */ *flags |= H5FD_FEAT_AGGREGATE_SMALLDATA; /* OK to aggregate "small" raw data allocations */ *flags |= H5FD_FEAT_HAS_MPI; /* This driver uses MPI */ - *flags |= H5FD_FEAT_ALLOCATE_EARLY; /* Allocate space early instead of late */ *flags |= H5FD_FEAT_DEFAULT_VFD_COMPATIBLE; /* VFD creates a file which can be opened with the default VFD */ } /* end if */ @@ -1380,8 +1414,8 @@ H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU #ifdef H5FDmpio_DEBUG if (H5FD_mpio_debug_r_flag) - HDfprintf(stderr, "%s: (%d) mpi_off = %ld bytes_read = %lld\n", __func__, file->mpi_rank, - (long)mpi_off, bytes_read); + HDfprintf(stderr, "%s: (%d) mpi_off = %ld bytes_read = %lld type = %s\n", __func__, file->mpi_rank, + (long)mpi_off, bytes_read, H5FD__mem_t_to_str(type)); #endif /* @@ -1601,8 +1635,8 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, h #ifdef H5FDmpio_DEBUG if (H5FD_mpio_debug_w_flag) - HDfprintf(stderr, "%s: (%d) mpi_off = %ld bytes_written = %lld\n", __func__, file->mpi_rank, - (long)mpi_off, bytes_written); + HDfprintf(stderr, "%s: (%d) mpi_off = %ld bytes_written = %lld type = %s\n", __func__, + file->mpi_rank, (long)mpi_off, bytes_written, H5FD__mem_t_to_str(type)); #endif /* Each process will keep track of its perceived EOF value locally, and diff --git a/src/H5Fmpi.c b/src/H5Fmpi.c index 78290c6..02d8d52 100644 --- a/src/H5Fmpi.c +++ b/src/H5Fmpi.c @@ -524,4 +524,68 @@ H5F_set_coll_metadata_reads(H5F_t *file, H5P_coll_md_read_flag_t *file_flag, hbo FUNC_LEAVE_NOAPI_VOID } /* end H5F_set_coll_metadata_reads() */ +/*------------------------------------------------------------------------- + * Function: H5F_mpi_get_file_block_type + * + * Purpose: Creates an MPI derived datatype for communicating an + * H5F_block_t structure. If `commit` is specified as TRUE, + * the resulting datatype will be committed and ready for + * use in communication. Otherwise, the type is only suitable + * for building other derived types. + * + * If TRUE is returned through `new_type_derived`, this lets + * the caller know that the datatype has been derived and + * should be freed with MPI_Type_free once it is no longer + * needed. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_mpi_get_file_block_type(hbool_t commit, MPI_Datatype *new_type, hbool_t *new_type_derived) +{ + MPI_Datatype types[2]; + MPI_Aint displacements[2]; + int block_lengths[2]; + int field_count; + int mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + HDassert(new_type); + HDassert(new_type_derived); + + *new_type_derived = FALSE; + + field_count = 2; + HDassert(field_count == sizeof(types) / sizeof(MPI_Datatype)); + + block_lengths[0] = 1; + block_lengths[1] = 1; + displacements[0] = offsetof(H5F_block_t, offset); + displacements[1] = offsetof(H5F_block_t, length); + types[0] = HADDR_AS_MPI_TYPE; + types[1] = HSIZE_AS_MPI_TYPE; + if (MPI_SUCCESS != + (mpi_code = MPI_Type_create_struct(field_count, block_lengths, displacements, types, new_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) + *new_type_derived = TRUE; + + if (commit && MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + +done: + if (ret_value < 0) { + if (*new_type_derived) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(new_type))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + *new_type_derived = FALSE; + } + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_mpi_get_file_block_type() */ + #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index af65c9d..67e153e 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -962,7 +962,8 @@ H5_DLL MPI_Comm H5F_mpi_get_comm(const H5F_t *f); H5_DLL int H5F_shared_mpi_get_size(const H5F_shared_t *f_sh); H5_DLL int H5F_mpi_get_size(const H5F_t *f); H5_DLL herr_t H5F_mpi_retrieve_comm(hid_t loc_id, hid_t acspl_id, MPI_Comm *mpi_comm); -H5_DLL hbool_t H5F_get_coll_metadata_reads(const H5F_t *f); +H5_DLL herr_t H5F_mpi_get_file_block_type(hbool_t commit, MPI_Datatype *new_type, hbool_t *new_type_derived); +H5_DLL hbool_t H5F_get_coll_metadata_reads(const H5F_t *f); H5_DLL void H5F_set_coll_metadata_reads(H5F_t *f, H5P_coll_md_read_flag_t *file_flag, hbool_t *context_flag); #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5mpi.c b/src/H5mpi.c index aea0104..15fb785 100644 --- a/src/H5mpi.c +++ b/src/H5mpi.c @@ -549,4 +549,237 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5_mpio_create_large_type() */ +/*------------------------------------------------------------------------- + * Function: H5_mpio_gatherv_alloc + * + * Purpose: A wrapper around MPI_(All)gatherv that performs allocation + * of the receive buffer on the caller's behalf. This + * routine's parameters are as follows: + * + * `send_buf` - The buffer that data will be sent from for + * the calling MPI rank. Analogous to + * MPI_(All)gatherv's `sendbuf` parameter. + * + * `send_count` - The number of `send_type` elements in the + * send buffer. Analogous to MPI_(All)gatherv's + * `sendcount` parameter. + * + * `send_type` - The MPI Datatype of the elements in the send + * buffer. Analogous to MPI_(All)gatherv's + * `sendtype` parameter. + * + * `recv_counts` - An array containing the number of elements + * to be received from each MPI rank. + * Analogous to MPI_(All)gatherv's `recvcount` + * parameter. + * + * `displacements` - An array containing the displacements + * in the receive buffer where data from + * each MPI rank should be placed. Analogous + * to MPI_(All)gatherv's `displs` parameter. + * + * `recv_type` - The MPI Datatype of the elements in the + * receive buffer. Analogous to + * MPI_(All)gatherv's `recvtype` parameter. + * + * `allgather` - Specifies whether the gather operation to be + * performed should be MPI_Allgatherv (TRUE) or + * MPI_Gatherv (FALSE). + * + * `root` - For MPI_Gatherv operations, specifies the rank + * that will receive the data sent by other ranks. + * Analogous to MPI_Gatherv's `root` parameter. For + * MPI_Allgatherv operations, this parameter is + * ignored. + * + * `comm` - Specifies the MPI Communicator for the operation. + * Analogous to MPI_(All)gatherv's `comm` parameter. + * + * `mpi_rank` - Specifies the calling rank's rank value, as + * obtained by calling MPI_Comm_rank on the + * MPI Communicator `comm`. + * + * `mpi_size` - Specifies the MPI Communicator size, as + * obtained by calling MPI_Comm_size on the + * MPI Communicator `comm`. + * + * `out_buf` - Resulting buffer that is allocated and + * returned to the caller after data has been + * gathered into it. Returned only to the rank + * specified by `root` for MPI_Gatherv + * operations, or to all ranks for + * MPI_Allgatherv operations. + * + * `out_buf_num_entries` - The number of elements in the + * resulting buffer, in terms of + * the MPI Datatype provided for + * `recv_type`. + * + * Notes: This routine is collective across `comm`. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5_mpio_gatherv_alloc(void *send_buf, int send_count, MPI_Datatype send_type, const int recv_counts[], + const int displacements[], MPI_Datatype recv_type, hbool_t allgather, int root, + MPI_Comm comm, int mpi_rank, int mpi_size, void **out_buf, size_t *out_buf_num_entries) +{ + size_t recv_buf_num_entries = 0; + void * recv_buf = NULL; +#if MPI_VERSION >= 3 + MPI_Count type_lb; + MPI_Count type_extent; +#else + MPI_Aint type_lb; + MPI_Aint type_extent; +#endif + int mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + HDassert(send_buf || send_count == 0); + if (allgather || (mpi_rank == root)) + HDassert(out_buf && out_buf_num_entries); + + /* Retrieve the extent of the MPI Datatype being used */ +#if MPI_VERSION >= 3 + if (MPI_SUCCESS != (mpi_code = MPI_Type_get_extent_x(recv_type, &type_lb, &type_extent))) +#else + if (MPI_SUCCESS != (mpi_code = MPI_Type_get_extent(recv_type, &type_lb, &type_extent))) +#endif + HMPI_GOTO_ERROR(FAIL, "MPI_Type_get_extent(_x) failed", mpi_code) + + if (type_extent < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "MPI recv_type had a negative extent") + + /* + * Calculate the total size of the buffer being + * returned and allocate it + */ + if (allgather || (mpi_rank == root)) { + size_t i; + size_t buf_size; + + for (i = 0, recv_buf_num_entries = 0; i < (size_t)mpi_size; i++) + recv_buf_num_entries += (size_t)recv_counts[i]; + buf_size = recv_buf_num_entries * (size_t)type_extent; + + /* If our buffer size is 0, there's nothing to do */ + if (buf_size == 0) + HGOTO_DONE(SUCCEED) + + if (NULL == (recv_buf = H5MM_malloc(buf_size))) + /* Push an error, but still participate in collective gather operation */ + HDONE_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "couldn't allocate receive buffer") + } + + /* Perform gather operation */ + if (allgather) { + if (MPI_SUCCESS != (mpi_code = MPI_Allgatherv(send_buf, send_count, send_type, recv_buf, recv_counts, + displacements, recv_type, comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Allgatherv failed", mpi_code) + } + else { + if (MPI_SUCCESS != (mpi_code = MPI_Gatherv(send_buf, send_count, send_type, recv_buf, recv_counts, + displacements, recv_type, root, comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Gatherv failed", mpi_code) + } + + if (allgather || (mpi_rank == root)) { + *out_buf = recv_buf; + *out_buf_num_entries = recv_buf_num_entries; + } + +done: + if (ret_value < 0) { + if (recv_buf) + H5MM_free(recv_buf); + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5_mpio_gatherv_alloc() */ + +/*------------------------------------------------------------------------- + * Function: H5_mpio_gatherv_alloc_simple + * + * Purpose: A slightly simplified interface to H5_mpio_gatherv_alloc + * which calculates the receive counts and receive buffer + * displacements for the caller. + * + * Notes: This routine is collective across `comm`. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5_mpio_gatherv_alloc_simple(void *send_buf, int send_count, MPI_Datatype send_type, MPI_Datatype recv_type, + hbool_t allgather, int root, MPI_Comm comm, int mpi_rank, int mpi_size, + void **out_buf, size_t *out_buf_num_entries) +{ + int * recv_counts_disps_array = NULL; + int mpi_code; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + HDassert(send_buf || send_count == 0); + if (allgather || (mpi_rank == root)) + HDassert(out_buf && out_buf_num_entries); + + /* + * Allocate array to store the receive counts of each rank, as well as + * the displacements into the final array where each rank will place + * their data. The first half of the array contains the receive counts + * (in rank order), while the latter half contains the displacements + * (also in rank order). + */ + if (allgather || (mpi_rank == root)) { + if (NULL == + (recv_counts_disps_array = H5MM_malloc(2 * (size_t)mpi_size * sizeof(*recv_counts_disps_array)))) + /* Push an error, but still participate in collective gather operation */ + HDONE_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, + "couldn't allocate receive counts and displacements array") + } + + /* Collect each rank's send count to interested ranks */ + if (allgather) { + if (MPI_SUCCESS != + (mpi_code = MPI_Allgather(&send_count, 1, MPI_INT, recv_counts_disps_array, 1, MPI_INT, comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Allgather failed", mpi_code) + } + else { + if (MPI_SUCCESS != + (mpi_code = MPI_Gather(&send_count, 1, MPI_INT, recv_counts_disps_array, 1, MPI_INT, root, comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Gather failed", mpi_code) + } + + /* Set the displacements into the receive buffer for the gather operation */ + if (allgather || (mpi_rank == root)) { + size_t i; + int * displacements_ptr; + + displacements_ptr = &recv_counts_disps_array[mpi_size]; + + *displacements_ptr = 0; + for (i = 1; i < (size_t)mpi_size; i++) + displacements_ptr[i] = displacements_ptr[i - 1] + recv_counts_disps_array[i - 1]; + } + + /* Perform gather operation */ + if (H5_mpio_gatherv_alloc(send_buf, send_count, send_type, recv_counts_disps_array, + &recv_counts_disps_array[mpi_size], recv_type, allgather, root, comm, mpi_rank, + mpi_size, out_buf, out_buf_num_entries) < 0) + HGOTO_ERROR(H5E_LIB, H5E_CANTGATHER, FAIL, "can't gather data") + +done: + if (recv_counts_disps_array) + H5MM_free(recv_counts_disps_array); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5_mpio_gatherv_alloc_simple() */ + #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5private.h b/src/H5private.h index 68aabc2..d67163f 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -387,6 +387,25 @@ #define HSSIZET_MAX ((hssize_t)LLONG_MAX) #define HSSIZET_MIN (~(HSSIZET_MAX)) +#ifdef H5_HAVE_PARALLEL + +/* Define a type for safely sending size_t values with MPI */ +#if SIZE_MAX == UCHAR_MAX +#define H5_SIZE_T_AS_MPI_TYPE MPI_UNSIGNED_CHAR +#elif SIZE_MAX == USHRT_MAX +#define H5_SIZE_T_AS_MPI_TYPE MPI_UNSIGNED_SHORT +#elif SIZE_MAX == UINT_MAX +#define H5_SIZE_T_AS_MPI_TYPE MPI_UNSIGNED +#elif SIZE_MAX == ULONG_MAX +#define H5_SIZE_T_AS_MPI_TYPE MPI_UNSIGNED_LONG +#elif SIZE_MAX == ULLONG_MAX +#define H5_SIZE_T_AS_MPI_TYPE MPI_UNSIGNED_LONG_LONG +#else +#error "no suitable MPI type for size_t" +#endif + +#endif /* H5_HAVE_PARALLEL */ + /* * Types and max sizes for POSIX I/O. * OS X (Darwin) is odd since the max I/O size does not match the types. @@ -508,6 +527,9 @@ #define H5_GCC_CLANG_DIAG_ON(x) #endif +/* Function pointer typedef for qsort */ +typedef int (*H5_sort_func_cb_t)(const void *, const void *); + /* Typedefs and functions for timing certain parts of the library. */ /* A set of elapsed/user/system times emitted as a time point by the @@ -2617,6 +2639,14 @@ H5_DLL herr_t H5_mpi_comm_cmp(MPI_Comm comm1, MPI_Comm comm2, int *result); H5_DLL herr_t H5_mpi_info_cmp(MPI_Info info1, MPI_Info info2, int *result); H5_DLL herr_t H5_mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes, MPI_Datatype old_type, MPI_Datatype *new_type); +H5_DLL herr_t H5_mpio_gatherv_alloc(void *send_buf, int send_count, MPI_Datatype send_type, + const int recv_counts[], const int displacements[], + MPI_Datatype recv_type, hbool_t allgather, int root, MPI_Comm comm, + int mpi_rank, int mpi_size, void **out_buf, size_t *out_buf_num_entries); +H5_DLL herr_t H5_mpio_gatherv_alloc_simple(void *send_buf, int send_count, MPI_Datatype send_type, + MPI_Datatype recv_type, hbool_t allgather, int root, MPI_Comm comm, + int mpi_rank, int mpi_size, void **out_buf, + size_t *out_buf_num_entries); #endif /* H5_HAVE_PARALLEL */ /* Functions for debugging */ diff --git a/src/H5public.h b/src/H5public.h index 6a3911c..037501b 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -289,6 +289,11 @@ typedef long long ssize_t; * \internal Defined as a (minimum) 64-bit integer type. */ typedef uint64_t hsize_t; + +#ifdef H5_HAVE_PARALLEL +#define HSIZE_AS_MPI_TYPE MPI_UINT64_T +#endif + /** * The size of file objects. Used when negative values are needed to indicate errors. * @@ -323,7 +328,7 @@ typedef uint64_t haddr_t; #define HADDR_MAX (HADDR_UNDEF - 1) #ifdef H5_HAVE_PARALLEL -#define HADDR_AS_MPI_TYPE MPI_LONG_LONG_INT +#define HADDR_AS_MPI_TYPE MPI_UINT64_T #endif //! diff --git a/testpar/t_2Gio.c b/testpar/t_2Gio.c index 2be4ae4..911be2c 100644 --- a/testpar/t_2Gio.c +++ b/testpar/t_2Gio.c @@ -3047,7 +3047,7 @@ compress_readAll(void) nerrors++; } -#if MPI_VERSION >= 3 +#ifdef H5_HAVE_PARALLEL_FILTERED_WRITES ret = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, xfer_plist, data_read); VRFY((ret >= 0), "H5Dwrite succeeded"); #endif @@ -3853,12 +3853,6 @@ actual_io_mode_tests(void) * TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_EXTERNAL: * Test for Externl-File storage as the cause of breaking collective I/O. * - * TEST_FILTERS: - * Test for using filter (checksum) as the cause of breaking collective I/O. - * Note: TEST_FILTERS mode will not work until H5Dcreate and H5write is supported for mpio and filter - * feature. Use test_no_collective_cause_mode_filter() function instead. - * - * * Programmer: Jonathan Kim * Date: Aug, 2012 */ @@ -3898,9 +3892,6 @@ test_no_collective_cause_mode(int selection_mode) hid_t file_space = -1; hsize_t chunk_dims[MAX_RANK]; herr_t ret; -#ifdef LATER /* fletcher32 */ - H5Z_filter_t filter_info; -#endif /* LATER */ /* set to global value as default */ int l_facc_type = facc_type; char message[256]; @@ -3932,21 +3923,6 @@ test_no_collective_cause_mode(int selection_mode) is_chunked = 0; } -#ifdef LATER /* fletcher32 */ - if (selection_mode & TEST_FILTERS) { - ret = H5Zfilter_avail(H5Z_FILTER_FLETCHER32); - VRFY((ret >= 0), "Fletcher32 filter is available.\n"); - - ret = H5Zget_filter_info(H5Z_FILTER_FLETCHER32, &filter_info); - VRFY(((filter_info & H5Z_FILTER_CONFIG_ENCODE_ENABLED) || - (filter_info & H5Z_FILTER_CONFIG_DECODE_ENABLED)), - "Fletcher32 filter encoding and decoding available.\n"); - - ret = H5Pset_fletcher32(dcpl); - VRFY((ret >= 0), "set filter (flecher32) succeeded"); - } -#endif /* LATER */ - if (selection_mode & TEST_NOT_SIMPLE_OR_SCALAR_DATASPACES) { sid = H5Screate(H5S_NULL); VRFY((sid >= 0), "H5Screate_simple succeeded"); @@ -4022,14 +3998,6 @@ test_no_collective_cause_mode(int selection_mode) no_collective_cause_global_expected |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; } -#ifdef LATER /* fletcher32 */ - if (selection_mode & TEST_FILTERS) { - test_name = "Broken Collective I/O - Filter is required"; - no_collective_cause_local_expected |= H5D_MPIO_FILTERS; - no_collective_cause_global_expected |= H5D_MPIO_FILTERS; - } -#endif /* LATER */ - if (selection_mode & TEST_COLLECTIVE) { test_name = "Broken Collective I/O - Not Broken"; no_collective_cause_local_expected = H5D_MPIO_COLLECTIVE; @@ -4166,240 +4134,6 @@ test_no_collective_cause_mode(int selection_mode) return; } -#if 0 -/* - * Function: test_no_collective_cause_mode_filter - * - * Purpose: - * Test specific for using filter as a caus of broken collective I/O and - * checks that the H5Pget_mpio_no_collective_cause properties in the DXPL - * have the correct values. - * - * NOTE: - * This is a temporary function. - * test_no_collective_cause_mode(TEST_FILTERS) will replace this when - * H5Dcreate and H5write support for mpio and filter feature. - * - * Input: - * TEST_FILTERS_READ: - * Test for using filter (checksum) as the cause of breaking collective I/O. - * - * Programmer: Jonathan Kim - * Date: Aug, 2012 - */ -static void -test_no_collective_cause_mode_filter(int selection_mode) -{ - uint32_t no_collective_cause_local_read = 0; - uint32_t no_collective_cause_local_expected = 0; - uint32_t no_collective_cause_global_read = 0; - uint32_t no_collective_cause_global_expected = 0; - - const char * filename; - const char * test_name; - hbool_t is_chunked=1; - int mpi_size = -1; - int mpi_rank = -1; - int length; - int * buffer; - int i; - MPI_Comm mpi_comm = MPI_COMM_NULL; - MPI_Info mpi_info = MPI_INFO_NULL; - hid_t fid = -1; - hid_t sid = -1; - hid_t dataset = -1; - hid_t data_type = H5T_NATIVE_INT; - hid_t fapl_write = -1; - hid_t fapl_read = -1; - hid_t dcpl = -1; - hid_t dxpl = -1; - hsize_t dims[MAX_RANK]; - hid_t mem_space = -1; - hid_t file_space = -1; - hsize_t chunk_dims[MAX_RANK]; - herr_t ret; -#ifdef LATER /* fletcher32 */ - H5Z_filter_t filter_info; -#endif /* LATER */ - char message[256]; - - /* Set up MPI parameters */ - MPI_Comm_size(test_comm, &mpi_size); - MPI_Comm_rank(test_comm, &mpi_rank); - - MPI_Barrier(test_comm); - - HDassert(mpi_size >= 1); - - mpi_comm = test_comm; - mpi_info = MPI_INFO_NULL; - - /* Create the dataset creation plist */ - dcpl = H5Pcreate(H5P_DATASET_CREATE); - VRFY((dcpl >= 0), "dataset creation plist created successfully"); - - if (selection_mode == TEST_FILTERS_READ ) { -#ifdef LATER /* fletcher32 */ - ret = H5Zfilter_avail(H5Z_FILTER_FLETCHER32); - VRFY ((ret >=0 ), "Fletcher32 filter is available.\n"); - - ret = H5Zget_filter_info (H5Z_FILTER_FLETCHER32, (unsigned int *) &filter_info); - VRFY ( ( (filter_info & H5Z_FILTER_CONFIG_ENCODE_ENABLED) || (filter_info & H5Z_FILTER_CONFIG_DECODE_ENABLED) ) , "Fletcher32 filter encoding and decoding available.\n"); - - ret = H5Pset_fletcher32(dcpl); - VRFY((ret >= 0),"set filter (flecher32) succeeded"); -#endif /* LATER */ - } - else { - VRFY(0, "Unexpected mode, only test for TEST_FILTERS_READ."); - } - - /* Create the basic Space */ - dims[0] = dim0; - dims[1] = dim1; - sid = H5Screate_simple (MAX_RANK, dims, NULL); - VRFY((sid >= 0), "H5Screate_simple succeeded"); - - - filename = (const char *)GetTestParameters(); - HDassert(filename != NULL); - - /* Setup the file access template */ - fapl_write = create_faccess_plist(mpi_comm, mpi_info, FACC_DEFAULT); - VRFY((fapl_write >= 0), "create_faccess_plist() succeeded"); - - fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_write); - VRFY((fid >= 0), "H5Fcreate succeeded"); - - /* If we are not testing contiguous datasets */ - if(is_chunked) { - /* Set up chunk information. */ - chunk_dims[0] = dims[0]/mpi_size; - chunk_dims[1] = dims[1]; - ret = H5Pset_chunk(dcpl, 2, chunk_dims); - VRFY((ret >= 0),"chunk creation property list succeeded"); - } - - - /* Create the dataset */ - dataset = H5Dcreate2(fid, DSET_NOCOLCAUSE, data_type, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); - VRFY((dataset >= 0), "H5Dcreate2() dataset succeeded"); - -#ifdef LATER /* fletcher32 */ - /* Set expected cause */ - test_name = "Broken Collective I/O - Filter is required"; - no_collective_cause_local_expected = H5D_MPIO_FILTERS; - no_collective_cause_global_expected = H5D_MPIO_FILTERS; -#endif /* LATER */ - - /* Get the file dataspace */ - file_space = H5Dget_space(dataset); - VRFY((file_space >= 0), "H5Dget_space succeeded"); - - /* Create the memory dataspace */ - mem_space = H5Screate_simple (MAX_RANK, dims, NULL); - VRFY((mem_space >= 0), "mem_space created"); - - /* Get the number of elements in the selection */ - length = dim0 * dim1; - - /* Allocate and initialize the buffer */ - buffer = (int *)HDmalloc(sizeof(int) * length); - VRFY((buffer != NULL), "HDmalloc of buffer succeeded"); - for(i = 0; i < length; i++) - buffer[i] = i; - - /* Set up the dxpl for the write */ - dxpl = H5Pcreate(H5P_DATASET_XFER); - VRFY((dxpl >= 0), "H5Pcreate(H5P_DATASET_XFER) succeeded"); - - if (selection_mode == TEST_FILTERS_READ) { - /* To test read in collective I/O mode , write in independent mode - * because write fails with mpio + filter */ - ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_INDEPENDENT); - VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); - } - else { - /* To test write in collective I/O mode. */ - ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE); - VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); - } - - - /* Write */ - ret = H5Dwrite(dataset, data_type, mem_space, file_space, dxpl, buffer); - - if(ret < 0) H5Eprint2(H5E_DEFAULT, stdout); - VRFY((ret >= 0), "H5Dwrite() dataset multichunk write succeeded"); - - - /* Make a copy of the dxpl to test the read operation */ - dxpl = H5Pcopy(dxpl); - VRFY((dxpl >= 0), "H5Pcopy succeeded"); - - if (dataset) - H5Dclose(dataset); - if (fapl_write) - H5Pclose(fapl_write); - if (fid) - H5Fclose(fid); - - - /*--------------------- - * Test Read access - *---------------------*/ - - /* Setup the file access template */ - fapl_read = create_faccess_plist(mpi_comm, mpi_info, facc_type); - VRFY((fapl_read >= 0), "create_faccess_plist() succeeded"); - - fid = H5Fopen (filename, H5F_ACC_RDONLY, fapl_read); - dataset = H5Dopen2 (fid, DSET_NOCOLCAUSE, H5P_DEFAULT); - - /* Set collective I/O properties in the dxpl. */ - ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE); - VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); - - /* Read */ - ret = H5Dread(dataset, data_type, mem_space, file_space, dxpl, buffer); - - if(ret < 0) H5Eprint2(H5E_DEFAULT, stdout); - VRFY((ret >= 0), "H5Dread() dataset multichunk read succeeded"); - - /* Get the cause of broken collective I/O */ - ret = H5Pget_mpio_no_collective_cause (dxpl, &no_collective_cause_local_read, &no_collective_cause_global_read); - VRFY((ret >= 0), "retrieving no collective cause succeeded" ); - - /* Test values */ - HDmemset (message, 0, sizeof (message)); - HDsprintf(message, "Local cause of Broken Collective I/O has the correct value for %s.\n",test_name); - VRFY((no_collective_cause_local_read == (uint32_t)no_collective_cause_local_expected), message); - HDmemset (message, 0, sizeof (message)); - HDsprintf(message, "Global cause of Broken Collective I/O has the correct value for %s.\n",test_name); - VRFY((no_collective_cause_global_read == (uint32_t)no_collective_cause_global_expected), message); - - /* Release some resources */ - if (sid) - H5Sclose(sid); - if (fapl_read) - H5Pclose(fapl_read); - if (dcpl) - H5Pclose(dcpl); - if (dxpl) - H5Pclose(dxpl); - if (dataset) - H5Dclose(dataset); - if (mem_space) - H5Sclose(mem_space); - if (file_space) - H5Sclose(file_space); - if (fid) - H5Fclose(fid); - HDfree(buffer); - return; -} -#endif - /* Function: no_collective_cause_tests * * Purpose: Tests cases for broken collective IO. @@ -4420,13 +4154,6 @@ no_collective_cause_tests(void) test_no_collective_cause_mode(TEST_NOT_SIMPLE_OR_SCALAR_DATASPACES); test_no_collective_cause_mode(TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_COMPACT); test_no_collective_cause_mode(TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_EXTERNAL); -#ifdef LATER /* fletcher32 */ - /* TODO: use this instead of below TEST_FILTERS_READ when H5Dcreate and - * H5Dwrite is ready for mpio + filter feature. - */ - /* test_no_collective_cause_mode (TEST_FILTERS); */ - test_no_collective_cause_mode_filter(TEST_FILTERS_READ); -#endif /* LATER */ /* * Test combined causes diff --git a/testpar/t_dset.c b/testpar/t_dset.c index 2aade32..8616bef 100644 --- a/testpar/t_dset.c +++ b/testpar/t_dset.c @@ -2605,7 +2605,7 @@ compress_readAll(void) nerrors++; } -#if MPI_VERSION >= 3 +#ifdef H5_HAVE_PARALLEL_FILTERED_WRITES ret = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, xfer_plist, data_read); VRFY((ret >= 0), "H5Dwrite succeeded"); #endif @@ -3418,12 +3418,6 @@ actual_io_mode_tests(void) * TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_EXTERNAL: * Test for Externl-File storage as the cause of breaking collective I/O. * - * TEST_FILTERS: - * Test for using filter (checksum) as the cause of breaking collective I/O. - * Note: TEST_FILTERS mode will not work until H5Dcreate and H5write is supported for mpio and filter - * feature. Use test_no_collective_cause_mode_filter() function instead. - * - * * Programmer: Jonathan Kim * Date: Aug, 2012 */ @@ -3465,9 +3459,6 @@ test_no_collective_cause_mode(int selection_mode) hid_t file_space = -1; hsize_t chunk_dims[RANK]; herr_t ret; -#ifdef LATER /* fletcher32 */ - H5Z_filter_t filter_info; -#endif /* LATER */ /* set to global value as default */ int l_facc_type = facc_type; char message[256]; @@ -3499,21 +3490,6 @@ test_no_collective_cause_mode(int selection_mode) is_chunked = 0; } -#ifdef LATER /* fletcher32 */ - if (selection_mode & TEST_FILTERS) { - ret = H5Zfilter_avail(H5Z_FILTER_FLETCHER32); - VRFY((ret >= 0), "Fletcher32 filter is available.\n"); - - ret = H5Zget_filter_info(H5Z_FILTER_FLETCHER32, &filter_info); - VRFY(((filter_info & H5Z_FILTER_CONFIG_ENCODE_ENABLED) || - (filter_info & H5Z_FILTER_CONFIG_DECODE_ENABLED)), - "Fletcher32 filter encoding and decoding available.\n"); - - ret = H5Pset_fletcher32(dcpl); - VRFY((ret >= 0), "set filter (flecher32) succeeded"); - } -#endif /* LATER */ - if (selection_mode & TEST_NOT_SIMPLE_OR_SCALAR_DATASPACES) { sid = H5Screate(H5S_NULL); VRFY((sid >= 0), "H5Screate_simple succeeded"); @@ -3589,14 +3565,6 @@ test_no_collective_cause_mode(int selection_mode) no_collective_cause_global_expected |= H5D_MPIO_NOT_CONTIGUOUS_OR_CHUNKED_DATASET; } -#ifdef LATER /* fletcher32 */ - if (selection_mode & TEST_FILTERS) { - test_name = "Broken Collective I/O - Filter is required"; - no_collective_cause_local_expected |= H5D_MPIO_FILTERS; - no_collective_cause_global_expected |= H5D_MPIO_FILTERS; - } -#endif /* LATER */ - if (selection_mode & TEST_COLLECTIVE) { test_name = "Broken Collective I/O - Not Broken"; no_collective_cause_local_expected = H5D_MPIO_COLLECTIVE; @@ -3735,242 +3703,6 @@ test_no_collective_cause_mode(int selection_mode) return; } -/* - * Function: test_no_collective_cause_mode_filter - * - * Purpose: - * Test specific for using filter as a caus of broken collective I/O and - * checks that the H5Pget_mpio_no_collective_cause properties in the DXPL - * have the correct values. - * - * NOTE: - * This is a temporary function. - * test_no_collective_cause_mode(TEST_FILTERS) will replace this when - * H5Dcreate and H5write support for mpio and filter feature. - * - * Input: - * TEST_FILTERS_READ: - * Test for using filter (checksum) as the cause of breaking collective I/O. - * - * Programmer: Jonathan Kim - * Date: Aug, 2012 - */ -#ifdef LATER -static void -test_no_collective_cause_mode_filter(int selection_mode) -{ - uint32_t no_collective_cause_local_read = 0; - uint32_t no_collective_cause_local_expected = 0; - uint32_t no_collective_cause_global_read = 0; - uint32_t no_collective_cause_global_expected = 0; - - const char *filename; - const char *test_name = "I/O"; - hbool_t is_chunked = 1; - int mpi_size = -1; - int mpi_rank = -1; - int length; - int * buffer; - int i; - MPI_Comm mpi_comm = MPI_COMM_NULL; - MPI_Info mpi_info = MPI_INFO_NULL; - hid_t fid = -1; - hid_t sid = -1; - hid_t dataset = -1; - hid_t data_type = H5T_NATIVE_INT; - hid_t fapl_write = -1; - hid_t fapl_read = -1; - hid_t dcpl = -1; - hid_t dxpl = -1; - hsize_t dims[RANK]; - hid_t mem_space = -1; - hid_t file_space = -1; - hsize_t chunk_dims[RANK]; - herr_t ret; -#ifdef LATER /* fletcher32 */ - H5Z_filter_t filter_info; -#endif /* LATER */ - char message[256]; - - /* Set up MPI parameters */ - MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); - MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); - - MPI_Barrier(MPI_COMM_WORLD); - - HDassert(mpi_size >= 1); - - mpi_comm = MPI_COMM_WORLD; - mpi_info = MPI_INFO_NULL; - - /* Create the dataset creation plist */ - dcpl = H5Pcreate(H5P_DATASET_CREATE); - VRFY((dcpl >= 0), "dataset creation plist created successfully"); - - if (selection_mode == TEST_FILTERS_READ) { -#ifdef LATER /* fletcher32 */ - ret = H5Zfilter_avail(H5Z_FILTER_FLETCHER32); - VRFY((ret >= 0), "Fletcher32 filter is available.\n"); - - ret = H5Zget_filter_info(H5Z_FILTER_FLETCHER32, (unsigned int *)&filter_info); - VRFY(((filter_info & H5Z_FILTER_CONFIG_ENCODE_ENABLED) || - (filter_info & H5Z_FILTER_CONFIG_DECODE_ENABLED)), - "Fletcher32 filter encoding and decoding available.\n"); - - ret = H5Pset_fletcher32(dcpl); - VRFY((ret >= 0), "set filter (flecher32) succeeded"); -#endif /* LATER */ - } - else { - VRFY(0, "Unexpected mode, only test for TEST_FILTERS_READ."); - } - - /* Create the basic Space */ - dims[0] = (hsize_t)dim0; - dims[1] = (hsize_t)dim1; - sid = H5Screate_simple(RANK, dims, NULL); - VRFY((sid >= 0), "H5Screate_simple succeeded"); - - filename = (const char *)GetTestParameters(); - HDassert(filename != NULL); - - /* Setup the file access template */ - fapl_write = create_faccess_plist(mpi_comm, mpi_info, FACC_DEFAULT); - VRFY((fapl_write >= 0), "create_faccess_plist() succeeded"); - - fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_write); - VRFY((fid >= 0), "H5Fcreate succeeded"); - - /* If we are not testing contiguous datasets */ - if (is_chunked) { - /* Set up chunk information. */ - chunk_dims[0] = dims[0] / (hsize_t)mpi_size; - chunk_dims[1] = dims[1]; - ret = H5Pset_chunk(dcpl, 2, chunk_dims); - VRFY((ret >= 0), "chunk creation property list succeeded"); - } - - /* Create the dataset */ - dataset = H5Dcreate2(fid, DSET_NOCOLCAUSE, data_type, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); - VRFY((dataset >= 0), "H5Dcreate2() dataset succeeded"); - -#ifdef LATER /* fletcher32 */ - /* Set expected cause */ - test_name = "Broken Collective I/O - Filter is required"; - no_collective_cause_local_expected = H5D_MPIO_FILTERS; - no_collective_cause_global_expected = H5D_MPIO_FILTERS; -#endif /* LATER */ - - /* Get the file dataspace */ - file_space = H5Dget_space(dataset); - VRFY((file_space >= 0), "H5Dget_space succeeded"); - - /* Create the memory dataspace */ - mem_space = H5Screate_simple(RANK, dims, NULL); - VRFY((mem_space >= 0), "mem_space created"); - - /* Get the number of elements in the selection */ - length = dim0 * dim1; - - /* Allocate and initialize the buffer */ - buffer = (int *)HDmalloc(sizeof(int) * length); - VRFY((buffer != NULL), "HDmalloc of buffer succeeded"); - for (i = 0; i < length; i++) - buffer[i] = i; - - /* Set up the dxpl for the write */ - dxpl = H5Pcreate(H5P_DATASET_XFER); - VRFY((dxpl >= 0), "H5Pcreate(H5P_DATASET_XFER) succeeded"); - - if (selection_mode == TEST_FILTERS_READ) { - /* To test read in collective I/O mode , write in independent mode - * because write fails with mpio + filter */ - ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_INDEPENDENT); - VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); - } - else { - /* To test write in collective I/O mode. */ - ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE); - VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); - } - - /* Write */ - ret = H5Dwrite(dataset, data_type, mem_space, file_space, dxpl, buffer); - - if (ret < 0) - H5Eprint2(H5E_DEFAULT, stdout); - VRFY((ret >= 0), "H5Dwrite() dataset multichunk write succeeded"); - - /* Make a copy of the dxpl to test the read operation */ - dxpl = H5Pcopy(dxpl); - VRFY((dxpl >= 0), "H5Pcopy succeeded"); - - if (dataset) - H5Dclose(dataset); - if (fapl_write) - H5Pclose(fapl_write); - if (fid) - H5Fclose(fid); - - /*--------------------- - * Test Read access - *---------------------*/ - - /* Setup the file access template */ - fapl_read = create_faccess_plist(mpi_comm, mpi_info, facc_type); - VRFY((fapl_read >= 0), "create_faccess_plist() succeeded"); - - fid = H5Fopen(filename, H5F_ACC_RDONLY, fapl_read); - dataset = H5Dopen2(fid, DSET_NOCOLCAUSE, H5P_DEFAULT); - - /* Set collective I/O properties in the dxpl. */ - ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE); - VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); - - /* Read */ - ret = H5Dread(dataset, data_type, mem_space, file_space, dxpl, buffer); - - if (ret < 0) - H5Eprint2(H5E_DEFAULT, stdout); - VRFY((ret >= 0), "H5Dread() dataset multichunk read succeeded"); - - /* Get the cause of broken collective I/O */ - ret = H5Pget_mpio_no_collective_cause(dxpl, &no_collective_cause_local_read, - &no_collective_cause_global_read); - VRFY((ret >= 0), "retrieving no collective cause succeeded"); - - /* Test values */ - HDmemset(message, 0, sizeof(message)); - HDsnprintf(message, sizeof(message), - "Local cause of Broken Collective I/O has the correct value for %s.\n", test_name); - VRFY((no_collective_cause_local_read == (uint32_t)no_collective_cause_local_expected), message); - HDmemset(message, 0, sizeof(message)); - HDsnprintf(message, sizeof(message), - "Global cause of Broken Collective I/O has the correct value for %s.\n", test_name); - VRFY((no_collective_cause_global_read == (uint32_t)no_collective_cause_global_expected), message); - - /* Release some resources */ - if (sid) - H5Sclose(sid); - if (fapl_read) - H5Pclose(fapl_read); - if (dcpl) - H5Pclose(dcpl); - if (dxpl) - H5Pclose(dxpl); - if (dataset) - H5Dclose(dataset); - if (mem_space) - H5Sclose(mem_space); - if (file_space) - H5Sclose(file_space); - if (fid) - H5Fclose(fid); - HDfree(buffer); - return; -} -#endif - /* Function: no_collective_cause_tests * * Purpose: Tests cases for broken collective IO. @@ -3991,13 +3723,6 @@ no_collective_cause_tests(void) test_no_collective_cause_mode(TEST_NOT_SIMPLE_OR_SCALAR_DATASPACES); test_no_collective_cause_mode(TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_COMPACT); test_no_collective_cause_mode(TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_EXTERNAL); -#ifdef LATER /* fletcher32 */ - /* TODO: use this instead of below TEST_FILTERS_READ when H5Dcreate and - * H5Dwrite is ready for mpio + filter feature. - */ - /* test_no_collective_cause_mode (TEST_FILTERS); */ - test_no_collective_cause_mode_filter(TEST_FILTERS_READ); -#endif /* LATER */ /* * Test combined causes diff --git a/testpar/t_filters_parallel.c b/testpar/t_filters_parallel.c index 78af0fb..8a55519 100644 --- a/testpar/t_filters_parallel.c +++ b/testpar/t_filters_parallel.c @@ -26,73 +26,139 @@ const char *FILENAME[] = {"t_filters_parallel", NULL}; char filenames[1][256]; +static MPI_Comm comm = MPI_COMM_WORLD; +static MPI_Info info = MPI_INFO_NULL; +static int mpi_rank; +static int mpi_size; + int nerrors = 0; -size_t cur_filter_idx = 0; -#define GZIP_INDEX 0 -#define FLETCHER32_INDEX 1 +/* Arrays of filter ID values and filter names (should match each other) */ +H5Z_filter_t filterIDs[] = { + H5Z_FILTER_DEFLATE, H5Z_FILTER_SHUFFLE, H5Z_FILTER_FLETCHER32, + H5Z_FILTER_SZIP, H5Z_FILTER_NBIT, H5Z_FILTER_SCALEOFFSET, +}; + +const char *filterNames[] = {"Deflate", "Shuffle", "Fletcher32", "SZIP", "Nbit", "ScaleOffset"}; + +/* Function pointer typedef for test functions */ +typedef void (*test_func)(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, hid_t dcpl_id, + hid_t dxpl_id); -#define ARRAY_SIZE(a) sizeof(a) / sizeof(a[0]) +/* Typedef for filter arguments for user-defined filters */ +typedef struct filter_options_t { + unsigned int flags; + size_t cd_nelmts; + const unsigned int cd_values[]; +} filter_options_t; /* - * Used to check if a filter is available before running a test. + * Enum for verify_space_alloc_status which specifies + * how many chunks have been written to in a dataset */ -#define CHECK_CUR_FILTER_AVAIL() \ - { \ - htri_t filter_is_avail; \ - \ - if (cur_filter_idx == GZIP_INDEX) { \ - if ((filter_is_avail = H5Zfilter_avail(H5Z_FILTER_DEFLATE)) != TRUE) { \ - if (MAINPROCESS) { \ - HDputs(" - SKIPPED - Deflate filter not available"); \ - } \ - return; \ - } \ - } \ - } +typedef enum num_chunks_written_t { + DATASET_JUST_CREATED, + NO_CHUNKS_WRITTEN, + SOME_CHUNKS_WRITTEN, + ALL_CHUNKS_WRITTEN +} num_chunks_written_t; -static herr_t set_dcpl_filter(hid_t dcpl); +static herr_t set_dcpl_filter(hid_t dcpl_id, H5Z_filter_t filter_id, filter_options_t *filter_options); +static herr_t verify_space_alloc_status(hid_t dset_id, hid_t dcpl_id, num_chunks_written_t chunks_written); -#if MPI_VERSION >= 3 +#ifdef H5_HAVE_PARALLEL_FILTERED_WRITES /* Tests for writing data in parallel */ -static void test_write_one_chunk_filtered_dataset(void); -static void test_write_filtered_dataset_no_overlap(void); -static void test_write_filtered_dataset_overlap(void); -static void test_write_filtered_dataset_single_no_selection(void); -static void test_write_filtered_dataset_all_no_selection(void); -static void test_write_filtered_dataset_point_selection(void); -static void test_write_filtered_dataset_interleaved_write(void); -static void test_write_transformed_filtered_dataset_no_overlap(void); -static void test_write_3d_filtered_dataset_no_overlap_separate_pages(void); -static void test_write_3d_filtered_dataset_no_overlap_same_pages(void); -static void test_write_3d_filtered_dataset_overlap(void); -static void test_write_cmpd_filtered_dataset_no_conversion_unshared(void); -static void test_write_cmpd_filtered_dataset_no_conversion_shared(void); -static void test_write_cmpd_filtered_dataset_type_conversion_unshared(void); -static void test_write_cmpd_filtered_dataset_type_conversion_shared(void); +static void test_write_one_chunk_filtered_dataset(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_no_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_no_overlap_partial(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_single_unlim_dim_no_overlap(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_single_unlim_dim_overlap(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_multi_unlim_dim_no_overlap(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_multi_unlim_dim_overlap(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_single_no_selection(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_all_no_selection(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_point_selection(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_write_filtered_dataset_interleaved_write(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_write_transformed_filtered_dataset_no_overlap(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_3d_filtered_dataset_no_overlap_separate_pages(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_3d_filtered_dataset_no_overlap_same_pages(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_3d_filtered_dataset_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_write_cmpd_filtered_dataset_no_conversion_unshared(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_cmpd_filtered_dataset_no_conversion_shared(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_cmpd_filtered_dataset_type_conversion_unshared(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_write_cmpd_filtered_dataset_type_conversion_shared(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); #endif /* Tests for reading data in parallel */ -static void test_read_one_chunk_filtered_dataset(void); -static void test_read_filtered_dataset_no_overlap(void); -static void test_read_filtered_dataset_overlap(void); -static void test_read_filtered_dataset_single_no_selection(void); -static void test_read_filtered_dataset_all_no_selection(void); -static void test_read_filtered_dataset_point_selection(void); -static void test_read_filtered_dataset_interleaved_read(void); -static void test_read_transformed_filtered_dataset_no_overlap(void); -static void test_read_3d_filtered_dataset_no_overlap_separate_pages(void); -static void test_read_3d_filtered_dataset_no_overlap_same_pages(void); -static void test_read_3d_filtered_dataset_overlap(void); -static void test_read_cmpd_filtered_dataset_no_conversion_unshared(void); -static void test_read_cmpd_filtered_dataset_no_conversion_shared(void); -static void test_read_cmpd_filtered_dataset_type_conversion_unshared(void); -static void test_read_cmpd_filtered_dataset_type_conversion_shared(void); - -#if MPI_VERSION >= 3 -/* Other miscellaneous tests */ -static void test_shrinking_growing_chunks(void); -#endif +static void test_read_one_chunk_filtered_dataset(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_read_filtered_dataset_no_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_read_filtered_dataset_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_read_filtered_dataset_single_no_selection(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_read_filtered_dataset_all_no_selection(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_read_filtered_dataset_point_selection(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_read_filtered_dataset_interleaved_read(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_read_transformed_filtered_dataset_no_overlap(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_read_3d_filtered_dataset_no_overlap_separate_pages(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_read_3d_filtered_dataset_no_overlap_same_pages(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_read_3d_filtered_dataset_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id); +static void test_read_cmpd_filtered_dataset_no_conversion_unshared(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_read_cmpd_filtered_dataset_no_conversion_shared(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_read_cmpd_filtered_dataset_type_conversion_unshared(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_read_cmpd_filtered_dataset_type_conversion_shared(const char * parent_group, + H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); /* * Tests for attempting to round-trip the data going from @@ -103,21 +169,40 @@ static void test_shrinking_growing_chunks(void); * * written in parallel -> read serially */ -static void test_write_serial_read_parallel(void); -#if MPI_VERSION >= 3 -static void test_write_parallel_read_serial(void); -#endif +static void test_write_serial_read_parallel(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); -static MPI_Comm comm = MPI_COMM_WORLD; -static MPI_Info info = MPI_INFO_NULL; -static int mpi_rank; -static int mpi_size; +#ifdef H5_HAVE_PARALLEL_FILTERED_WRITES +static void test_write_parallel_read_serial(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); -static void (*tests[])(void) = { -#if MPI_VERSION >= 3 +/* Other miscellaneous tests */ +static void test_shrinking_growing_chunks(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_edge_chunks_no_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_edge_chunks_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_edge_chunks_partial_write(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_fill_values(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, hid_t dcpl_id, + hid_t dxpl_id); +static void test_fill_value_undefined(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +static void test_fill_time_never(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id); +#endif + +static test_func tests[] = { +#ifdef H5_HAVE_PARALLEL_FILTERED_WRITES test_write_one_chunk_filtered_dataset, test_write_filtered_dataset_no_overlap, + test_write_filtered_dataset_no_overlap_partial, test_write_filtered_dataset_overlap, + test_write_filtered_dataset_single_unlim_dim_no_overlap, + test_write_filtered_dataset_single_unlim_dim_overlap, + test_write_filtered_dataset_multi_unlim_dim_no_overlap, + test_write_filtered_dataset_multi_unlim_dim_overlap, test_write_filtered_dataset_single_no_selection, test_write_filtered_dataset_all_no_selection, test_write_filtered_dataset_point_selection, @@ -147,33 +232,168 @@ static void (*tests[])(void) = { test_read_cmpd_filtered_dataset_type_conversion_unshared, test_read_cmpd_filtered_dataset_type_conversion_shared, test_write_serial_read_parallel, -#if MPI_VERSION >= 3 +#ifdef H5_HAVE_PARALLEL_FILTERED_WRITES test_write_parallel_read_serial, test_shrinking_growing_chunks, + test_edge_chunks_no_overlap, + test_edge_chunks_overlap, + test_edge_chunks_partial_write, + test_fill_values, + test_fill_value_undefined, + test_fill_time_never, #endif }; /* * Function to call the appropriate HDF5 filter-setting function - * depending on the currently set index. Used to re-run the tests + * depending on the given filter ID. Used to re-run the tests * with different filters to check that the data still comes back * correctly under a variety of circumstances, such as the * Fletcher32 checksum filter increasing the size of the chunk. */ static herr_t -set_dcpl_filter(hid_t dcpl) +set_dcpl_filter(hid_t dcpl_id, H5Z_filter_t filter_id, filter_options_t *filter_options) +{ + switch (filter_id) { + case H5Z_FILTER_DEFLATE: + return H5Pset_deflate(dcpl_id, DEFAULT_DEFLATE_LEVEL); + case H5Z_FILTER_SHUFFLE: + return H5Pset_shuffle(dcpl_id); + case H5Z_FILTER_FLETCHER32: + return H5Pset_fletcher32(dcpl_id); + case H5Z_FILTER_SZIP: { + unsigned pixels_per_block = H5_SZIP_MAX_PIXELS_PER_BLOCK; + hsize_t chunk_dims[H5S_MAX_RANK] = {0}; + size_t i, chunk_nelemts; + + VRFY(H5Pget_chunk(dcpl_id, H5S_MAX_RANK, chunk_dims) >= 0, "H5Pget_chunk succeeded"); + + for (i = 0, chunk_nelemts = 1; i < H5S_MAX_RANK; i++) + if (chunk_dims[i] > 0) + chunk_nelemts *= chunk_dims[i]; + + if (chunk_nelemts < H5_SZIP_MAX_PIXELS_PER_BLOCK) { + /* + * Can't set SZIP for chunk of 1 data element. + * Pixels-per-block value must be both even + * and non-zero. + */ + if (chunk_nelemts == 1) + return SUCCEED; + + if ((chunk_nelemts % 2) == 0) + pixels_per_block = (unsigned)chunk_nelemts; + else + pixels_per_block = (unsigned)(chunk_nelemts - 1); + } + else + pixels_per_block = H5_SZIP_MAX_PIXELS_PER_BLOCK; + + return H5Pset_szip(dcpl_id, 0, pixels_per_block); + } + case H5Z_FILTER_NBIT: + return H5Pset_nbit(dcpl_id); + case H5Z_FILTER_SCALEOFFSET: + return H5Pset_scaleoffset(dcpl_id, H5Z_SO_INT, 0); + default: { + if (!filter_options) + return FAIL; + + return H5Pset_filter(dcpl_id, filter_id, filter_options->flags, filter_options->cd_nelmts, + filter_options->cd_values); + } + } +} + +/* + * Function to verify the status of dataset storage space allocation + * based on the dataset's allocation time setting and how many chunks + * in the dataset have been written to. + */ +static herr_t +verify_space_alloc_status(hid_t dset_id, hid_t dcpl_id, num_chunks_written_t chunks_written) { - switch (cur_filter_idx) { - case GZIP_INDEX: - return H5Pset_deflate(dcpl, DEFAULT_DEFLATE_LEVEL); - case FLETCHER32_INDEX: - return H5Pset_fletcher32(dcpl); - default: - return H5Pset_deflate(dcpl, DEFAULT_DEFLATE_LEVEL); + int nfilters; + herr_t ret_value = SUCCEED; + + VRFY(((nfilters = H5Pget_nfilters(dcpl_id)) >= 0), "H5Pget_nfilters succeeded"); + + /* + * Only verify space allocation status when there are filters + * in the dataset's filter pipeline. When filters aren't in the + * pipeline, the space allocation time and status can vary based + * on whether the file was created in parallel or serial mode. + */ + if (nfilters > 0) { + H5D_space_status_t space_status; + H5D_alloc_time_t alloc_time; + + VRFY((H5Pget_alloc_time(dcpl_id, &alloc_time) >= 0), "H5Pget_alloc_time succeeded"); + VRFY((H5Dget_space_status(dset_id, &space_status) >= 0), "H5Dget_space_status succeeded"); + + switch (alloc_time) { + case H5D_ALLOC_TIME_EARLY: + /* + * Early space allocation should always result in the + * full dataset storage space being allocated. + */ + VRFY(space_status == H5D_SPACE_STATUS_ALLOCATED, "verified space allocation status"); + break; + case H5D_ALLOC_TIME_LATE: + /* + * Late space allocation should always result in the + * full dataset storage space being allocated when + * the dataset gets written to. However, if the dataset + * is extended the dataset's space allocation status + * can become partly allocated until the dataset is + * written to again. + */ + if (chunks_written == SOME_CHUNKS_WRITTEN || chunks_written == ALL_CHUNKS_WRITTEN) + VRFY((space_status == H5D_SPACE_STATUS_ALLOCATED) || + (space_status == H5D_SPACE_STATUS_PART_ALLOCATED), + "verified space allocation status"); + else if (chunks_written == NO_CHUNKS_WRITTEN) + /* + * A special case where we wrote to a dataset that + * uses late space allocation, but the write was + * either a no-op (no selection in the dataset + * from any rank) or something caused the write to + * fail late in the process of performing the actual + * write. In either case, space should still have + * been allocated. + */ + VRFY(space_status == H5D_SPACE_STATUS_ALLOCATED, "verified space allocation status"); + else + VRFY(space_status == H5D_SPACE_STATUS_NOT_ALLOCATED, "verified space allocation status"); + break; + case H5D_ALLOC_TIME_DEFAULT: + case H5D_ALLOC_TIME_INCR: + /* + * Incremental space allocation should result in + * the dataset's storage space being incrementally + * allocated as chunks are written to. Once all chunks + * have been written to, the space allocation should be + * seen as fully allocated. + */ + if (chunks_written == SOME_CHUNKS_WRITTEN) + VRFY((space_status == H5D_SPACE_STATUS_PART_ALLOCATED), + "verified space allocation status"); + else if (chunks_written == ALL_CHUNKS_WRITTEN) + VRFY((space_status == H5D_SPACE_STATUS_ALLOCATED), "verified space allocation status"); + else + VRFY(space_status == H5D_SPACE_STATUS_NOT_ALLOCATED, "verified space allocation status"); + break; + default: + if (MAINPROCESS) + MESG("unknown space allocation time"); + MPI_Abort(MPI_COMM_WORLD, 1); + } } + + return ret_value; } -#if MPI_VERSION >= 3 +#ifdef H5_HAVE_PARALLEL_FILTERED_WRITES /* * Tests parallel write of filtered data in the special * case where a dataset is composed of a single chunk. @@ -182,7 +402,8 @@ set_dcpl_filter(hid_t dcpl) * 02/01/2017 */ static void -test_write_one_chunk_filtered_dataset(void) +test_write_one_chunk_filtered_dataset(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *data = NULL; C_DATATYPE *read_buf = NULL; @@ -195,26 +416,18 @@ test_write_one_chunk_filtered_dataset(void) hsize_t count[WRITE_ONE_CHUNK_FILTERED_DATASET_DIMS]; hsize_t block[WRITE_ONE_CHUNK_FILTERED_DATASET_DIMS]; size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) HDputs("Testing write to one-chunk filtered dataset"); - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ dataset_dims[0] = (hsize_t)WRITE_ONE_CHUNK_FILTERED_DATASET_NROWS; @@ -231,19 +444,21 @@ test_write_one_chunk_filtered_dataset(void) VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); VRFY((H5Pset_chunk(plist_id, WRITE_ONE_CHUNK_FILTERED_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_ONE_CHUNK_FILTERED_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + dset_id = H5Dcreate2(group_id, WRITE_ONE_CHUNK_FILTERED_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes @@ -293,15 +508,12 @@ test_write_one_chunk_filtered_dataset(void) ((C_DATATYPE)i / (WRITE_ONE_CHUNK_FILTERED_DATASET_CH_NROWS / mpi_size * WRITE_ONE_CHUNK_FILTERED_DATASET_CH_NCOLS)); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + if (data) HDfree(data); @@ -311,10 +523,10 @@ test_write_one_chunk_filtered_dataset(void) read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_ONE_CHUNK_FILTERED_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_ONE_CHUNK_FILTERED_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -324,10 +536,11 @@ test_write_one_chunk_filtered_dataset(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; @@ -343,7 +556,8 @@ test_write_one_chunk_filtered_dataset(void) * 02/01/2017 */ static void -test_write_filtered_dataset_no_overlap(void) +test_write_filtered_dataset_no_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *data = NULL; C_DATATYPE *read_buf = NULL; @@ -356,27 +570,18 @@ test_write_filtered_dataset_no_overlap(void) hsize_t count[WRITE_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; hsize_t block[WRITE_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) HDputs("Testing write to unshared filtered chunks"); - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ dataset_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_NROWS; @@ -393,20 +598,22 @@ test_write_filtered_dataset_no_overlap(void) VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_UNSHARED_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + dset_id = H5Dcreate2(group_id, WRITE_UNSHARED_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes @@ -454,15 +661,12 @@ test_write_filtered_dataset_no_overlap(void) correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + if (data) HDfree(data); @@ -472,10 +676,10 @@ test_write_filtered_dataset_no_overlap(void) read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_UNSHARED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_UNSHARED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -485,102 +689,91 @@ test_write_filtered_dataset_no_overlap(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of filtered data in the case where - * more than one process is writing to a particular chunk - * in the operation. In this case, the chunks have to be - * redistributed before the operation so that only one process - * writes to a particular chunk. - * - * Programmer: Jordan Henderson - * 02/01/2017 + * Tests parallel write of filtered data in the case where only + * one process is writing to a particular chunk in the operation + * and that process only writes to part of a chunk. */ static void -test_write_filtered_dataset_overlap(void) +test_write_filtered_dataset_no_overlap_partial(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *data = NULL; C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t start[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t stride[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t count[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t block[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t dataset_dims[WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS]; + hsize_t sel_dims[WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS]; + hsize_t start[WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS]; + hsize_t stride[WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS]; + hsize_t count[WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS]; + hsize_t block[WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS]; size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to shared filtered chunks"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + HDputs("Testing partial write to unshared filtered chunks"); - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_NCOLS; - chunk_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS; - sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; - sel_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS * (hsize_t)DIM1_SCALE_FACTOR; - - filespace = H5Screate_simple(WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + dataset_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_NROWS; + dataset_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NCOLS; + sel_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NROWS; + sel_dims[1] = (hsize_t)(WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_NCOLS / + WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NCOLS); + + filespace = H5Screate_simple(WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_SHARED_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, - H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_NAME, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes * it to the hyperslab in the file */ - count[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_NROWS / (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NROWS; - count[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_NCOLS / (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS; - stride[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NROWS; - stride[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS; - block[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NROWS / (hsize_t)mpi_size; - block[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS; - start[0] = (hsize_t)mpi_rank * block[0]; + count[0] = 1; + count[1] = (hsize_t)(WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_NCOLS / + WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NCOLS); + stride[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NROWS; + stride[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NCOLS; + block[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NROWS; + block[1] = (hsize_t)1; + start[0] = ((hsize_t)mpi_rank * (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NROWS * count[0]); start[1] = 0; if (VERBOSE_MED) { @@ -611,33 +804,38 @@ test_write_filtered_dataset_overlap(void) for (i = 0; i < data_size / sizeof(*data); i++) data[i] = (C_DATATYPE)GEN_DATA(i); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)( - (dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); - - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + for (i = 0; i < (size_t)mpi_size; i++) { + size_t rank_n_elems = (size_t)(mpi_size * (WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NROWS * + WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NCOLS)); + size_t data_idx = i; - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + for (size_t j = 0; j < rank_n_elems; j++) { + if ((j % WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NCOLS) == 0) { + correct_buf[(i * rank_n_elems) + j] = (C_DATATYPE)data_idx; + data_idx++; + } + } + } - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + if (data) HDfree(data); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - /* Verify correct data was written */ + /* Verify the correct data was written */ read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_SHARED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -647,10 +845,10 @@ test_write_filtered_dataset_overlap(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; @@ -658,98 +856,85 @@ test_write_filtered_dataset_overlap(void) /* * Tests parallel write of filtered data in the case where - * a single process in the write operation has no selection - * in the dataset's dataspace. In this case, the process with - * no selection still has to participate in the collective - * space re-allocation for the filtered chunks and also must - * participate in the re-insertion of the filtered chunks - * into the chunk index. + * more than one process is writing to a particular chunk + * in the operation. In this case, the chunks have to be + * redistributed before the operation so that only one process + * writes to a particular chunk. * * Programmer: Jordan Henderson * 02/01/2017 */ static void -test_write_filtered_dataset_single_no_selection(void) +test_write_filtered_dataset_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *data = NULL; C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t start[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t stride[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t count[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t block[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t dataset_dims[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t start[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t stride[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t count[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t block[WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; size_t i, data_size, correct_buf_size; - size_t segment_length; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to filtered chunks with a single process having no selection"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + HDputs("Testing write to shared filtered chunks"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS; - chunk_dims[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - sel_dims[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - sel_dims[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS; - - if (mpi_rank == WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) - sel_dims[0] = sel_dims[1] = 0; + dataset_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS; + sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; + sel_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS * (hsize_t)DIM1_SCALE_FACTOR; - filespace = H5Screate_simple(WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); + memspace = H5Screate_simple(WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, - filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, WRITE_SHARED_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes * it to the hyperslab in the file */ - count[0] = 1; - count[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS / - (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - stride[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - stride[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - block[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - block[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - start[0] = (hsize_t)mpi_rank * (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS * count[0]; + count[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_NROWS / (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NROWS; + count[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_NCOLS / (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS; + stride[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NROWS / (hsize_t)mpi_size; + block[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS; + start[0] = (hsize_t)mpi_rank * block[0]; start[1] = 0; if (VERBOSE_MED) { @@ -764,11 +949,8 @@ test_write_filtered_dataset_single_no_selection(void) filespace = H5Dget_space(dset_id); VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - if (mpi_rank == WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) - VRFY((H5Sselect_none(filespace) >= 0), "Select none succeeded"); - else - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), - "Hyperslab selection succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); /* Fill data buffer */ data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); @@ -784,37 +966,29 @@ test_write_filtered_dataset_single_no_selection(void) data[i] = (C_DATATYPE)GEN_DATA(i); for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + - (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); - - /* Compute the correct offset into the buffer for the process having no selection and clear it */ - segment_length = dataset_dims[0] * dataset_dims[1] / (hsize_t)mpi_size; - HDmemset(correct_buf + - ((size_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC * segment_length), - 0, segment_length * sizeof(*data)); - - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + correct_buf[i] = (C_DATATYPE)( + (dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + if (data) HDfree(data); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - /* Verify the correct data was written */ + /* Verify correct data was written */ read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_SHARED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -824,294 +998,318 @@ test_write_filtered_dataset_single_no_selection(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of filtered data in the case - * where no process in the write operation has a - * selection in the dataset's dataspace. This test is - * to ensure that there are no assertion failures or - * similar issues due to size 0 allocations and the - * like. In this case, the file and dataset are created - * but the dataset is populated with the default fill - * value. - * - * Programmer: Jordan Henderson - * 02/02/2017 + * Tests parallel write of filtered data in the case where + * a dataset has a single unlimited dimension and each + * MPI rank writes to its own separate chunk. On each + * iteration, the dataset is extended in its extensible + * dimension by "MPI size" chunks per rank and the new + * chunks are written to, read back and verified. */ static void -test_write_filtered_dataset_all_no_selection(void) +test_write_filtered_dataset_single_unlim_dim_no_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - C_DATATYPE *data = NULL; - C_DATATYPE *read_buf = NULL; - C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + hsize_t dataset_dims[WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t max_dims[WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t sel_dims[WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t start[WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t stride[WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t count[WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t block[WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + size_t i, data_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to filtered chunks with all processes having no selection"); + HDputs("Testing write to unshared filtered chunks w/ single unlimited dimension"); - CHECK_CUR_FILTER_AVAIL(); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_NROWS; + dataset_dims[1] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_NCOLS; + max_dims[0] = dataset_dims[0]; + max_dims[1] = H5S_UNLIMITED; + chunk_dims[0] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NCOLS; + sel_dims[0] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NROWS; + sel_dims[1] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_NCOLS; + + filespace = H5Screate_simple(WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS, dataset_dims, max_dims); + VRFY((filespace >= 0), "File dataspace creation succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_NCOLS; - chunk_dims[0] = (hsize_t)WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - sel_dims[0] = sel_dims[1] = 0; - - filespace = H5Screate_simple(WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); - VRFY((filespace >= 0), "File dataspace creation succeeded"); - - memspace = H5Screate_simple(WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - - /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); - - VRFY((H5Pset_chunk(plist_id, WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), - "Chunk size set"); - - /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); - - dset_id = H5Dcreate2(file_id, WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, - filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - VRFY((H5Sselect_none(filespace) >= 0), "Select none succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Fill data buffer */ - data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); data = (C_DATATYPE *)HDcalloc(1, data_size); VRFY((NULL != data), "HDcalloc succeeded"); - correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); + read_buf = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); for (i = 0; i < data_size / sizeof(*data); i++) data[i] = (C_DATATYPE)GEN_DATA(i); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + for (i = 0; i < (size_t)WRITE_UNSHARED_ONE_UNLIM_DIM_NLOOPS; i++) { + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = 1; + count[1] = + (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_NCOLS / (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NCOLS; + stride[0] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NROWS; + stride[1] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NCOLS; + block[0] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NROWS; + block[1] = (hsize_t)WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NCOLS; + start[0] = ((hsize_t)mpi_rank * block[0] * count[0]); + start[1] = i * count[1] * block[1]; + + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], + block[1]); + HDfflush(stdout); + } - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), - "Dataset write succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - if (data) - HDfree(data); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); - /* Verify the correct data was written */ - read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + dset_id = H5Dopen2(group_id, WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), - "Dataset read succeeded"); + HDmemset(read_buf, 255, data_size); - VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - if (correct_buf) - HDfree(correct_buf); + /* Verify the correct data was written */ + VRFY((0 == HDmemcmp(read_buf, data, data_size)), "Data verification succeeded"); + + if (i < (size_t)WRITE_UNSHARED_ONE_UNLIM_DIM_NLOOPS - 1) { + /* Extend the dataset by count[1] chunks in the extensible dimension */ + dataset_dims[1] += count[1] * block[1]; + VRFY(H5Dset_extent(dset_id, dataset_dims) >= 0, "H5Dset_extent succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); + } + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + } + + if (data) + HDfree(data); if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of filtered data by using - * point selections instead of hyperslab selections. - * - * Programmer: Jordan Henderson - * 02/02/2017 + * Tests parallel write of filtered data in the case where + * a dataset has a single unlimited dimension and each + * MPI rank writes to a portion of each chunk in the dataset. + * On each iteration, the dataset is extended in its extensible + * dimension by two chunks and the new chunks are written to + * by all ranks, then read back and verified. */ static void -test_write_filtered_dataset_point_selection(void) +test_write_filtered_dataset_single_unlim_dim_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - C_DATATYPE *data = NULL; - C_DATATYPE *correct_buf = NULL; - C_DATATYPE *read_buf = NULL; - hsize_t * coords = NULL; - hsize_t dataset_dims[WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - size_t i, j, data_size, correct_buf_size; - size_t num_points; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + hsize_t dataset_dims[WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t max_dims[WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t sel_dims[WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t start[WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t stride[WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t count[WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + hsize_t block[WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS]; + size_t i, data_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to filtered chunks with point selection"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + HDputs("Testing write to shared filtered chunks w/ single unlimited dimension"); - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS; - chunk_dims[0] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - sel_dims[0] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NROWS / (hsize_t)mpi_size; - sel_dims[1] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS; + dataset_dims[0] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_NROWS; + dataset_dims[1] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_NCOLS; + max_dims[0] = dataset_dims[0]; + max_dims[1] = H5S_UNLIMITED; + chunk_dims[0] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_CH_NCOLS; + sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; + sel_dims[1] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_CH_NCOLS * (hsize_t)DIM1_SCALE_FACTOR; - filespace = H5Screate_simple(WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS, dataset_dims, max_dims); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, - filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, WRITE_SHARED_ONE_UNLIM_DIM_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - - /* Set up point selection */ - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - - num_points = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NROWS * - (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS / (hsize_t)mpi_size; - coords = (hsize_t *)HDcalloc(1, 2 * num_points * sizeof(*coords)); - VRFY((NULL != coords), "Coords HDcalloc succeeded"); - - for (i = 0; i < num_points; i++) - for (j = 0; j < WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS; j++) - coords[(i * WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS) + j] = - (j > 0) ? (i % (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS) - : ((hsize_t)mpi_rank + - ((hsize_t)mpi_size * (i / (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS))); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - VRFY((H5Sselect_elements(filespace, H5S_SELECT_SET, (hsize_t)num_points, (const hsize_t *)coords) >= 0), - "Point selection succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Fill data buffer */ - data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); data = (C_DATATYPE *)HDcalloc(1, data_size); VRFY((NULL != data), "HDcalloc succeeded"); - correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); + read_buf = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); for (i = 0; i < data_size / sizeof(*data); i++) data[i] = (C_DATATYPE)GEN_DATA(i); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)( - (dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + for (i = 0; i < (size_t)WRITE_SHARED_ONE_UNLIM_DIM_NLOOPS; i++) { + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_NROWS / (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_CH_NROWS; + count[1] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_NCOLS / (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_CH_NCOLS; + stride[0] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_CH_NROWS; + stride[1] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_CH_NCOLS; + block[0] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_CH_NROWS / (hsize_t)mpi_size; + block[1] = (hsize_t)WRITE_SHARED_ONE_UNLIM_DIM_CH_NCOLS; + start[0] = (hsize_t)mpi_rank * block[0]; + start[1] = i * count[1] * block[1]; + + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], + block[1]); + HDfflush(stdout); + } - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), - "Dataset write succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - if (data) - HDfree(data); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - /* Verify the correct data was written */ - read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); + dset_id = H5Dopen2(group_id, WRITE_SHARED_ONE_UNLIM_DIM_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + HDmemset(read_buf, 255, data_size); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), - "Dataset read succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + /* Verify correct data was written */ + VRFY((0 == HDmemcmp(read_buf, data, data_size)), "Data verification succeeded"); - if (coords) - HDfree(coords); - if (correct_buf) - HDfree(correct_buf); + if (i < (size_t)WRITE_SHARED_ONE_UNLIM_DIM_NLOOPS - 1) { + /* Extend the dataset by count[1] chunks in the extensible dimension */ + dataset_dims[1] += count[1] * block[1]; + VRFY(H5Dset_extent(dset_id, dataset_dims) >= 0, "H5Dset_extent succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); + } + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + } + + if (data) + HDfree(data); if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; @@ -1119,272 +1317,424 @@ test_write_filtered_dataset_point_selection(void) /* * Tests parallel write of filtered data in the case where - * each process writes an equal amount of data to each chunk - * in the dataset. Each chunk is distributed among the - * processes in round-robin fashion by blocks of size 1 until - * the whole chunk is selected, leading to an interleaved - * write pattern. - * - * Programmer: Jordan Henderson - * 02/02/2017 + * a dataset has two unlimited dimensions and each + * MPI rank writes to its own separate chunks. On each + * iteration, the dataset is extended in its first + * extensible dimension by the size of one chunk per rank + * and in its second extensible dimension by the size of + * one chunk. Then, all chunks are written to, read back + * and verified. */ static void -test_write_filtered_dataset_interleaved_write(void) +test_write_filtered_dataset_multi_unlim_dim_no_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - C_DATATYPE *data = NULL; - C_DATATYPE *read_buf = NULL; - C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; - hsize_t chunk_dims[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; - hsize_t sel_dims[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; - hsize_t start[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; - hsize_t stride[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; - hsize_t count[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; - hsize_t block[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; - size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + hsize_t dataset_dims[WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t max_dims[WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t sel_dims[WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t start[WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t stride[WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t count[WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t block[WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + size_t i, data_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing interleaved write to filtered chunks"); + HDputs("Testing write to unshared filtered chunks w/ two unlimited dimensions"); - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NROWS; - dataset_dims[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS; - chunk_dims[0] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NROWS; - chunk_dims[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NCOLS; - sel_dims[0] = (hsize_t)(INTERLEAVED_WRITE_FILTERED_DATASET_NROWS / mpi_size); - sel_dims[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS; - - filespace = H5Screate_simple(INTERLEAVED_WRITE_FILTERED_DATASET_DIMS, dataset_dims, NULL); + dataset_dims[0] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_NROWS; + dataset_dims[1] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_NCOLS; + max_dims[0] = H5S_UNLIMITED; + max_dims[1] = H5S_UNLIMITED; + chunk_dims[0] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NCOLS; + sel_dims[0] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NROWS; + sel_dims[1] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_NCOLS; + + filespace = H5Screate_simple(WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS, dataset_dims, max_dims); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(INTERLEAVED_WRITE_FILTERED_DATASET_DIMS, sel_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, INTERLEAVED_WRITE_FILTERED_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, INTERLEAVED_WRITE_FILTERED_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + dset_id = H5Dcreate2(group_id, WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - /* Each process defines the dataset selection in memory and writes - * it to the hyperslab in the file - */ - count[0] = - (hsize_t)(INTERLEAVED_WRITE_FILTERED_DATASET_NROWS / INTERLEAVED_WRITE_FILTERED_DATASET_CH_NROWS); - count[1] = - (hsize_t)(INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS / INTERLEAVED_WRITE_FILTERED_DATASET_CH_NCOLS); - stride[0] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NROWS; - stride[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NCOLS; - block[0] = 1; - block[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NCOLS; - start[0] = (hsize_t)mpi_rank; - start[1] = 0; + for (i = 0; i < (size_t)WRITE_UNSHARED_TWO_UNLIM_DIM_NLOOPS; i++) { + C_DATATYPE *tmp_realloc = NULL; + size_t j; - if (VERBOSE_MED) { - HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE - ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE - ", %" PRIuHSIZE " ]\n", - mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); - HDfflush(stdout); - } + /* Set selected dimensions */ + sel_dims[0] = (i + 1) * WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NROWS; + sel_dims[1] = (i + 1) * WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NCOLS; - /* Select hyperslab in the file */ - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), - "Hyperslab selection succeeded"); + tmp_realloc = (C_DATATYPE *)HDrealloc(data, data_size); + VRFY((NULL != tmp_realloc), "HDrealloc succeeded"); + data = tmp_realloc; - /* Fill data buffer */ - data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + tmp_realloc = (C_DATATYPE *)HDrealloc(read_buf, data_size); + VRFY((NULL != tmp_realloc), "HDrealloc succeeded"); + read_buf = tmp_realloc; - data = (C_DATATYPE *)HDcalloc(1, data_size); - VRFY((NULL != data), "HDcalloc succeeded"); + for (j = 0; j < data_size / sizeof(*data); j++) + data[j] = (C_DATATYPE)GEN_DATA(j); - correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - for (i = 0; i < data_size / sizeof(*data); i++) - data[i] = (C_DATATYPE)GEN_DATA(i); + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = (i + 1); + count[1] = (i + 1); + stride[0] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NROWS; + stride[1] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NCOLS; + block[0] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NROWS; + block[1] = (hsize_t)WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NCOLS; + start[0] = ((hsize_t)mpi_rank * block[0] * count[0]); + start[1] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], + block[1]); + HDfflush(stdout); + } - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - /* Add Column Index */ - correct_buf[i] = - (C_DATATYPE)((i % (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS) + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - /* Add the Row Index */ - + ((i % (hsize_t)(mpi_size * INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS)) / - (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS) + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - /* Add the amount that gets added when a rank moves down to its next section - vertically in the dataset */ - + ((hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS * - (i / (hsize_t)(mpi_size * INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS)))); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + + dset_id = H5Dopen2(group_id, WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + HDmemset(read_buf, 255, data_size); - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), - "Dataset write succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + + /* Verify the correct data was written */ + VRFY((0 == HDmemcmp(read_buf, data, data_size)), "Data verification succeeded"); + + if (i < (size_t)WRITE_UNSHARED_TWO_UNLIM_DIM_NLOOPS - 1) { + /* + * Extend the dataset by the size of one chunk per rank + * in the first extensible dimension. Extend the dataset + * by the size of chunk in the second extensible dimension. + */ + dataset_dims[0] += (hsize_t)mpi_size * block[0]; + dataset_dims[1] += block[1]; + VRFY(H5Dset_extent(dset_id, dataset_dims) >= 0, "H5Dset_extent succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); + } + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + } if (data) HDfree(data); + if (read_buf) + HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); - /* Verify the correct data was written */ - read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); + return; +} - dset_id = H5Dopen2(file_id, "/" INTERLEAVED_WRITE_FILTERED_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); +/* + * Tests parallel write of filtered data in the case where + * a dataset has two unlimited dimensions and each MPI + * rank writes to a portion of each chunk in the dataset. + * On each iteration, the dataset is extended in its extensible + * dimensions by the size of a chunk and then all chunks are + * written to by all ranks, then read back and verified. + */ +static void +test_write_filtered_dataset_multi_unlim_dim_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) +{ + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + hsize_t dataset_dims[WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t max_dims[WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t sel_dims[WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t start[WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t stride[WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t count[WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + hsize_t block[WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS]; + size_t i, data_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), - "Dataset read succeeded"); + if (MAINPROCESS) + HDputs("Testing write to shared filtered chunks w/ two unlimited dimensions"); - VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); - if (correct_buf) - HDfree(correct_buf); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_NROWS; + dataset_dims[1] = (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_NCOLS; + max_dims[0] = H5S_UNLIMITED; + max_dims[1] = H5S_UNLIMITED; + chunk_dims[0] = (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_CH_NCOLS; + sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; + sel_dims[1] = (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_CH_NCOLS * (hsize_t)DIM1_SCALE_FACTOR; + + filespace = H5Screate_simple(WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS, dataset_dims, max_dims); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, WRITE_SHARED_TWO_UNLIM_DIM_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + for (i = 0; i < (size_t)WRITE_SHARED_TWO_UNLIM_DIM_NLOOPS; i++) { + C_DATATYPE *tmp_realloc = NULL; + size_t j; + + /* Set selected dimensions */ + sel_dims[0] = (i + 1); + sel_dims[1] = (i + 1) * (size_t)WRITE_SHARED_TWO_UNLIM_DIM_CH_NCOLS; + + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); + + tmp_realloc = (C_DATATYPE *)HDrealloc(data, data_size); + VRFY((NULL != tmp_realloc), "HDrealloc succeeded"); + data = tmp_realloc; + + tmp_realloc = (C_DATATYPE *)HDrealloc(read_buf, data_size); + VRFY((NULL != tmp_realloc), "HDrealloc succeeded"); + read_buf = tmp_realloc; + + for (j = 0; j < data_size / sizeof(*data); j++) + data[j] = (C_DATATYPE)GEN_DATA(j); + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = (i + 1); + count[1] = (i + 1); + stride[0] = (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_CH_NROWS; + stride[1] = (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_CH_NCOLS; + block[0] = 1; + block[1] = (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_CH_NROWS; + start[0] = (hsize_t)mpi_rank; + start[1] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], + block[1]); + HDfflush(stdout); + } + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + + dset_id = H5Dopen2(group_id, WRITE_SHARED_TWO_UNLIM_DIM_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + HDmemset(read_buf, 255, data_size); + + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + + /* Verify correct data was written */ + VRFY((0 == HDmemcmp(read_buf, data, data_size)), "Data verification succeeded"); + + if (i < (size_t)WRITE_SHARED_TWO_UNLIM_DIM_NLOOPS - 1) { + /* Extend the dataset by the size of a chunk in each extensible dimension */ + dataset_dims[0] += (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_CH_NROWS; + dataset_dims[1] += (hsize_t)WRITE_SHARED_TWO_UNLIM_DIM_CH_NCOLS; + VRFY(H5Dset_extent(dset_id, dataset_dims) >= 0, "H5Dset_extent succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); + } + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + } + + if (data) + HDfree(data); if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of transformed and filtered data - * in the case where only one process is writing to a - * particular chunk in the operation. Normally, a data - * transform function will cause the parallel library to - * break to independent I/O and this isn't allowed when - * there are filters in the pipeline. However, in this - * case the parallel library recognizes that the used - * data transform function "x" is the same as not applying - * the transform function. Therefore it does not apply - * the transform function resulting in not breaking to - * independent I/O. + * Tests parallel write of filtered data in the case where + * a single process in the write operation has no selection + * in the dataset's dataspace. In this case, the process with + * no selection still has to participate in the collective + * space re-allocation for the filtered chunks and also must + * participate in the re-insertion of the filtered chunks + * into the chunk index. * - * Programmer: Jan-Willem Blokland - * 08/20/2021 + * Programmer: Jordan Henderson + * 02/01/2017 */ static void -test_write_transformed_filtered_dataset_no_overlap(void) +test_write_filtered_dataset_single_no_selection(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *data = NULL; C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t start[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t stride[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t count[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t block[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t dataset_dims[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t start[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t stride[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t count[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t block[WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + size_t segment_length; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to unshared transformed and filtered chunks"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + HDputs("Testing write to filtered chunks with a single process having no selection"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS; - chunk_dims[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; - sel_dims[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; - sel_dims[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS; + dataset_dims[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + sel_dims[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + sel_dims[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS; - filespace = H5Screate_simple(WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + if (mpi_rank == WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) + sel_dims[0] = sel_dims[1] = 0; + + filespace = H5Screate_simple(WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); + memspace = H5Screate_simple(WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, + dset_id = H5Dcreate2(group_id, WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes * it to the hyperslab in the file */ count[0] = 1; - count[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS / - (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; - stride[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; - stride[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; - block[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; - block[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; - start[0] = ((hsize_t)mpi_rank * (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS * count[0]); + count[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS / + (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + stride[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + block[1] = (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + start[0] = (hsize_t)mpi_rank * (hsize_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS * count[0]; start[1] = 0; if (VERBOSE_MED) { @@ -1399,38 +1749,43 @@ test_write_transformed_filtered_dataset_no_overlap(void) filespace = H5Dget_space(dset_id); VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), - "Hyperslab selection succeeded"); + if (mpi_rank == WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) + VRFY((H5Sselect_none(filespace) >= 0), "Select none succeeded"); + else + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); /* Fill data buffer */ data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); - data = (C_DATATYPE *)HDcalloc(1, data_size); - VRFY((NULL != data), "HDcalloc succeeded"); + if (mpi_rank != WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) { + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); + + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); + } correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != correct_buf), "HDcalloc succeeded"); - for (i = 0; i < data_size / sizeof(*data); i++) - data[i] = (C_DATATYPE)GEN_DATA(i); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); - /* Create property list for collective dataset write and data transform */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - /* Set data transform expression */ - VRFY((H5Pset_data_transform(plist_id, "x") >= 0), "Set data transform expression succeeded"); + /* Compute the correct offset into the buffer for the process having no selection and clear it */ + segment_length = dataset_dims[0] * dataset_dims[1] / (hsize_t)mpi_size; + HDmemset(correct_buf + + ((size_t)WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC * segment_length), + 0, segment_length * sizeof(*data)); - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); + /* Verify space allocation status - data should only have been written if MPI size > 1 */ + verify_space_alloc_status(dset_id, plist_id, (mpi_size > 1 ? SOME_CHUNKS_WRITTEN : NO_CHUNKS_WRITTEN)); + if (data) HDfree(data); @@ -1440,10 +1795,10 @@ test_write_transformed_filtered_dataset_no_overlap(void) read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -1453,131 +1808,93 @@ test_write_transformed_filtered_dataset_no_overlap(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of filtered data in the case where - * the dataset has 3 dimensions and each process writes - * to its own "page" in the 3rd dimension. + * Tests parallel write of filtered data in the case + * where no process in the write operation has a + * selection in the dataset's dataspace. This test is + * to ensure that there are no assertion failures or + * similar issues due to size 0 allocations and the + * like. In this case, the file and dataset are created + * but the dataset is populated with the default fill + * value. * * Programmer: Jordan Henderson - * 02/06/2017 + * 02/02/2017 */ static void -test_write_3d_filtered_dataset_no_overlap_separate_pages(void) +test_write_filtered_dataset_all_no_selection(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *data = NULL; C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t sel_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t start[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t stride[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t count[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t block[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t dataset_dims[WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to unshared filtered chunks on separate pages in 3D dataset"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + HDputs("Testing write to filtered chunks with all processes having no selection"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS; - dataset_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS; - dataset_dims[2] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DEPTH; - chunk_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; - chunk_dims[2] = 1; - sel_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS; - sel_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS; - sel_dims[2] = 1; + dataset_dims[0] = (hsize_t)WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + sel_dims[0] = sel_dims[1] = 0; - filespace = H5Screate_simple(WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, sel_dims, NULL); + memspace = H5Screate_simple(WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_NAME, HDF5_DATATYPE_NAME, + dset_id = H5Dcreate2(group_id, WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - - /* Each process defines the dataset selection in memory and writes - * it to the hyperslab in the file - */ - count[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS / - (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; - count[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS / - (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; - count[2] = 1; - stride[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; - stride[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; - stride[2] = 1; - block[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; - block[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; - block[2] = 1; - start[0] = 0; - start[1] = 0; - start[2] = (hsize_t)mpi_rank; + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - if (VERBOSE_MED) { - HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE - " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE - ", %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE - " ]\n", - mpi_rank, count[0], count[1], count[2], stride[0], stride[1], stride[2], start[0], start[1], - start[2], block[0], block[1], block[2]); - HDfflush(stdout); - } + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - /* Select hyperslab in the file */ filespace = H5Dget_space(dset_id); VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), - "Hyperslab selection succeeded"); + VRFY((H5Sselect_none(filespace) >= 0), "Select none succeeded"); /* Fill data buffer */ - data_size = sel_dims[0] * sel_dims[1] * sel_dims[2] * sizeof(*data); - correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); data = (C_DATATYPE *)HDcalloc(1, data_size); VRFY((NULL != data), "HDcalloc succeeded"); @@ -1588,18 +1905,12 @@ test_write_3d_filtered_dataset_no_overlap_separate_pages(void) for (i = 0; i < data_size / sizeof(*data); i++) data[i] = (C_DATATYPE)GEN_DATA(i); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)((i % (hsize_t)mpi_size) + (i / (hsize_t)mpi_size)); - - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); + /* Verify space allocation status - no ranks should have written any data */ + verify_space_alloc_status(dset_id, plist_id, NO_CHUNKS_WRITTEN); + if (data) HDfree(data); @@ -1609,10 +1920,10 @@ test_write_3d_filtered_dataset_no_overlap_separate_pages(void) read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -1622,132 +1933,104 @@ test_write_3d_filtered_dataset_no_overlap_separate_pages(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of filtered data in the case where - * the dataset has 3 dimensions and each process writes - * to each "page" in the 3rd dimension. However, no chunk - * on a given "page" is written to by more than one process. + * Tests parallel write of filtered data by using + * point selections instead of hyperslab selections. * * Programmer: Jordan Henderson - * 02/06/2017 + * 02/02/2017 */ static void -test_write_3d_filtered_dataset_no_overlap_same_pages(void) +test_write_filtered_dataset_point_selection(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *data = NULL; - C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t sel_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t start[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t stride[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t count[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t block[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - size_t i, data_size, correct_buf_size; - hid_t file_id, dset_id, plist_id; - hid_t filespace, memspace; + C_DATATYPE *read_buf = NULL; + hsize_t * coords = NULL; + hsize_t dataset_dims[WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + size_t i, j, data_size, correct_buf_size; + size_t num_points; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to unshared filtered chunks on the same pages in 3D dataset"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + HDputs("Testing write to filtered chunks with point selection"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NROWS; - dataset_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS; - dataset_dims[2] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DEPTH; - chunk_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; - chunk_dims[2] = 1; - sel_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; - sel_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS; - sel_dims[2] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DEPTH; + dataset_dims[0] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + sel_dims[0] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NROWS / (hsize_t)mpi_size; + sel_dims[1] = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS; - filespace = - H5Screate_simple(WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, sel_dims, NULL); + memspace = H5Screate_simple(WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_NAME, - HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - - /* Each process defines the dataset selection in memory and writes - * it to the hyperslab in the file - */ - count[0] = 1; - count[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS / - (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; - count[2] = (hsize_t)mpi_size; - stride[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; - stride[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; - stride[2] = 1; - block[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; - block[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; - block[2] = 1; - start[0] = ((hsize_t)mpi_rank * (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS * count[0]); - start[1] = 0; - start[2] = 0; + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - if (VERBOSE_MED) { - HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE - " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE - ", %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE - " ]\n", - mpi_rank, count[0], count[1], count[2], stride[0], stride[1], stride[2], start[0], start[1], - start[2], block[0], block[1], block[2]); - HDfflush(stdout); - } + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - /* Select hyperslab in the file */ + /* Set up point selection */ filespace = H5Dget_space(dset_id); VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), - "Hyperslab selection succeeded"); + num_points = (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NROWS * + (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS / (hsize_t)mpi_size; + coords = (hsize_t *)HDcalloc(1, 2 * num_points * sizeof(*coords)); + VRFY((NULL != coords), "Coords HDcalloc succeeded"); + + for (i = 0; i < num_points; i++) + for (j = 0; j < WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS; j++) + coords[(i * WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS) + j] = + (j > 0) ? (i % (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS) + : ((hsize_t)mpi_rank + + ((hsize_t)mpi_size * (i / (hsize_t)WRITE_POINT_SELECTION_FILTERED_CHUNKS_NCOLS))); + + VRFY((H5Sselect_elements(filespace, H5S_SELECT_SET, (hsize_t)num_points, (const hsize_t *)coords) >= 0), + "Point selection succeeded"); /* Fill data buffer */ - data_size = sel_dims[0] * sel_dims[1] * sel_dims[2] * sizeof(*data); - correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); data = (C_DATATYPE *)HDcalloc(1, data_size); VRFY((NULL != data), "HDcalloc succeeded"); @@ -1759,18 +2042,16 @@ test_write_3d_filtered_dataset_no_overlap_same_pages(void) data[i] = (C_DATATYPE)GEN_DATA(i); for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] * dataset_dims[1])) + - (i / (dataset_dims[0] * dataset_dims[1]))); - - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + correct_buf[i] = (C_DATATYPE)( + (dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + if (data) HDfree(data); @@ -1780,23 +2061,26 @@ test_write_3d_filtered_dataset_no_overlap_same_pages(void) read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_POINT_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + if (coords) + HDfree(coords); if (correct_buf) HDfree(correct_buf); if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; @@ -1804,106 +2088,95 @@ test_write_3d_filtered_dataset_no_overlap_same_pages(void) /* * Tests parallel write of filtered data in the case where - * the dataset has 3 dimensions and each process writes - * to each "page" in the 3rd dimension. Further, each chunk - * in each "page" is written to equally by all processes. + * each process writes an equal amount of data to each chunk + * in the dataset. Each chunk is distributed among the + * processes in round-robin fashion by blocks of size 1 until + * the whole chunk is selected, leading to an interleaved + * write pattern. * * Programmer: Jordan Henderson - * 02/06/2017 + * 02/02/2017 */ static void -test_write_3d_filtered_dataset_overlap(void) +test_write_filtered_dataset_interleaved_write(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *data = NULL; C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t sel_dims[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t start[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t stride[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t count[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t block[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t dataset_dims[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; + hsize_t chunk_dims[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; + hsize_t sel_dims[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; + hsize_t start[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; + hsize_t stride[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; + hsize_t count[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; + hsize_t block[INTERLEAVED_WRITE_FILTERED_DATASET_DIMS]; size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to shared filtered chunks in 3D dataset"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + HDputs("Testing interleaved write to filtered chunks"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_NROWS; - dataset_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS; - dataset_dims[2] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH; - chunk_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; - chunk_dims[2] = 1; - sel_dims[0] = (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_NROWS / mpi_size); - sel_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS; - sel_dims[2] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH; + dataset_dims[0] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NROWS; + dataset_dims[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS; + chunk_dims[0] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NROWS; + chunk_dims[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NCOLS; + sel_dims[0] = (hsize_t)(INTERLEAVED_WRITE_FILTERED_DATASET_NROWS / mpi_size); + sel_dims[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS; - filespace = H5Screate_simple(WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(INTERLEAVED_WRITE_FILTERED_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, sel_dims, NULL); + memspace = H5Screate_simple(INTERLEAVED_WRITE_FILTERED_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, INTERLEAVED_WRITE_FILTERED_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + dset_id = H5Dcreate2(group_id, INTERLEAVED_WRITE_FILTERED_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes * it to the hyperslab in the file */ - count[0] = (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_NROWS / WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NROWS); - count[1] = (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS / WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS); - count[2] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH; - stride[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NROWS; - stride[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; - stride[2] = 1; + count[0] = + (hsize_t)(INTERLEAVED_WRITE_FILTERED_DATASET_NROWS / INTERLEAVED_WRITE_FILTERED_DATASET_CH_NROWS); + count[1] = + (hsize_t)(INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS / INTERLEAVED_WRITE_FILTERED_DATASET_CH_NCOLS); + stride[0] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NROWS; + stride[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NCOLS; block[0] = 1; - block[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; - block[2] = 1; + block[1] = (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_CH_NCOLS; start[0] = (hsize_t)mpi_rank; start[1] = 0; - start[2] = 0; if (VERBOSE_MED) { - HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE - " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE - ", %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE - " ]\n", - mpi_rank, count[0], count[1], count[2], stride[0], stride[1], stride[2], start[0], start[1], - start[2], block[0], block[1], block[2]); + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); HDfflush(stdout); } @@ -1915,8 +2188,8 @@ test_write_3d_filtered_dataset_overlap(void) "Hyperslab selection succeeded"); /* Fill data buffer */ - data_size = sel_dims[0] * sel_dims[1] * sel_dims[2] * sizeof(*data); - correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); data = (C_DATATYPE *)HDcalloc(1, data_size); VRFY((NULL != data), "HDcalloc succeeded"); @@ -1928,30 +2201,25 @@ test_write_3d_filtered_dataset_overlap(void) data[i] = (C_DATATYPE)GEN_DATA(i); for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - /* Add the Column Index */ - correct_buf[i] = (C_DATATYPE)( - (i % (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS)) - - /* Add the Row Index */ - + ((i % (hsize_t)(mpi_size * WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * - WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS)) / - (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS)) - - /* Add the amount that gets added when a rank moves down to its next - section vertically in the dataset */ - + ((hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS) * - (i / (hsize_t)(mpi_size * WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * - WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS)))); + /* Add Column Index */ + correct_buf[i] = + (C_DATATYPE)((i % (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS) - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + /* Add the Row Index */ + + ((i % (hsize_t)(mpi_size * INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS)) / + (hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS) - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + /* Add the amount that gets added when a rank moves down to its next section + vertically in the dataset */ + + ((hsize_t)INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS * + (i / (hsize_t)(mpi_size * INTERLEAVED_WRITE_FILTERED_DATASET_NCOLS)))); - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + if (data) HDfree(data); @@ -1961,10 +2229,10 @@ test_write_3d_filtered_dataset_overlap(void) read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, INTERLEAVED_WRITE_FILTERED_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -1974,116 +2242,106 @@ test_write_3d_filtered_dataset_overlap(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of filtered data to unshared - * chunks using a compound datatype which doesn't - * require a datatype conversion. + * Tests parallel write of transformed and filtered data + * in the case where only one process is writing to a + * particular chunk in the operation. Normally, a data + * transform function will cause the parallel library to + * break to independent I/O and this isn't allowed when + * there are filters in the pipeline. However, in this + * case the parallel library recognizes that the used + * data transform function "x" is the same as not applying + * the transform function. Therefore it does not apply + * the transform function resulting in not breaking to + * independent I/O. * - * Programmer: Jordan Henderson - * 02/10/2017 + * Programmer: Jan-Willem Blokland + * 08/20/2021 */ static void -test_write_cmpd_filtered_dataset_no_conversion_unshared(void) +test_write_transformed_filtered_dataset_no_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - COMPOUND_C_DATATYPE *data = NULL; - COMPOUND_C_DATATYPE *read_buf = NULL; - COMPOUND_C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t sel_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t start[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t stride[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t count[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t block[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - size_t i, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1, memtype = -1; - hid_t filespace = -1, memspace = -1; + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t start[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t stride[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t count[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t block[WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + size_t i, data_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to unshared filtered chunks in Compound Datatype dataset without Datatype " - "conversion"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + HDputs("Testing write to unshared transformed and filtered chunks"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_NROWS; - dataset_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_NCOLS; - chunk_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; - chunk_dims[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; - sel_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; - sel_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + dataset_dims[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; + sel_dims[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; + sel_dims[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS; - filespace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, - dataset_dims, NULL); + filespace = H5Screate_simple(WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = - H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, sel_dims, NULL); + memspace = H5Screate_simple(WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, - chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); - - /* Create the compound type for memory. */ - memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); - VRFY((memtype >= 0), "Datatype creation succeeded"); - - VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), - "Datatype insertion succeeded"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_NAME, memtype, - filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_NAME, + HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes * it to the hyperslab in the file */ - count[0] = 1; - count[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; - stride[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; - stride[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; - block[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; - block[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; - start[0] = 0; - start[1] = ((hsize_t)mpi_rank * WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS); + count[0] = 1; + count[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS / + (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; + stride[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; + block[1] = (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; + start[0] = ((hsize_t)mpi_rank * (hsize_t)WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS * count[0]); + start[1] = 0; if (VERBOSE_MED) { HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE @@ -2100,37 +2358,32 @@ test_write_cmpd_filtered_dataset_no_conversion_unshared(void) VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - data = (COMPOUND_C_DATATYPE *)HDcalloc( - 1, (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC * sizeof(*data)); - VRFY((NULL != data), "HDcalloc succeeded"); + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(COMPOUND_C_DATATYPE); + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); - correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != correct_buf), "HDcalloc succeeded"); - /* Fill data buffer */ - for (i = 0; i < (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; i++) { - data[i].field1 = (short)GEN_DATA(i); - data[i].field2 = (int)GEN_DATA(i); - data[i].field3 = (long)GEN_DATA(i); - } - - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { - correct_buf[i].field1 = (short)((i % dataset_dims[1]) + (i / dataset_dims[1])); - - correct_buf[i].field2 = (int)((i % dataset_dims[1]) + (i / dataset_dims[1])); + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); - correct_buf[i].field3 = (long)((i % dataset_dims[1]) + (i / dataset_dims[1])); - } + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + + (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + /* Create property list for data transform */ + plist_id = H5Pcopy(dxpl_id); + VRFY((plist_id >= 0), "DXPL copy succeeded"); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + /* Set data transform expression */ + VRFY((H5Pset_data_transform(plist_id, "x") >= 0), "Set data transform expression succeeded"); - VRFY((H5Dwrite(dset_id, memtype, memspace, filespace, plist_id, data) >= 0), "Dataset write succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + "Dataset write succeeded"); if (data) HDfree(data); @@ -2138,17 +2391,24 @@ test_write_cmpd_filtered_dataset_no_conversion_unshared(void) VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); /* Verify the correct data was written */ - read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_NAME, - H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), "Dataset read succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + + /* Verify space allocation status */ + plist_id = H5Dget_create_plist(dset_id); + VRFY((plist_id >= 0), "H5Dget_create_plist succeeded"); + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + if (correct_buf) HDfree(correct_buf); if (read_buf) @@ -2157,120 +2417,110 @@ test_write_cmpd_filtered_dataset_no_conversion_unshared(void) VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Tclose(memtype) >= 0), "Datatype close succeeded"); VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of filtered data to shared - * chunks using a compound datatype which doesn't - * require a datatype conversion. + * Tests parallel write of filtered data in the case where + * the dataset has 3 dimensions and each process writes + * to its own "page" in the 3rd dimension. * * Programmer: Jordan Henderson - * 02/10/2017 + * 02/06/2017 */ static void -test_write_cmpd_filtered_dataset_no_conversion_shared(void) +test_write_3d_filtered_dataset_no_overlap_separate_pages(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - COMPOUND_C_DATATYPE *data = NULL; - COMPOUND_C_DATATYPE *read_buf = NULL; - COMPOUND_C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t sel_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t start[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t stride[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t count[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t block[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - size_t i, correct_buf_size; - hid_t file_id, dset_id, plist_id, memtype; - hid_t filespace, memspace; + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t sel_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t start[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t stride[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t count[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t block[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + size_t i, data_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to shared filtered chunks in Compound Datatype dataset without Datatype " - "conversion"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + HDputs("Testing write to unshared filtered chunks on separate pages in 3D dataset"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_NROWS; - dataset_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_NCOLS; - chunk_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS; - chunk_dims[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; - sel_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; - sel_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; + dataset_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS; + dataset_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS; + dataset_dims[2] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DEPTH; + chunk_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; + chunk_dims[2] = 1; + sel_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS; + sel_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS; + sel_dims[2] = 1; - filespace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, - dataset_dims, NULL); + filespace = H5Screate_simple(WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = - H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, sel_dims, NULL); + memspace = H5Screate_simple(WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, - chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); - - /* Create the compound type for memory. */ - memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); - VRFY((memtype >= 0), "Datatype creation succeeded"); - - VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), - "Datatype insertion succeeded"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_NAME, memtype, - filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_NAME, + HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes * it to the hyperslab in the file */ - count[0] = 1; - count[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; - stride[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS; - stride[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; - block[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; - block[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; - start[0] = (hsize_t)mpi_rank; - start[1] = 0; - + count[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS / + (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; + count[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS / + (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; + count[2] = 1; + stride[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; + stride[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; + stride[2] = 1; + block[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; + block[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; + block[2] = 1; + start[0] = 0; + start[1] = 0; + start[2] = (hsize_t)mpi_rank; + if (VERBOSE_MED) { - HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE - ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE - ", %" PRIuHSIZE " ]\n", - mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE + " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE + ", %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE + " ]\n", + mpi_rank, count[0], count[1], count[2], stride[0], stride[1], stride[2], start[0], start[1], + start[2], block[0], block[1], block[2]); HDfflush(stdout); } @@ -2281,43 +2531,27 @@ test_write_cmpd_filtered_dataset_no_conversion_shared(void) VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - data = (COMPOUND_C_DATATYPE *)HDcalloc( - 1, (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC * sizeof(*data)); - VRFY((NULL != data), "HDcalloc succeeded"); - - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(COMPOUND_C_DATATYPE); - - correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); - /* Fill data buffer */ - for (i = 0; i < (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; i++) { - data[i].field1 = (short)GEN_DATA(i); - data[i].field2 = (int)GEN_DATA(i); - data[i].field3 = (long)GEN_DATA(i); - } + data_size = sel_dims[0] * sel_dims[1] * sel_dims[2] * sizeof(*data); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { - correct_buf[i].field1 = - (short)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); - correct_buf[i].field2 = - (int)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); - correct_buf[i].field3 = - (long)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); - } + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + correct_buf[i] = (C_DATATYPE)((i % (hsize_t)mpi_size) + (i / (hsize_t)mpi_size)); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - VRFY((H5Dwrite(dset_id, memtype, memspace, filespace, plist_id, data) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); if (data) HDfree(data); @@ -2325,14 +2559,14 @@ test_write_cmpd_filtered_dataset_no_conversion_shared(void) VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); /* Verify the correct data was written */ - read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = - H5Dopen2(file_id, "/" WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), "Dataset read succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -2341,136 +2575,114 @@ test_write_cmpd_filtered_dataset_no_conversion_shared(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Tclose(memtype) >= 0), "Datatype close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of filtered data to unshared - * chunks using a compound datatype which requires a - * datatype conversion. - * - * NOTE: This test currently should fail because the - * datatype conversion causes the parallel library to - * break to independent I/O and this isn't allowed when - * there are filters in the pipeline. + * Tests parallel write of filtered data in the case where + * the dataset has 3 dimensions and each process writes + * to each "page" in the 3rd dimension. However, no chunk + * on a given "page" is written to by more than one process. * * Programmer: Jordan Henderson - * 02/07/2017 + * 02/06/2017 */ static void -test_write_cmpd_filtered_dataset_type_conversion_unshared(void) +test_write_3d_filtered_dataset_no_overlap_same_pages(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - COMPOUND_C_DATATYPE *data = NULL; - COMPOUND_C_DATATYPE *read_buf = NULL; - COMPOUND_C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t sel_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t start[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t stride[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t count[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t block[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - size_t i, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1, filetype = -1, memtype = -1; - hid_t filespace = -1, memspace = -1; + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t sel_dims[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t start[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t stride[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t count[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t block[WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + size_t i, data_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write to unshared filtered chunks in Compound Datatype dataset with Datatype " - "conversion"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + HDputs("Testing write to unshared filtered chunks on the same pages in 3D dataset"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_NROWS; - dataset_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_NCOLS; - chunk_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; - chunk_dims[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; - sel_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; - sel_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + dataset_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NROWS; + dataset_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS; + dataset_dims[2] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DEPTH; + chunk_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; + chunk_dims[2] = 1; + sel_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; + sel_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS; + sel_dims[2] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DEPTH; - filespace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, - dataset_dims, NULL); + filespace = + H5Screate_simple(WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, - sel_dims, NULL); + memspace = H5Screate_simple(WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, - chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); - - /* Create the compound type for memory. */ - memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); - VRFY((memtype >= 0), "Datatype creation succeeded"); - - VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), - "Datatype insertion succeeded"); - - /* Create the compound type for file. */ - filetype = H5Tcreate(H5T_COMPOUND, 32); - VRFY((filetype >= 0), "Datatype creation succeeded"); - - VRFY((H5Tinsert(filetype, "ShortData", 0, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - VRFY((H5Tinsert(filetype, "IntData", 8, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - VRFY((H5Tinsert(filetype, "LongData", 16, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_NAME, - filetype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_NAME, + HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes * it to the hyperslab in the file */ - count[0] = 1; - count[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; - stride[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; - stride[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; - block[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; - block[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; - start[0] = 0; - start[1] = ((hsize_t)mpi_rank * WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS); + count[0] = 1; + count[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS / + (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; + count[2] = (hsize_t)mpi_size; + stride[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; + stride[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; + stride[2] = 1; + block[0] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; + block[1] = (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; + block[2] = 1; + start[0] = ((hsize_t)mpi_rank * (hsize_t)WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS * count[0]); + start[1] = 0; + start[2] = 0; if (VERBOSE_MED) { - HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE - ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE - ", %" PRIuHSIZE " ]\n", - mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE + " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE + ", %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE + " ]\n", + mpi_rank, count[0], count[1], count[2], stride[0], stride[1], stride[2], start[0], start[1], + start[2], block[0], block[1], block[2]); HDfflush(stdout); } @@ -2481,50 +2693,43 @@ test_write_cmpd_filtered_dataset_type_conversion_unshared(void) VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - data = (COMPOUND_C_DATATYPE *)HDcalloc( - 1, (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC * sizeof(*data)); - VRFY((NULL != data), "HDcalloc succeeded"); + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sel_dims[2] * sizeof(*data); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(COMPOUND_C_DATATYPE); + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); - correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != correct_buf), "HDcalloc succeeded"); - /* Fill data buffer */ - for (i = 0; i < (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; i++) { - data[i].field1 = (short)GEN_DATA(i); - data[i].field2 = (int)GEN_DATA(i); - data[i].field3 = (long)GEN_DATA(i); - } + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] * dataset_dims[1])) + + (i / (dataset_dims[0] * dataset_dims[1]))); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - /* Ensure that this test currently fails since type conversions break collective mode */ - H5E_BEGIN_TRY - { - VRFY((H5Dwrite(dset_id, memtype, memspace, filespace, plist_id, data) < 0), - "Dataset write succeeded"); - } - H5E_END_TRY; + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); if (data) HDfree(data); - /* Verify that no data was written */ VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + /* Verify the correct data was written */ + read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_NAME, - H5P_DEFAULT); + dset_id = H5Dopen2(group_id, WRITE_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), "Dataset read succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -2533,93 +2738,260 @@ test_write_cmpd_filtered_dataset_type_conversion_unshared(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Tclose(filetype) >= 0), "File datatype close succeeded"); - VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel write of filtered data to shared - * chunks using a compound datatype which requires - * a datatype conversion. - * - * NOTE: This test currently should fail because the - * datatype conversion causes the parallel library to - * break to independent I/O and this isn't allowed when - * there are filters in the pipeline. + * Tests parallel write of filtered data in the case where + * the dataset has 3 dimensions and each process writes + * to each "page" in the 3rd dimension. Further, each chunk + * in each "page" is written to equally by all processes. * * Programmer: Jordan Henderson - * 02/10/2017 + * 02/06/2017 */ static void -test_write_cmpd_filtered_dataset_type_conversion_shared(void) +test_write_3d_filtered_dataset_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { - COMPOUND_C_DATATYPE *data = NULL; - COMPOUND_C_DATATYPE *read_buf = NULL; - COMPOUND_C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t sel_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t start[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t stride[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t count[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t block[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - size_t i, correct_buf_size; - hid_t file_id, dset_id, plist_id, filetype, memtype; - hid_t filespace, memspace; - - if (MAINPROCESS) - HDputs( - "Testing write to shared filtered chunks in Compound Datatype dataset with Datatype conversion"); - - CHECK_CUR_FILTER_AVAIL(); - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t sel_dims[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t start[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t stride[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t count[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t block[WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + size_t i, data_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + + if (MAINPROCESS) + HDputs("Testing write to shared filtered chunks in 3D dataset"); - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_NROWS; + dataset_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS; + dataset_dims[2] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH; + chunk_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; + chunk_dims[2] = 1; + sel_dims[0] = (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_NROWS / mpi_size); + sel_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS; + sel_dims[2] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH; + + filespace = H5Screate_simple(WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + memspace = H5Screate_simple(WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, sel_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_NAME, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_NROWS / WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NROWS); + count[1] = (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS / WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS); + count[2] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH; + stride[0] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NROWS; + stride[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; + stride[2] = 1; + block[0] = 1; + block[1] = (hsize_t)WRITE_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; + block[2] = 1; + start[0] = (hsize_t)mpi_rank; + start[1] = 0; + start[2] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE + " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE + ", %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE + " ]\n", + mpi_rank, count[0], count[1], count[2], stride[0], stride[1], stride[2], start[0], start[1], + start[2], block[0], block[1], block[2]); + HDfflush(stdout); + } + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sel_dims[2] * sizeof(*data); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); + + correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); + + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); + + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + /* Add the Column Index */ + correct_buf[i] = (C_DATATYPE)( + (i % (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS)) + + /* Add the Row Index */ + + ((i % (hsize_t)(mpi_size * WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * + WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS)) / + (hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS)) + + /* Add the amount that gets added when a rank moves down to its next + section vertically in the dataset */ + + ((hsize_t)(WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS) * + (i / (hsize_t)(mpi_size * WRITE_SHARED_FILTERED_CHUNKS_3D_DEPTH * + WRITE_SHARED_FILTERED_CHUNKS_3D_NCOLS)))); + + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + if (data) + HDfree(data); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + + /* Verify the correct data was written */ + read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + dset_id = H5Dopen2(group_id, WRITE_SHARED_FILTERED_CHUNKS_3D_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + + VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + if (correct_buf) + HDfree(correct_buf); + if (read_buf) + HDfree(read_buf); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + return; +} + +/* + * Tests parallel write of filtered data to unshared + * chunks using a compound datatype which doesn't + * require a datatype conversion. + * + * Programmer: Jordan Henderson + * 02/10/2017 + */ +static void +test_write_cmpd_filtered_dataset_no_conversion_unshared(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) +{ + COMPOUND_C_DATATYPE *data = NULL; + COMPOUND_C_DATATYPE *read_buf = NULL; + COMPOUND_C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t sel_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t start[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t stride[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t count[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t block[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + size_t i, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID, + memtype = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + + if (MAINPROCESS) + HDputs("Testing write to unshared filtered chunks in Compound Datatype dataset without Datatype " + "conversion"); + + /* SZIP and ScaleOffset filters don't support compound types */ + if (filter_id == H5Z_FILTER_SZIP || filter_id == H5Z_FILTER_SCALEOFFSET) { + if (MAINPROCESS) + SKIPPED(); + return; + } - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_NROWS; - dataset_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_NCOLS; - chunk_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS; - chunk_dims[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; - sel_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; - sel_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; + dataset_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_NROWS; + dataset_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_NCOLS; + chunk_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; + chunk_dims[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; + sel_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; + sel_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; - filespace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, + filespace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); memspace = - H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, sel_dims, NULL); + H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, + VRFY((H5Pset_chunk(plist_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); /* Create the compound type for memory. */ memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); @@ -2632,32 +3004,26 @@ test_write_cmpd_filtered_dataset_type_conversion_shared(void) VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), "Datatype insertion succeeded"); - /* Create the compound type for file. */ - filetype = H5Tcreate(H5T_COMPOUND, 32); - VRFY((filetype >= 0), "Datatype creation succeeded"); - - VRFY((H5Tinsert(filetype, "ShortData", 0, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - VRFY((H5Tinsert(filetype, "IntData", 8, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - VRFY((H5Tinsert(filetype, "LongData", 16, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - - dset_id = H5Dcreate2(file_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_NAME, - filetype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_NAME, + memtype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* Each process defines the dataset selection in memory and writes * it to the hyperslab in the file */ count[0] = 1; - count[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; - stride[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS; - stride[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; - block[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; - block[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; - start[0] = (hsize_t)mpi_rank; - start[1] = 0; + count[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + stride[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; + stride[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; + block[0] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; + block[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; + start[0] = 0; + start[1] = ((hsize_t)mpi_rank * WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS); if (VERBOSE_MED) { HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE @@ -2675,7 +3041,7 @@ test_write_cmpd_filtered_dataset_type_conversion_shared(void) "Hyperslab selection succeeded"); data = (COMPOUND_C_DATATYPE *)HDcalloc( - 1, (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC * sizeof(*data)); + 1, (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC * sizeof(*data)); VRFY((NULL != data), "HDcalloc succeeded"); correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(COMPOUND_C_DATATYPE); @@ -2684,40 +3050,39 @@ test_write_cmpd_filtered_dataset_type_conversion_shared(void) VRFY((NULL != correct_buf), "HDcalloc succeeded"); /* Fill data buffer */ - for (i = 0; i < (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; i++) { + for (i = 0; i < (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; i++) { data[i].field1 = (short)GEN_DATA(i); data[i].field2 = (int)GEN_DATA(i); data[i].field3 = (long)GEN_DATA(i); } - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { + correct_buf[i].field1 = (short)((i % dataset_dims[1]) + (i / dataset_dims[1])); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + correct_buf[i].field2 = (int)((i % dataset_dims[1]) + (i / dataset_dims[1])); - /* Ensure that this test currently fails since type conversions break collective mode */ - H5E_BEGIN_TRY - { - VRFY((H5Dwrite(dset_id, memtype, memspace, filespace, plist_id, data) < 0), - "Dataset write succeeded"); + correct_buf[i].field3 = (long)((i % dataset_dims[1]) + (i / dataset_dims[1])); } - H5E_END_TRY; + + VRFY((H5Dwrite(dset_id, memtype, memspace, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); if (data) HDfree(data); - /* Verify that no data was written */ VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + /* Verify the correct data was written */ read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_NAME, - H5P_DEFAULT); + dset_id = + H5Dopen2(group_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), "Dataset read succeeded"); + VRFY((H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -2726,689 +3091,642 @@ test_write_cmpd_filtered_dataset_type_conversion_shared(void) if (read_buf) HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Tclose(filetype) >= 0), "File datatype close succeeded"); - VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Tclose(memtype) >= 0), "Datatype close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } -#endif /* - * Tests parallel read of filtered data in the special - * case where a dataset is composed of a single chunk. - * - * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads a part of - * the singular chunk and contributes its piece to a - * global buffer that is checked for consistency. + * Tests parallel write of filtered data to shared + * chunks using a compound datatype which doesn't + * require a datatype conversion. * * Programmer: Jordan Henderson - * 05/14/2018 + * 02/10/2017 */ static void -test_read_one_chunk_filtered_dataset(void) +test_write_cmpd_filtered_dataset_no_conversion_shared(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - C_DATATYPE *read_buf = NULL; - C_DATATYPE *correct_buf = NULL; - C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; - hsize_t chunk_dims[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; - hsize_t sel_dims[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; - hsize_t start[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; - hsize_t stride[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; - hsize_t count[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; - hsize_t block[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; - hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; - int * recvcounts = NULL; - int * displs = NULL; + COMPOUND_C_DATATYPE *data = NULL; + COMPOUND_C_DATATYPE *read_buf = NULL; + COMPOUND_C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t sel_dims[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t start[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t stride[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t count[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t block[WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + size_t i, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID, + memtype = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing read from one-chunk filtered dataset"); - - CHECK_CUR_FILTER_AVAIL(); - - dataset_dims[0] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_NROWS; - dataset_dims[1] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_NCOLS; - - /* Setup the buffer for writing and for comparison */ - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); - - correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); - - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = ((C_DATATYPE)i % (READ_ONE_CHUNK_FILTERED_DATASET_CH_NROWS / mpi_size * - READ_ONE_CHUNK_FILTERED_DATASET_CH_NCOLS)) + - ((C_DATATYPE)i / (READ_ONE_CHUNK_FILTERED_DATASET_CH_NROWS / mpi_size * - READ_ONE_CHUNK_FILTERED_DATASET_CH_NCOLS)); - - if (MAINPROCESS) { - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); - - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); - - /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_ONE_CHUNK_FILTERED_DATASET_DIMS, dataset_dims, NULL); - VRFY((filespace >= 0), "File dataspace creation succeeded"); - - /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_CH_NCOLS; - - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); - - VRFY((H5Pset_chunk(plist_id, READ_ONE_CHUNK_FILTERED_DATASET_DIMS, chunk_dims) >= 0), - "Chunk size set"); - - /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + HDputs("Testing write to shared filtered chunks in Compound Datatype dataset without Datatype " + "conversion"); - dset_id = H5Dcreate2(file_id, READ_ONE_CHUNK_FILTERED_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, - H5P_DEFAULT, plist_id, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset creation succeeded"); + /* SZIP and ScaleOffset filters don't support compound types */ + if (filter_id == H5Z_FILTER_SZIP || filter_id == H5Z_FILTER_SCALEOFFSET) { + if (MAINPROCESS) + SKIPPED(); + return; + } - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), - "Dataset write succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); - } + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_NROWS; + dataset_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS; + chunk_dims[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; + sel_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; + sel_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + filespace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, + dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + memspace = + H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, sel_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + VRFY((H5Pset_chunk(plist_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, + chunk_dims) >= 0), + "Chunk size set"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dopen2(file_id, "/" READ_ONE_CHUNK_FILTERED_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + /* Create the compound type for memory. */ + memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); + VRFY((memtype >= 0), "Datatype creation succeeded"); - sel_dims[0] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_NROWS / (hsize_t)mpi_size; - sel_dims[1] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_NCOLS; + VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), + "Datatype insertion succeeded"); - /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ - flat_dims[0] = sel_dims[0] * sel_dims[1]; + dset_id = H5Dcreate2(group_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_NAME, memtype, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); - memspace = H5Screate_simple(1, flat_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - /* Select hyperslab in the file */ - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - /* - * Each process defines the dataset selection in the file and - * reads it to the selection in memory + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file */ count[0] = 1; - count[1] = 1; - stride[0] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_CH_NROWS; - stride[1] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_CH_NCOLS; - block[0] = sel_dims[0]; - block[1] = sel_dims[1]; - start[0] = ((hsize_t)mpi_rank * sel_dims[0]); + count[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; + stride[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS; + stride[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; + block[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; + block[1] = WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; + start[0] = (hsize_t)mpi_rank; start[1] = 0; if (VERBOSE_MED) { - HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE " ]\n", mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); HDfflush(stdout); } + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + data = (COMPOUND_C_DATATYPE *)HDcalloc( + 1, (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC * sizeof(*data)); + VRFY((NULL != data), "HDcalloc succeeded"); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(COMPOUND_C_DATATYPE); - read_buf_size = flat_dims[0] * sizeof(*read_buf); + correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); - read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); + /* Fill data buffer */ + for (i = 0; i < (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; i++) { + data[i].field1 = (short)GEN_DATA(i); + data[i].field2 = (int)GEN_DATA(i); + data[i].field3 = (long)GEN_DATA(i); + } - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), - "Dataset read succeeded"); + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { + correct_buf[i].field1 = + (short)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); - global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != global_buf), "HDcalloc succeeded"); + correct_buf[i].field2 = + (int)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); - /* Collect each piece of data from all ranks into a global buffer on all ranks */ - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); + correct_buf[i].field3 = + (long)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + } - for (i = 0; i < (size_t)mpi_size; i++) - recvcounts[i] = (int)flat_dims[0]; + VRFY((H5Dwrite(dset_id, memtype, memspace, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * flat_dims[0]); + if (data) + HDfree(data); - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, recvcounts, - displs, C_DATATYPE_MPI, comm)), - "MPI_Allgatherv succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + /* Verify the correct data was written */ + read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + dset_id = + H5Dopen2(group_id, WRITE_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + VRFY((H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); + + VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); - if (displs) - HDfree(displs); - if (recvcounts) - HDfree(recvcounts); - if (global_buf) - HDfree(global_buf); - if (read_buf) - HDfree(read_buf); if (correct_buf) HDfree(correct_buf); + if (read_buf) + HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Tclose(memtype) >= 0), "Datatype close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel read of filtered data in the case where only - * one process is reading from a particular chunk in the operation. + * Tests parallel write of filtered data to unshared + * chunks using a compound datatype which requires a + * datatype conversion. * - * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads a part of - * the dataset and contributes its piece to a global buffer - * that is checked for consistency. + * NOTE: This test currently should fail for mpi_size > 1 + * because the datatype conversion causes the parallel + * library to break to independent I/O and this isn't + * allowed when there are filters in the pipeline, + * unless there is only one MPI rank. * * Programmer: Jordan Henderson - * 05/15/2018 + * 02/07/2017 */ static void -test_read_filtered_dataset_no_overlap(void) +test_write_cmpd_filtered_dataset_type_conversion_unshared(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - C_DATATYPE *read_buf = NULL; - C_DATATYPE *correct_buf = NULL; - C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t start[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t stride[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t count[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t block[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; - int * recvcounts = NULL; - int * displs = NULL; + COMPOUND_C_DATATYPE *data = NULL; + COMPOUND_C_DATATYPE *read_buf = NULL; + COMPOUND_C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t sel_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t start[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t stride[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t count[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t block[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + size_t i, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID, + filetype = H5I_INVALID_HID, memtype = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing read from unshared filtered chunks"); + HDputs("Testing write to unshared filtered chunks in Compound Datatype dataset with Datatype " + "conversion"); - CHECK_CUR_FILTER_AVAIL(); + /* Skip for MPI communicator size of 1 */ + if (mpi_size == 1) { + SKIPPED(); + return; + } - dataset_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NCOLS; + /* SZIP and ScaleOffset filters don't support compound types */ + if (filter_id == H5Z_FILTER_SZIP || filter_id == H5Z_FILTER_SCALEOFFSET) { + if (MAINPROCESS) + SKIPPED(); + return; + } - /* Setup the buffer for writing and for comparison */ - correct_buf_size = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NROWS * - (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NCOLS * sizeof(*correct_buf); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); - correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); - - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + - (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); - - if (MAINPROCESS) { - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); - - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); - - /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); - VRFY((filespace >= 0), "File dataspace creation succeeded"); - - /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NCOLS; - - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); - - VRFY((H5Pset_chunk(plist_id, READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), - "Chunk size set"); - - /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - dset_id = H5Dcreate2(file_id, READ_UNSHARED_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, - filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset creation succeeded"); - - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + /* Create the dataspace for the dataset */ + dataset_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_NROWS; + dataset_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_NCOLS; + chunk_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; + chunk_dims[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; + sel_dims[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; + sel_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), - "Dataset write succeeded"); + filespace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, + dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); - } + memspace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, + sel_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + VRFY((H5Pset_chunk(plist_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, + chunk_dims) >= 0), + "Chunk size set"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + /* Create the compound type for memory. */ + memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); + VRFY((memtype >= 0), "Datatype creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), + "Datatype insertion succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_UNSHARED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + /* Create the compound type for file. */ + filetype = H5Tcreate(H5T_COMPOUND, 32); + VRFY((filetype >= 0), "Datatype creation succeeded"); - sel_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS; - sel_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NCOLS; + VRFY((H5Tinsert(filetype, "ShortData", 0, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((H5Tinsert(filetype, "IntData", 8, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((H5Tinsert(filetype, "LongData", 16, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ - flat_dims[0] = sel_dims[0] * sel_dims[1]; + dset_id = H5Dcreate2(group_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_NAME, + filetype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); - memspace = H5Screate_simple(1, flat_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - /* Select hyperslab in the file */ - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - /* - * Each process defines the dataset selection in the file and reads - * it to the selection in memory + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file */ - count[0] = 1; - count[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NCOLS / (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NCOLS; - stride[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS; - stride[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NCOLS; - block[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS; - block[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NCOLS; - start[0] = ((hsize_t)mpi_rank * (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS * count[0]); - start[1] = 0; + count[0] = 1; + count[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + stride[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; + stride[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; + block[0] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; + block[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; + start[0] = 0; + start[1] = ((hsize_t)mpi_rank * WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS); if (VERBOSE_MED) { - HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE " ]\n", mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); HDfflush(stdout); } + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + data = (COMPOUND_C_DATATYPE *)HDcalloc( + 1, (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC * sizeof(*data)); + VRFY((NULL != data), "HDcalloc succeeded"); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(COMPOUND_C_DATATYPE); - read_buf_size = flat_dims[0] * sizeof(*read_buf); + correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); - read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); + /* Fill data buffer */ + for (i = 0; i < (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; i++) { + data[i].field1 = (short)GEN_DATA(i); + data[i].field2 = (int)GEN_DATA(i); + data[i].field3 = (long)GEN_DATA(i); + } - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), - "Dataset read succeeded"); + /* Ensure that this test currently fails since type conversions break collective mode */ + H5E_BEGIN_TRY + { + VRFY((H5Dwrite(dset_id, memtype, memspace, filespace, dxpl_id, data) < 0), "Dataset write succeeded"); + } + H5E_END_TRY; - global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != global_buf), "HDcalloc succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, NO_CHUNKS_WRITTEN); - /* Collect each piece of data from all ranks into a global buffer on all ranks */ - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); + if (data) + HDfree(data); - for (i = 0; i < (size_t)mpi_size; i++) - recvcounts[i] = (int)flat_dims[0]; + /* Verify that no data was written */ + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * flat_dims[0]); + dset_id = + H5Dopen2(group_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, recvcounts, - displs, C_DATATYPE_MPI, comm)), - "MPI_Allgatherv succeeded"); + VRFY((H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); - VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); - if (displs) - HDfree(displs); - if (recvcounts) - HDfree(recvcounts); - if (global_buf) - HDfree(global_buf); - if (read_buf) - HDfree(read_buf); if (correct_buf) HDfree(correct_buf); + if (read_buf) + HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Tclose(filetype) >= 0), "File datatype close succeeded"); + VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel read of filtered data in the case where - * more than one process is reading from a particular chunk - * in the operation. + * Tests parallel write of filtered data to shared + * chunks using a compound datatype which requires + * a datatype conversion. * - * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads a part of - * each chunk of the dataset and contributes its pieces - * to a global buffer that is checked for consistency. + * NOTE: This test currently should fail for mpi_size > 1 + * because the datatype conversion causes the parallel + * library to break to independent I/O and this isn't + * allowed when there are filters in the pipeline, + * unless there is only one MPI rank. * * Programmer: Jordan Henderson - * 05/15/2018 + * 02/10/2017 */ static void -test_read_filtered_dataset_overlap(void) +test_write_cmpd_filtered_dataset_type_conversion_shared(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - C_DATATYPE *read_buf = NULL; - C_DATATYPE *correct_buf = NULL; - C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t start[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t stride[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t count[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t block[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; - int * recvcounts = NULL; - int * displs = NULL; + COMPOUND_C_DATATYPE *data = NULL; + COMPOUND_C_DATATYPE *read_buf = NULL; + COMPOUND_C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t sel_dims[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t start[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t stride[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t count[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t block[WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + size_t i, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t filetype = H5I_INVALID_HID, memtype = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing read from shared filtered chunks"); + HDputs( + "Testing write to shared filtered chunks in Compound Datatype dataset with Datatype conversion"); - CHECK_CUR_FILTER_AVAIL(); + /* Skip for MPI communicator size of 1 */ + if (mpi_size == 1) { + SKIPPED(); + return; + } - dataset_dims[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_NCOLS; + /* SZIP and ScaleOffset filters don't support compound types */ + if (filter_id == H5Z_FILTER_SZIP || filter_id == H5Z_FILTER_SCALEOFFSET) { + if (MAINPROCESS) + SKIPPED(); + return; + } - /* Setup the buffer for writing and for comparison */ - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); - correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)( - (dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_NROWS; + dataset_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS; + chunk_dims[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; + sel_dims[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; + sel_dims[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; - if (MAINPROCESS) { - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + filespace = H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, + dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + memspace = + H5Screate_simple(WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, sel_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + VRFY((H5Pset_chunk(plist_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, + chunk_dims) >= 0), + "Chunk size set"); - /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); - VRFY((filespace >= 0), "File dataspace creation succeeded"); - - /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS; - - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); - - VRFY((H5Pset_chunk(plist_id, READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), - "Chunk size set"); - - /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); - - dset_id = H5Dcreate2(file_id, READ_SHARED_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, - H5P_DEFAULT, plist_id, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset creation succeeded"); - - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), - "Dataset write succeeded"); - - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); - } - - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + /* Create the compound type for memory. */ + memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); + VRFY((memtype >= 0), "Datatype creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), + "Datatype insertion succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_SHARED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + /* Create the compound type for file. */ + filetype = H5Tcreate(H5T_COMPOUND, 32); + VRFY((filetype >= 0), "Datatype creation succeeded"); - sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; - sel_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS * (hsize_t)DIM1_SCALE_FACTOR; + VRFY((H5Tinsert(filetype, "ShortData", 0, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((H5Tinsert(filetype, "IntData", 8, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((H5Tinsert(filetype, "LongData", 16, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ - flat_dims[0] = sel_dims[0] * sel_dims[1]; + dset_id = H5Dcreate2(group_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_NAME, + filetype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); - memspace = H5Screate_simple(1, flat_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - /* Select hyperslab in the file */ - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - /* - * Each process defines the dataset selection in the file and - * reads it to the selection in memory + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file */ - count[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_NROWS / (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NROWS; - count[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_NCOLS / (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS; - stride[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NROWS; - stride[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS; - block[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NROWS / (hsize_t)mpi_size; - block[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS; - start[0] = (hsize_t)mpi_rank * block[0]; + count[0] = 1; + count[1] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; + stride[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS; + stride[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; + block[0] = (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; + block[1] = WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; + start[0] = (hsize_t)mpi_rank; start[1] = 0; if (VERBOSE_MED) { - HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE " ]\n", mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); HDfflush(stdout); } + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - read_buf_size = flat_dims[0] * sizeof(*read_buf); + data = (COMPOUND_C_DATATYPE *)HDcalloc( + 1, (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC * sizeof(*data)); + VRFY((NULL != data), "HDcalloc succeeded"); - read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(COMPOUND_C_DATATYPE); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), - "Dataset read succeeded"); + correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); - global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != global_buf), "HDcalloc succeeded"); + /* Fill data buffer */ + for (i = 0; i < (hsize_t)WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; i++) { + data[i].field1 = (short)GEN_DATA(i); + data[i].field2 = (int)GEN_DATA(i); + data[i].field3 = (long)GEN_DATA(i); + } - /* - * Since these chunks are shared, run multiple rounds of MPI_Allgatherv - * to collect all of the pieces into their appropriate locations. The - * number of times MPI_Allgatherv is run should be equal to the number - * of chunks in the first dimension of the dataset. - */ + /* Ensure that this test currently fails since type conversions break collective mode */ + H5E_BEGIN_TRY { - size_t loop_count = count[0]; - size_t total_recvcounts = 0; + VRFY((H5Dwrite(dset_id, memtype, memspace, filespace, dxpl_id, data) < 0), "Dataset write succeeded"); + } + H5E_END_TRY; - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, NO_CHUNKS_WRITTEN); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + if (data) + HDfree(data); - for (i = 0; i < (size_t)mpi_size; i++) { - recvcounts[i] = (int)dataset_dims[1]; - total_recvcounts += (size_t)recvcounts[i]; - } + /* Verify that no data was written */ + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * dataset_dims[1]); + read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); - for (; loop_count; loop_count--) { - VRFY((MPI_SUCCESS == MPI_Allgatherv(&read_buf[(count[0] - loop_count) * dataset_dims[1]], - recvcounts[mpi_rank], C_DATATYPE_MPI, - &global_buf[(count[0] - loop_count) * total_recvcounts], - recvcounts, displs, C_DATATYPE_MPI, comm)), - "MPI_Allgatherv succeeded"); - } - } + dset_id = + H5Dopen2(group_id, WRITE_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + VRFY((H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); + + VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); - if (displs) - HDfree(displs); - if (recvcounts) - HDfree(recvcounts); - if (global_buf) - HDfree(global_buf); - if (read_buf) - HDfree(read_buf); if (correct_buf) HDfree(correct_buf); + if (read_buf) + HDfree(read_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Tclose(filetype) >= 0), "File datatype close succeeded"); + VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } +#endif /* - * Tests parallel read of filtered data in the case where - * a single process in the read operation has no selection - * in the dataset's dataspace. + * Tests parallel read of filtered data in the special + * case where a dataset is composed of a single chunk. * * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank (except for one) - * reads a part of the dataset and contributes its piece - * to a global buffer that is checked for consistency. + * data to the dataset. Then, each rank reads a part of + * the singular chunk and contributes its piece to a + * global buffer that is checked for consistency. * * Programmer: Jordan Henderson - * 05/15/2018 + * 05/14/2018 */ static void -test_read_filtered_dataset_single_no_selection(void) +test_read_one_chunk_filtered_dataset(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t start[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t stride[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t count[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t block[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t dataset_dims[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; + hsize_t chunk_dims[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; + hsize_t sel_dims[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; + hsize_t start[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; + hsize_t stride[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; + hsize_t count[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; + hsize_t block[READ_ONE_CHUNK_FILTERED_DATASET_DIMS]; hsize_t flat_dims[1]; size_t i, read_buf_size, correct_buf_size; - size_t segment_length; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; int * recvcounts = NULL; int * displs = NULL; if (MAINPROCESS) - HDputs("Testing read from filtered chunks with a single process having no selection"); - - CHECK_CUR_FILTER_AVAIL(); + HDputs("Testing read from one-chunk filtered dataset"); - dataset_dims[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS; + dataset_dims[0] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_NROWS; + dataset_dims[1] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_NCOLS; /* Setup the buffer for writing and for comparison */ correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); @@ -3417,13 +3735,10 @@ test_read_filtered_dataset_single_no_selection(void) VRFY((NULL != correct_buf), "HDcalloc succeeded"); for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + - (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); - - /* Compute the correct offset into the buffer for the process having no selection and clear it */ - segment_length = dataset_dims[0] * dataset_dims[1] / (hsize_t)mpi_size; - HDmemset(correct_buf + ((size_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC * segment_length), - 0, segment_length * sizeof(*correct_buf)); + correct_buf[i] = ((C_DATATYPE)i % (READ_ONE_CHUNK_FILTERED_DATASET_CH_NROWS / mpi_size * + READ_ONE_CHUNK_FILTERED_DATASET_CH_NCOLS)) + + ((C_DATATYPE)i / (READ_ONE_CHUNK_FILTERED_DATASET_CH_NROWS / mpi_size * + READ_ONE_CHUNK_FILTERED_DATASET_CH_NCOLS)); if (MAINPROCESS) { plist_id = H5Pcreate(H5P_FILE_ACCESS); @@ -3437,60 +3752,58 @@ test_read_filtered_dataset_single_no_selection(void) VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + /* Create the dataspace for the dataset */ - filespace = - H5Screate_simple(READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(READ_ONE_CHUNK_FILTERED_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + chunk_dims[0] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_CH_NCOLS; - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, READ_ONE_CHUNK_FILTERED_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, - HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, READ_ONE_CHUNK_FILTERED_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); } - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, READ_ONE_CHUNK_FILTERED_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - sel_dims[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - sel_dims[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS; - - if (mpi_rank == READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) - sel_dims[0] = sel_dims[1] = 0; + sel_dims[0] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_NROWS / (hsize_t)mpi_size; + sel_dims[1] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_NCOLS; /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ flat_dims[0] = sel_dims[0] * sel_dims[1]; @@ -3506,14 +3819,13 @@ test_read_filtered_dataset_single_no_selection(void) * Each process defines the dataset selection in the file and * reads it to the selection in memory */ - count[0] = 1; - count[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS / - (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - stride[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - stride[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - block[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - block[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - start[0] = (hsize_t)mpi_rank * (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS * count[0]; + count[0] = 1; + count[1] = 1; + stride[0] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_CH_NROWS; + stride[1] = (hsize_t)READ_ONE_CHUNK_FILTERED_DATASET_CH_NCOLS; + block[0] = sel_dims[0]; + block[1] = sel_dims[1]; + start[0] = ((hsize_t)mpi_rank * sel_dims[0]); start[1] = 0; if (VERBOSE_MED) { @@ -3524,24 +3836,15 @@ test_read_filtered_dataset_single_no_selection(void) HDfflush(stdout); } - if (mpi_rank == READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) - VRFY((H5Sselect_none(filespace) >= 0), "Select none succeeded"); - else - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), - "Hyperslab selection succeeded"); - - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); read_buf_size = flat_dims[0] * sizeof(*read_buf); read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); @@ -3552,25 +3855,17 @@ test_read_filtered_dataset_single_no_selection(void) VRFY((NULL != recvcounts), "HDcalloc succeeded"); for (i = 0; i < (size_t)mpi_size; i++) - recvcounts[i] = (int)(READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS * - READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS); - recvcounts[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC] = 0; + recvcounts[i] = (int)flat_dims[0]; displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); VRFY((NULL != displs), "HDcalloc succeeded"); for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * (size_t)(READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS * - READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS)); + displs[i] = (int)(i * flat_dims[0]); - if (mpi_rank == READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, 0, C_DATATYPE_MPI, global_buf, recvcounts, displs, - C_DATATYPE_MPI, comm)), - "MPI_Allgatherv succeeded"); - else - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, - recvcounts, displs, C_DATATYPE_MPI, comm)), - "MPI_Allgatherv succeeded"); + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, recvcounts, + displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -3588,52 +3883,63 @@ test_read_filtered_dataset_single_no_selection(void) VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel read of filtered data in the case where - * no process in the read operation has a selection in the - * dataset's dataspace. This test is to ensure that there - * are no assertion failures or similar issues due to size - * 0 allocations and the like. + * Tests parallel read of filtered data in the case where only + * one process is reading from a particular chunk in the operation. * * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank will simply issue - * a no-op read. + * data to the dataset. Then, each rank reads a part of + * the dataset and contributes its piece to a global buffer + * that is checked for consistency. * * Programmer: Jordan Henderson * 05/15/2018 */ static void -test_read_filtered_dataset_all_no_selection(void) +test_read_filtered_dataset_no_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - size_t read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + C_DATATYPE *global_buf = NULL; + hsize_t dataset_dims[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t start[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t stride[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t count[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t block[READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t flat_dims[1]; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; if (MAINPROCESS) - HDputs("Testing read from filtered chunks with all processes having no selection"); - - CHECK_CUR_FILTER_AVAIL(); + HDputs("Testing read from unshared filtered chunks"); - dataset_dims[0] = (hsize_t)READ_ALL_NO_SELECTION_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)READ_ALL_NO_SELECTION_FILTERED_CHUNKS_NCOLS; + dataset_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NCOLS; /* Setup the buffer for writing and for comparison */ - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + correct_buf_size = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NROWS * + (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NCOLS * sizeof(*correct_buf); correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != correct_buf), "HDcalloc succeeded"); + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + + (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); + if (MAINPROCESS) { plist_id = H5Pcreate(H5P_FILE_ACCESS); VRFY((plist_id >= 0), "FAPL creation succeeded"); @@ -3646,79 +3952,129 @@ test_read_filtered_dataset_all_no_selection(void) VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_ALL_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_ALL_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + chunk_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NCOLS; - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, READ_UNSHARED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, + dset_id = H5Dcreate2(group_id, READ_UNSHARED_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); } - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, READ_UNSHARED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - sel_dims[0] = sel_dims[1] = 0; + sel_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS; + sel_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NCOLS; - memspace = H5Screate_simple(READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); + /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ + flat_dims[0] = sel_dims[0] * sel_dims[1]; + + memspace = H5Screate_simple(1, flat_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Select hyperslab in the file */ filespace = H5Dget_space(dset_id); VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - VRFY((H5Sselect_none(filespace) >= 0), "Select none succeeded"); + /* + * Each process defines the dataset selection in the file and reads + * it to the selection in memory + */ + count[0] = 1; + count[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_NCOLS / (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NCOLS; + stride[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS; + block[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NCOLS; + start[0] = ((hsize_t)mpi_rank * (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_CH_NROWS * count[0]); + start[1] = 0; - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + if (VERBOSE_MED) { + HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - read_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*read_buf); + read_buf_size = flat_dims[0] * sizeof(*read_buf); read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); + global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != global_buf), "HDcalloc succeeded"); + + /* Collect each piece of data from all ranks into a global buffer on all ranks */ + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + recvcounts[i] = (int)flat_dims[0]; + + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * flat_dims[0]); + + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, recvcounts, + displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); + + VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + if (displs) + HDfree(displs); + if (recvcounts) + HDfree(recvcounts); + if (global_buf) + HDfree(global_buf); if (read_buf) HDfree(read_buf); if (correct_buf) @@ -3727,50 +4083,52 @@ test_read_filtered_dataset_all_no_selection(void) VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel read of filtered data by using point - * selections instead of hyperslab selections. + * Tests parallel read of filtered data in the case where + * more than one process is reading from a particular chunk + * in the operation. * * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank will read part - * of the dataset using a point selection and will - * contribute its piece to a global buffer that is - * checked for consistency. + * data to the dataset. Then, each rank reads a part of + * each chunk of the dataset and contributes its pieces + * to a global buffer that is checked for consistency. * * Programmer: Jordan Henderson * 05/15/2018 */ static void -test_read_filtered_dataset_point_selection(void) +test_read_filtered_dataset_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { - C_DATATYPE *correct_buf = NULL; C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; C_DATATYPE *global_buf = NULL; - hsize_t * coords = NULL; - hsize_t dataset_dims[READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t dataset_dims[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t start[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t stride[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t count[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t block[READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS]; hsize_t flat_dims[1]; - size_t i, j, read_buf_size, correct_buf_size; - size_t num_points; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; int * recvcounts = NULL; int * displs = NULL; if (MAINPROCESS) - HDputs("Testing read from filtered chunks with point selection"); - - CHECK_CUR_FILTER_AVAIL(); + HDputs("Testing read from shared filtered chunks"); - dataset_dims[0] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS; + dataset_dims[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_NCOLS; /* Setup the buffer for writing and for comparison */ correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); @@ -3795,56 +4153,58 @@ test_read_filtered_dataset_point_selection(void) VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + chunk_dims[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS; - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, READ_SHARED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, + dset_id = H5Dcreate2(group_id, READ_SHARED_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); } - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, READ_SHARED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - sel_dims[0] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NROWS / (hsize_t)mpi_size; - sel_dims[1] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS; + sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; + sel_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS * (hsize_t)DIM1_SCALE_FACTOR; /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ flat_dims[0] = sel_dims[0] * sel_dims[1]; @@ -3852,37 +4212,40 @@ test_read_filtered_dataset_point_selection(void) memspace = H5Screate_simple(1, flat_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - /* Set up point selection */ + /* Select hyperslab in the file */ filespace = H5Dget_space(dset_id); VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - num_points = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NROWS * - (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS / (hsize_t)mpi_size; - coords = (hsize_t *)HDcalloc(1, 2 * num_points * sizeof(*coords)); - VRFY((NULL != coords), "Coords HDcalloc succeeded"); - - for (i = 0; i < num_points; i++) - for (j = 0; j < READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS; j++) - coords[(i * READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS) + j] = - (j > 0) ? (i % (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS) - : ((hsize_t)mpi_rank + - ((hsize_t)mpi_size * (i / (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS))); - - VRFY((H5Sselect_elements(filespace, H5S_SELECT_SET, (hsize_t)num_points, (const hsize_t *)coords) >= 0), - "Point selection succeeded"); + /* + * Each process defines the dataset selection in the file and + * reads it to the selection in memory + */ + count[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_NROWS / (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NROWS; + count[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_NCOLS / (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS; + stride[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NROWS / (hsize_t)mpi_size; + block[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_CH_NCOLS; + start[0] = (hsize_t)mpi_rank * block[0]; + start[1] = 0; - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + if (VERBOSE_MED) { + HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); read_buf_size = flat_dims[0] * sizeof(*read_buf); read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); @@ -3895,9 +4258,8 @@ test_read_filtered_dataset_point_selection(void) * of chunks in the first dimension of the dataset. */ { - size_t original_loop_count = dataset_dims[0] / (hsize_t)mpi_size; - size_t cur_loop_count = original_loop_count; - size_t total_recvcounts = 0; + size_t loop_count = count[0]; + size_t total_recvcounts = 0; recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); VRFY((NULL != recvcounts), "HDcalloc succeeded"); @@ -3913,12 +4275,11 @@ test_read_filtered_dataset_point_selection(void) for (i = 0; i < (size_t)mpi_size; i++) displs[i] = (int)(i * dataset_dims[1]); - for (; cur_loop_count; cur_loop_count--) { - VRFY((MPI_SUCCESS == - MPI_Allgatherv(&read_buf[(original_loop_count - cur_loop_count) * dataset_dims[1]], - recvcounts[mpi_rank], C_DATATYPE_MPI, - &global_buf[(original_loop_count - cur_loop_count) * total_recvcounts], - recvcounts, displs, C_DATATYPE_MPI, comm)), + for (; loop_count; loop_count--) { + VRFY((MPI_SUCCESS == MPI_Allgatherv(&read_buf[(count[0] - loop_count) * dataset_dims[1]], + recvcounts[mpi_rank], C_DATATYPE_MPI, + &global_buf[(count[0] - loop_count) * total_recvcounts], + recvcounts, displs, C_DATATYPE_MPI, comm)), "MPI_Allgatherv succeeded"); } } @@ -3936,12 +4297,10 @@ test_read_filtered_dataset_point_selection(void) if (correct_buf) HDfree(correct_buf); - HDfree(coords); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; @@ -3949,47 +4308,45 @@ test_read_filtered_dataset_point_selection(void) /* * Tests parallel read of filtered data in the case where - * each process reads an equal amount of data from each - * chunk in the dataset. Each chunk is distributed among the - * processes in round-robin fashion by blocks of size 1 until - * the whole chunk is selected, leading to an interleaved - * read pattern. + * a single process in the read operation has no selection + * in the dataset's dataspace. * * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank will read part - * of each chunk of the dataset and will contribute its - * pieces to a global buffer that is checked for consistency. + * data to the dataset. Then, each rank (except for one) + * reads a part of the dataset and contributes its piece + * to a global buffer that is checked for consistency. * * Programmer: Jordan Henderson * 05/15/2018 */ static void -test_read_filtered_dataset_interleaved_read(void) +test_read_filtered_dataset_single_no_selection(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; - hsize_t chunk_dims[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; - hsize_t sel_dims[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; - hsize_t start[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; - hsize_t stride[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; - hsize_t count[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; - hsize_t block[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; + hsize_t dataset_dims[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t start[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t stride[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t count[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t block[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; hsize_t flat_dims[1]; size_t i, read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + size_t segment_length; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; int * recvcounts = NULL; int * displs = NULL; if (MAINPROCESS) - HDputs("Testing interleaved read from filtered chunks"); - - CHECK_CUR_FILTER_AVAIL(); + HDputs("Testing read from filtered chunks with a single process having no selection"); - dataset_dims[0] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NROWS; - dataset_dims[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS; + dataset_dims[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS; /* Setup the buffer for writing and for comparison */ correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); @@ -3998,18 +4355,13 @@ test_read_filtered_dataset_interleaved_read(void) VRFY((NULL != correct_buf), "HDcalloc succeeded"); for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - /* Add Column Index */ - correct_buf[i] = - (C_DATATYPE)((i % (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS) - - /* Add the Row Index */ - + ((i % (hsize_t)(mpi_size * INTERLEAVED_READ_FILTERED_DATASET_NCOLS)) / - (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS) + correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + + (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); - /* Add the amount that gets added when a rank moves down to its next section - vertically in the dataset */ - + ((hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS * - (i / (hsize_t)(mpi_size * INTERLEAVED_READ_FILTERED_DATASET_NCOLS)))); + /* Compute the correct offset into the buffer for the process having no selection and clear it */ + segment_length = dataset_dims[0] * dataset_dims[1] / (hsize_t)mpi_size; + HDmemset(correct_buf + ((size_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC * segment_length), + 0, segment_length * sizeof(*correct_buf)); if (MAINPROCESS) { plist_id = H5Pcreate(H5P_FILE_ACCESS); @@ -4023,56 +4375,62 @@ test_read_filtered_dataset_interleaved_read(void) VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(INTERLEAVED_READ_FILTERED_DATASET_DIMS, dataset_dims, NULL); + filespace = + H5Screate_simple(READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NROWS; - chunk_dims[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NCOLS; + chunk_dims[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, INTERLEAVED_READ_FILTERED_DATASET_DIMS, chunk_dims) >= 0), + VRFY((H5Pset_chunk(plist_id, READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, INTERLEAVED_READ_FILTERED_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, - H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, + HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); } - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - dset_id = H5Dopen2(file_id, "/" INTERLEAVED_READ_FILTERED_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - sel_dims[0] = (hsize_t)(INTERLEAVED_READ_FILTERED_DATASET_NROWS / mpi_size); - sel_dims[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS; + sel_dims[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + sel_dims[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS; + + if (mpi_rank == READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) + sel_dims[0] = sel_dims[1] = 0; /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ flat_dims[0] = sel_dims[0] * sel_dims[1]; @@ -4088,15 +4446,14 @@ test_read_filtered_dataset_interleaved_read(void) * Each process defines the dataset selection in the file and * reads it to the selection in memory */ - count[0] = - (hsize_t)(INTERLEAVED_READ_FILTERED_DATASET_NROWS / INTERLEAVED_READ_FILTERED_DATASET_CH_NROWS); - count[1] = - (hsize_t)(INTERLEAVED_READ_FILTERED_DATASET_NCOLS / INTERLEAVED_READ_FILTERED_DATASET_CH_NCOLS); - stride[0] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NROWS; - stride[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NCOLS; - block[0] = 1; - block[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NCOLS; - start[0] = (hsize_t)mpi_rank; + count[0] = 1; + count[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS / + (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + stride[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + block[1] = (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; + start[0] = (hsize_t)mpi_rank * (hsize_t)READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS * count[0]; start[1] = 0; if (VERBOSE_MED) { @@ -4107,58 +4464,53 @@ test_read_filtered_dataset_interleaved_read(void) HDfflush(stdout); } - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), - "Hyperslab selection succeeded"); - - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + if (mpi_rank == READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) + VRFY((H5Sselect_none(filespace) >= 0), "Select none succeeded"); + else + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); read_buf_size = flat_dims[0] * sizeof(*read_buf); read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), - "Dataset read succeeded"); + if (mpi_rank == READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) { + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, NULL) >= 0), + "Dataset read succeeded"); + } + else { + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + } global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != global_buf), "HDcalloc succeeded"); - /* - * Since these chunks are shared, run multiple rounds of MPI_Allgatherv - * to collect all of the pieces into their appropriate locations. The - * number of times MPI_Allgatherv is run should be equal to the number - * of chunks in the first dimension of the dataset. - */ - { - size_t loop_count = count[0]; - size_t total_recvcounts = 0; - - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); + /* Collect each piece of data from all ranks into a global buffer on all ranks */ + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + for (i = 0; i < (size_t)mpi_size; i++) + recvcounts[i] = (int)(READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS * + READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS); + recvcounts[READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC] = 0; - for (i = 0; i < (size_t)mpi_size; i++) { - recvcounts[i] = (int)dataset_dims[1]; - total_recvcounts += (size_t)recvcounts[i]; - } + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * dataset_dims[1]); + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * (size_t)(READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS * + READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NCOLS)); - for (; loop_count; loop_count--) { - VRFY((MPI_SUCCESS == MPI_Allgatherv(&read_buf[(count[0] - loop_count) * dataset_dims[1]], - recvcounts[mpi_rank], C_DATATYPE_MPI, - &global_buf[(count[0] - loop_count) * total_recvcounts], - recvcounts, displs, C_DATATYPE_MPI, comm)), - "MPI_Allgatherv succeeded"); - } - } + if (mpi_rank == READ_SINGLE_NO_SELECTION_FILTERED_CHUNKS_NO_SELECT_PROC) + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, 0, C_DATATYPE_MPI, global_buf, recvcounts, displs, + C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); + else + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, + recvcounts, displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -4176,7 +4528,7 @@ test_read_filtered_dataset_interleaved_read(void) VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; @@ -4184,55 +4536,44 @@ test_read_filtered_dataset_interleaved_read(void) /* * Tests parallel read of filtered data in the case where - * the dataset has 3 dimensions and each process reads from - * its own "page" in the 3rd dimension. + * no process in the read operation has a selection in the + * dataset's dataspace. This test is to ensure that there + * are no assertion failures or similar issues due to size + * 0 allocations and the like. * * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads its own "page" - * of the dataset and contributes its piece to a global buffer - * that is checked for consistency. + * data to the dataset. Then, each rank will simply issue + * a no-op read. * * Programmer: Jordan Henderson - * 05/16/2018 + * 05/15/2018 */ static void -test_read_3d_filtered_dataset_no_overlap_separate_pages(void) +test_read_filtered_dataset_all_no_selection(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { - MPI_Datatype vector_type; - MPI_Datatype resized_vector_type; - C_DATATYPE * read_buf = NULL; - C_DATATYPE * correct_buf = NULL; - C_DATATYPE * global_buf = NULL; - hsize_t dataset_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t chunk_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t sel_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t start[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t stride[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t count[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t block[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; - hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + size_t read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing read from unshared filtered chunks on separate pages in 3D dataset"); - - CHECK_CUR_FILTER_AVAIL(); + HDputs("Testing read from filtered chunks with all processes having no selection"); - dataset_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS; - dataset_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS; - dataset_dims[2] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DEPTH; + dataset_dims[0] = (hsize_t)READ_ALL_NO_SELECTION_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)READ_ALL_NO_SELECTION_FILTERED_CHUNKS_NCOLS; /* Setup the buffer for writing and for comparison */ - correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != correct_buf), "HDcalloc succeeded"); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)((i % (hsize_t)mpi_size) + (i / (hsize_t)mpi_size)); - if (MAINPROCESS) { plist_id = H5Pcreate(H5P_FILE_ACCESS); VRFY((plist_id >= 0), "FAPL creation succeeded"); @@ -4245,145 +4586,77 @@ test_read_3d_filtered_dataset_no_overlap_separate_pages(void) VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + /* Create the dataspace for the dataset */ - filespace = - H5Screate_simple(READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; - chunk_dims[2] = 1; + chunk_dims[0] = (hsize_t)READ_ALL_NO_SELECTION_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_ALL_NO_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY( - (H5Pset_chunk(plist_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, chunk_dims) >= 0), - "Chunk size set"); + VRFY((H5Pset_chunk(plist_id, READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_NAME, - HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); } - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - sel_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS; - sel_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS; - sel_dims[2] = 1; - - /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ - flat_dims[0] = sel_dims[0] * sel_dims[1] * sel_dims[2]; + sel_dims[0] = sel_dims[1] = 0; - memspace = H5Screate_simple(1, flat_dims, NULL); + memspace = H5Screate_simple(READ_ALL_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); /* Select hyperslab in the file */ filespace = H5Dget_space(dset_id); VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - /* - * Each process defines the dataset selection in the file and - * reads it to the selection in memory - */ - count[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS / - (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; - count[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS / - (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; - count[2] = 1; - stride[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; - stride[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; - stride[2] = 1; - block[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; - block[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; - block[2] = 1; - start[0] = 0; - start[1] = 0; - start[2] = (hsize_t)mpi_rank; - - if (VERBOSE_MED) { - HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE - ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE - ", %" PRIuHSIZE " ]\n", - mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); - HDfflush(stdout); - } - - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), - "Hyperslab selection succeeded"); - - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + VRFY((H5Sselect_none(filespace) >= 0), "Select none succeeded"); - read_buf_size = flat_dims[0] * sizeof(*read_buf); + read_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*read_buf); read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); - global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != global_buf), "HDcalloc succeeded"); - - /* - * Due to the nature of 3-dimensional reading, create an MPI vector type that allows each - * rank to write to the nth position of the global data buffer, where n is the rank number. - */ - VRFY((MPI_SUCCESS == MPI_Type_vector((int)flat_dims[0], 1, mpi_size, C_DATATYPE_MPI, &vector_type)), - "MPI_Type_vector succeeded"); - VRFY((MPI_SUCCESS == MPI_Type_commit(&vector_type)), "MPI_Type_commit succeeded"); - - /* - * Resize the type to allow interleaving, - * so make it only one MPI_LONG wide - */ - VRFY((MPI_SUCCESS == MPI_Type_create_resized(vector_type, 0, sizeof(long), &resized_vector_type)), - "MPI_Type_create_resized"); - VRFY((MPI_SUCCESS == MPI_Type_commit(&resized_vector_type)), "MPI_Type_commit succeeded"); - - VRFY((MPI_SUCCESS == MPI_Allgather(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, 1, - resized_vector_type, comm)), - "MPI_Allgather succeeded"); - - VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); - - VRFY((MPI_SUCCESS == MPI_Type_free(&vector_type)), "MPI_Type_free succeeded"); - VRFY((MPI_SUCCESS == MPI_Type_free(&resized_vector_type)), "MPI_Type_free succeeded"); + VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); - if (global_buf) - HDfree(global_buf); if (read_buf) HDfree(read_buf); if (correct_buf) @@ -4392,70 +4665,61 @@ test_read_3d_filtered_dataset_no_overlap_separate_pages(void) VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel read of transformed and filtered data in the - * case where only one process is reading from a particular - * chunk in the operation. Normally, a data transform function - * will cause the parallel library to break to independent I/O - * and this isn't allowed when there are filters in the pipeline. - * However, in this case the parallel library recognizes that - * the used data transform function "x" is the same as not - * applying the transform function. Therefore it does not apply - * the transform function resulting in not breaking to - * independent I/O. + * Tests parallel read of filtered data by using point + * selections instead of hyperslab selections. * * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads a part of - * the dataset and contributes its piece to a global buffer - * that is checked for consistency. + * data to the dataset. Then, each rank will read part + * of the dataset using a point selection and will + * contribute its piece to a global buffer that is + * checked for consistency. * - * Programmer: Jan-Willem Blokland - * 08/20/2021 + * Programmer: Jordan Henderson + * 05/15/2018 */ static void -test_read_transformed_filtered_dataset_no_overlap(void) +test_read_filtered_dataset_point_selection(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { - C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; + C_DATATYPE *read_buf = NULL; C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t start[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t stride[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t count[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; - hsize_t block[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t * coords = NULL; + hsize_t dataset_dims[READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS]; hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + size_t i, j, read_buf_size, correct_buf_size; + size_t num_points; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; int * recvcounts = NULL; int * displs = NULL; if (MAINPROCESS) - HDputs("Testing read from unshared transformed and filtered chunks"); - - CHECK_CUR_FILTER_AVAIL(); + HDputs("Testing read from filtered chunks with point selection"); - dataset_dims[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS; + dataset_dims[0] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS; /* Setup the buffer for writing and for comparison */ - correct_buf_size = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NROWS * - (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS * sizeof(*correct_buf); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != correct_buf), "HDcalloc succeeded"); for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + - (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); + correct_buf[i] = (C_DATATYPE)( + (dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); if (MAINPROCESS) { plist_id = H5Pcreate(H5P_FILE_ACCESS); @@ -4469,66 +4733,58 @@ test_read_transformed_filtered_dataset_no_overlap(void) VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + /* Create the dataspace for the dataset */ - filespace = - H5Screate_simple(READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; + chunk_dims[0] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_CH_NCOLS; - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY( - (H5Pset_chunk(plist_id, READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), - "Chunk size set"); + VRFY((H5Pset_chunk(plist_id, READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_NAME, - HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - /* Set data transform expression */ - VRFY((H5Pset_data_transform(plist_id, "x") >= 0), "Set data transform expression succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, correct_buf) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), "Dataset write succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); } - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - sel_dims[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; - sel_dims[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS; + sel_dims[0] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NROWS / (hsize_t)mpi_size; + sel_dims[1] = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS; /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ flat_dims[0] = sel_dims[0] * sel_dims[1]; @@ -4536,71 +4792,70 @@ test_read_transformed_filtered_dataset_no_overlap(void) memspace = H5Screate_simple(1, flat_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - /* Select hyperslab in the file */ + /* Set up point selection */ filespace = H5Dget_space(dset_id); VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - /* - * Each process defines the dataset selection in the file and reads - * it to the selection in memory - */ - count[0] = 1; - count[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS / - (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; - stride[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; - stride[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; - block[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; - block[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; - start[0] = ((hsize_t)mpi_rank * (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS * count[0]); - start[1] = 0; - - if (VERBOSE_MED) { - HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE - ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE - ", %" PRIuHSIZE " ]\n", - mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); - HDfflush(stdout); - } - - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), - "Hyperslab selection succeeded"); - - /* Create property list for collective dataset read and data transform */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + num_points = (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NROWS * + (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS / (hsize_t)mpi_size; + coords = (hsize_t *)HDcalloc(1, 2 * num_points * sizeof(*coords)); + VRFY((NULL != coords), "Coords HDcalloc succeeded"); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + for (i = 0; i < num_points; i++) + for (j = 0; j < READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS; j++) + coords[(i * READ_POINT_SELECTION_FILTERED_CHUNKS_DATASET_DIMS) + j] = + (j > 0) ? (i % (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS) + : ((hsize_t)mpi_rank + + ((hsize_t)mpi_size * (i / (hsize_t)READ_POINT_SELECTION_FILTERED_CHUNKS_NCOLS))); - /* Set data transform expression */ - VRFY((H5Pset_data_transform(plist_id, "x") >= 0), "Set data transform expression succeeded"); + VRFY((H5Sselect_elements(filespace, H5S_SELECT_SET, (hsize_t)num_points, (const hsize_t *)coords) >= 0), + "Point selection succeeded"); read_buf_size = flat_dims[0] * sizeof(*read_buf); read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != global_buf), "HDcalloc succeeded"); - /* Collect each piece of data from all ranks into a global buffer on all ranks */ - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); + /* + * Since these chunks are shared, run multiple rounds of MPI_Allgatherv + * to collect all of the pieces into their appropriate locations. The + * number of times MPI_Allgatherv is run should be equal to the number + * of chunks in the first dimension of the dataset. + */ + { + size_t original_loop_count = dataset_dims[0] / (hsize_t)mpi_size; + size_t cur_loop_count = original_loop_count; + size_t total_recvcounts = 0; - for (i = 0; i < (size_t)mpi_size; i++) - recvcounts[i] = (int)flat_dims[0]; + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * flat_dims[0]); + for (i = 0; i < (size_t)mpi_size; i++) { + recvcounts[i] = (int)dataset_dims[1]; + total_recvcounts += (size_t)recvcounts[i]; + } - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, recvcounts, - displs, C_DATATYPE_MPI, comm)), - "MPI_Allgatherv succeeded"); + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * dataset_dims[1]); + + for (; cur_loop_count; cur_loop_count--) { + VRFY((MPI_SUCCESS == + MPI_Allgatherv(&read_buf[(original_loop_count - cur_loop_count) * dataset_dims[1]], + recvcounts[mpi_rank], C_DATATYPE_MPI, + &global_buf[(original_loop_count - cur_loop_count) * total_recvcounts], + recvcounts, displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); + } + } VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -4615,10 +4870,12 @@ test_read_transformed_filtered_dataset_no_overlap(void) if (correct_buf) HDfree(correct_buf); + HDfree(coords); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; @@ -4626,56 +4883,67 @@ test_read_transformed_filtered_dataset_no_overlap(void) /* * Tests parallel read of filtered data in the case where - * the dataset has 3 dimensions and each process reads from - * each "page" in the 3rd dimension. However, no chunk on a - * given "page" is read from by more than one process. + * each process reads an equal amount of data from each + * chunk in the dataset. Each chunk is distributed among the + * processes in round-robin fashion by blocks of size 1 until + * the whole chunk is selected, leading to an interleaved + * read pattern. * * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads a part of - * each "page" of the dataset and contributes its piece to a - * global buffer that is checked for consistency. + * data to the dataset. Then, each rank will read part + * of each chunk of the dataset and will contribute its + * pieces to a global buffer that is checked for consistency. * * Programmer: Jordan Henderson - * 05/16/2018 + * 05/15/2018 */ static void -test_read_3d_filtered_dataset_no_overlap_same_pages(void) +test_read_filtered_dataset_interleaved_read(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) { C_DATATYPE *read_buf = NULL; C_DATATYPE *correct_buf = NULL; C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t chunk_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t sel_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t start[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t stride[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t count[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t block[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; - hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id, dset_id, plist_id; - hid_t filespace, memspace; - int * recvcounts = NULL; - int * displs = NULL; - - if (MAINPROCESS) - HDputs("Testing read from unshared filtered chunks on the same pages in 3D dataset"); + hsize_t dataset_dims[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; + hsize_t chunk_dims[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; + hsize_t sel_dims[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; + hsize_t start[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; + hsize_t stride[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; + hsize_t count[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; + hsize_t block[INTERLEAVED_READ_FILTERED_DATASET_DIMS]; + hsize_t flat_dims[1]; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; - CHECK_CUR_FILTER_AVAIL(); + if (MAINPROCESS) + HDputs("Testing interleaved read from filtered chunks"); - dataset_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NROWS; - dataset_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS; - dataset_dims[2] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DEPTH; + dataset_dims[0] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NROWS; + dataset_dims[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS; /* Setup the buffer for writing and for comparison */ - correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != correct_buf), "HDcalloc succeeded"); for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] * dataset_dims[1])) + - (i / (dataset_dims[0] * dataset_dims[1]))); + /* Add Column Index */ + correct_buf[i] = + (C_DATATYPE)((i % (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS) + + /* Add the Row Index */ + + ((i % (hsize_t)(mpi_size * INTERLEAVED_READ_FILTERED_DATASET_NCOLS)) / + (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS) + + /* Add the amount that gets added when a rank moves down to its next section + vertically in the dataset */ + + ((hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS * + (i / (hsize_t)(mpi_size * INTERLEAVED_READ_FILTERED_DATASET_NCOLS)))); if (MAINPROCESS) { plist_id = H5Pcreate(H5P_FILE_ACCESS); @@ -4689,63 +4957,61 @@ test_read_3d_filtered_dataset_no_overlap_same_pages(void) VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + /* Create the dataspace for the dataset */ - filespace = - H5Screate_simple(READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(INTERLEAVED_READ_FILTERED_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; - chunk_dims[2] = 1; + chunk_dims[0] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NROWS; + chunk_dims[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NCOLS; - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, chunk_dims) >= - 0), + VRFY((H5Pset_chunk(plist_id, INTERLEAVED_READ_FILTERED_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_NAME, - HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, INTERLEAVED_READ_FILTERED_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); } - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, INTERLEAVED_READ_FILTERED_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - sel_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; - sel_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS; - sel_dims[2] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DEPTH; + sel_dims[0] = (hsize_t)(INTERLEAVED_READ_FILTERED_DATASET_NROWS / mpi_size); + sel_dims[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_NCOLS; /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ - flat_dims[0] = sel_dims[0] * sel_dims[1] * sel_dims[2]; + flat_dims[0] = sel_dims[0] * sel_dims[1]; memspace = H5Screate_simple(1, flat_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); @@ -4758,19 +5024,16 @@ test_read_3d_filtered_dataset_no_overlap_same_pages(void) * Each process defines the dataset selection in the file and * reads it to the selection in memory */ - count[0] = 1; - count[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS / - (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; - count[2] = (hsize_t)mpi_size; - stride[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; - stride[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; - stride[2] = 1; - block[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; - block[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; - block[2] = 1; - start[0] = ((hsize_t)mpi_rank * (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS * count[0]); + count[0] = + (hsize_t)(INTERLEAVED_READ_FILTERED_DATASET_NROWS / INTERLEAVED_READ_FILTERED_DATASET_CH_NROWS); + count[1] = + (hsize_t)(INTERLEAVED_READ_FILTERED_DATASET_NCOLS / INTERLEAVED_READ_FILTERED_DATASET_CH_NCOLS); + stride[0] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NROWS; + stride[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NCOLS; + block[0] = 1; + block[1] = (hsize_t)INTERLEAVED_READ_FILTERED_DATASET_CH_NCOLS; + start[0] = (hsize_t)mpi_rank; start[1] = 0; - start[2] = 0; if (VERBOSE_MED) { HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE @@ -4783,39 +5046,49 @@ test_read_3d_filtered_dataset_no_overlap_same_pages(void) VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - read_buf_size = flat_dims[0] * sizeof(*read_buf); read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != global_buf), "HDcalloc succeeded"); - /* Collect each piece of data from all ranks into a global buffer on all ranks */ - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); + /* + * Since these chunks are shared, run multiple rounds of MPI_Allgatherv + * to collect all of the pieces into their appropriate locations. The + * number of times MPI_Allgatherv is run should be equal to the number + * of chunks in the first dimension of the dataset. + */ + { + size_t loop_count = count[0]; + size_t total_recvcounts = 0; - for (i = 0; i < (size_t)mpi_size; i++) - recvcounts[i] = (int)flat_dims[0]; + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * flat_dims[0]); + for (i = 0; i < (size_t)mpi_size; i++) { + recvcounts[i] = (int)dataset_dims[1]; + total_recvcounts += (size_t)recvcounts[i]; + } - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, recvcounts, - displs, C_DATATYPE_MPI, comm)), - "MPI_Allgatherv succeeded"); + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * dataset_dims[1]); + + for (; loop_count; loop_count--) { + VRFY((MPI_SUCCESS == MPI_Allgatherv(&read_buf[(count[0] - loop_count) * dataset_dims[1]], + recvcounts[mpi_rank], C_DATATYPE_MPI, + &global_buf[(count[0] - loop_count) * total_recvcounts], + recvcounts, displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); + } + } VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -4833,7 +5106,7 @@ test_read_3d_filtered_dataset_no_overlap_same_pages(void) VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; @@ -4842,45 +5115,44 @@ test_read_3d_filtered_dataset_no_overlap_same_pages(void) /* * Tests parallel read of filtered data in the case where * the dataset has 3 dimensions and each process reads from - * each "page" in the 3rd dimension. Further, each chunk in - * each "page" is read from equally by all processes. + * its own "page" in the 3rd dimension. * * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads part of each - * chunk of each "page" and contributes its pieces to a - * global buffer that is checked for consistency. + * data to the dataset. Then, each rank reads its own "page" + * of the dataset and contributes its piece to a global buffer + * that is checked for consistency. * * Programmer: Jordan Henderson * 05/16/2018 */ static void -test_read_3d_filtered_dataset_overlap(void) +test_read_3d_filtered_dataset_no_overlap_separate_pages(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { MPI_Datatype vector_type; MPI_Datatype resized_vector_type; C_DATATYPE * read_buf = NULL; C_DATATYPE * correct_buf = NULL; C_DATATYPE * global_buf = NULL; - hsize_t dataset_dims[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t chunk_dims[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t sel_dims[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t start[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t stride[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t count[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; - hsize_t block[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t dataset_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t chunk_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t sel_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t start[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t stride[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t count[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; + hsize_t block[READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS]; hsize_t flat_dims[1]; size_t i, read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing read from shared filtered chunks in 3D dataset"); - - CHECK_CUR_FILTER_AVAIL(); + HDputs("Testing read from unshared filtered chunks on separate pages in 3D dataset"); - dataset_dims[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_NROWS; - dataset_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_NCOLS; - dataset_dims[2] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_DEPTH; + dataset_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS; + dataset_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS; + dataset_dims[2] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DEPTH; /* Setup the buffer for writing and for comparison */ correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); @@ -4889,20 +5161,7 @@ test_read_3d_filtered_dataset_overlap(void) VRFY((NULL != correct_buf), "HDcalloc succeeded"); for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - /* Add the Column Index */ - correct_buf[i] = (C_DATATYPE)( - (i % (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * READ_SHARED_FILTERED_CHUNKS_3D_NCOLS)) - - /* Add the Row Index */ - + ((i % (hsize_t)(mpi_size * READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * - READ_SHARED_FILTERED_CHUNKS_3D_NCOLS)) / - (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * READ_SHARED_FILTERED_CHUNKS_3D_NCOLS)) - - /* Add the amount that gets added when a rank moves down to its next - section vertically in the dataset */ - + ((hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * READ_SHARED_FILTERED_CHUNKS_3D_NCOLS) * - (i / (hsize_t)(mpi_size * READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * - READ_SHARED_FILTERED_CHUNKS_3D_NCOLS)))); + correct_buf[i] = (C_DATATYPE)((i % (hsize_t)mpi_size) + (i / (hsize_t)mpi_size)); if (MAINPROCESS) { plist_id = H5Pcreate(H5P_FILE_ACCESS); @@ -4916,58 +5175,62 @@ test_read_3d_filtered_dataset_overlap(void) VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, dataset_dims, NULL); + filespace = + H5Screate_simple(READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; + chunk_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; chunk_dims[2] = 1; - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, chunk_dims) >= 0), - "Chunk size set"); + VRFY( + (H5Pset_chunk(plist_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, READ_SHARED_FILTERED_CHUNKS_3D_DATASET_NAME, HDF5_DATATYPE_NAME, - filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + dset_id = H5Dcreate2(group_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_NAME, + HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), "Dataset write succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); } - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_SHARED_FILTERED_CHUNKS_3D_DATASET_NAME, H5P_DEFAULT); + dset_id = H5Dopen2(group_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_DATASET_NAME, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - sel_dims[0] = (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_NROWS / mpi_size); - sel_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_NCOLS; - sel_dims[2] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_DEPTH; + sel_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS; + sel_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS; + sel_dims[2] = 1; /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ flat_dims[0] = sel_dims[0] * sel_dims[1] * sel_dims[2]; @@ -4983,18 +5246,20 @@ test_read_3d_filtered_dataset_overlap(void) * Each process defines the dataset selection in the file and * reads it to the selection in memory */ - count[0] = (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_NROWS / READ_SHARED_FILTERED_CHUNKS_3D_CH_NROWS); - count[1] = (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_NCOLS / READ_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS); - count[2] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_DEPTH; - stride[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NROWS; - stride[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; + count[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NROWS / + (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; + count[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_NCOLS / + (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; + count[2] = 1; + stride[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; + stride[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; stride[2] = 1; - block[0] = 1; - block[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; + block[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NROWS; + block[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SEP_PAGE_CH_NCOLS; block[2] = 1; - start[0] = (hsize_t)mpi_rank; + start[0] = 0; start[1] = 0; - start[2] = 0; + start[2] = (hsize_t)mpi_rank; if (VERBOSE_MED) { HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE @@ -5007,51 +5272,36 @@ test_read_3d_filtered_dataset_overlap(void) VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - read_buf_size = flat_dims[0] * sizeof(*read_buf); read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != global_buf), "HDcalloc succeeded"); - { - size_t run_length = - (size_t)(READ_SHARED_FILTERED_CHUNKS_3D_NCOLS * READ_SHARED_FILTERED_CHUNKS_3D_DEPTH); - size_t num_blocks = (size_t)(READ_SHARED_FILTERED_CHUNKS_3D_NROWS / mpi_size); + /* + * Due to the nature of 3-dimensional reading, create an MPI vector type that allows each + * rank to write to the nth position of the global data buffer, where n is the rank number. + */ + VRFY((MPI_SUCCESS == MPI_Type_vector((int)flat_dims[0], 1, mpi_size, C_DATATYPE_MPI, &vector_type)), + "MPI_Type_vector succeeded"); + VRFY((MPI_SUCCESS == MPI_Type_commit(&vector_type)), "MPI_Type_commit succeeded"); - /* - * Due to the nature of 3-dimensional reading, create an MPI vector type that allows each - * rank to write to the nth position of the global data buffer, where n is the rank number. - */ - VRFY( - (MPI_SUCCESS == MPI_Type_vector((int)num_blocks, (int)run_length, - (int)(mpi_size * (int)run_length), C_DATATYPE_MPI, &vector_type)), - "MPI_Type_vector succeeded"); - VRFY((MPI_SUCCESS == MPI_Type_commit(&vector_type)), "MPI_Type_commit succeeded"); - - /* - * Resize the type to allow interleaving, - * so make it "run_length" MPI_LONGs wide - */ - VRFY((MPI_SUCCESS == MPI_Type_create_resized(vector_type, 0, (MPI_Aint)(run_length * sizeof(long)), - &resized_vector_type)), - "MPI_Type_create_resized"); - VRFY((MPI_SUCCESS == MPI_Type_commit(&resized_vector_type)), "MPI_Type_commit succeeded"); - } + /* + * Resize the type to allow interleaving, + * so make it only one MPI_LONG wide + */ + VRFY((MPI_SUCCESS == MPI_Type_create_resized(vector_type, 0, sizeof(long), &resized_vector_type)), + "MPI_Type_create_resized"); + VRFY((MPI_SUCCESS == MPI_Type_commit(&resized_vector_type)), "MPI_Type_commit succeeded"); VRFY((MPI_SUCCESS == MPI_Allgather(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, 1, resized_vector_type, comm)), - "MPI_Allgatherv succeeded"); + "MPI_Allgather succeeded"); VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); @@ -5068,78 +5318,1890 @@ test_read_3d_filtered_dataset_overlap(void) VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel read of filtered data to unshared - * chunks using a compound datatype which doesn't - * require a datatype conversion. + * Tests parallel read of transformed and filtered data in the + * case where only one process is reading from a particular + * chunk in the operation. Normally, a data transform function + * will cause the parallel library to break to independent I/O + * and this isn't allowed when there are filters in the pipeline. + * However, in this case the parallel library recognizes that + * the used data transform function "x" is the same as not + * applying the transform function. Therefore it does not apply + * the transform function resulting in not breaking to + * independent I/O. * * The MAINPROCESS rank will first write out all of the * data to the dataset. Then, each rank reads a part of - * the dataset and contributes its piece to a global - * buffer that is checked for consistency. + * the dataset and contributes its piece to a global buffer + * that is checked for consistency. * - * Programmer: Jordan Henderson - * 05/17/2018 + * Programmer: Jan-Willem Blokland + * 08/20/2021 */ static void -test_read_cmpd_filtered_dataset_no_conversion_unshared(void) +test_read_transformed_filtered_dataset_no_overlap(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) { - COMPOUND_C_DATATYPE *read_buf = NULL; - COMPOUND_C_DATATYPE *correct_buf = NULL; - COMPOUND_C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t chunk_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t sel_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t start[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t stride[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t count[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t block[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1, memtype = -1; - hid_t filespace = -1, memspace = -1; - int * recvcounts = NULL; - int * displs = NULL; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + C_DATATYPE *global_buf = NULL; + hsize_t dataset_dims[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t start[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t stride[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t count[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t block[READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS]; + hsize_t flat_dims[1]; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; if (MAINPROCESS) - HDputs("Testing read from unshared filtered chunks in Compound Datatype dataset without Datatype " - "conversion"); - - CHECK_CUR_FILTER_AVAIL(); + HDputs("Testing read from unshared transformed and filtered chunks"); - dataset_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_NROWS; - dataset_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_NCOLS; + dataset_dims[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS; /* Setup the buffer for writing and for comparison */ - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + correct_buf_size = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NROWS * + (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS * sizeof(*correct_buf); - correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); VRFY((NULL != correct_buf), "HDcalloc succeeded"); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { - correct_buf[i].field1 = (short)((i % dataset_dims[1]) + (i / dataset_dims[1])); + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1])) + + (i / (dataset_dims[0] / (hsize_t)mpi_size * dataset_dims[1]))); + + if (MAINPROCESS) { + plist_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((plist_id >= 0), "FAPL creation succeeded"); + + VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), + "Set libver bounds succeeded"); + + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + filespace = + H5Screate_simple(READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + /* Create chunked dataset */ + chunk_dims[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; + + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY( + (H5Pset_chunk(plist_id, READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_NAME, + HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + /* Create property list for collective dataset read */ + plist_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((plist_id >= 0), "DXPL creation succeeded"); + + /* Set data transform expression */ + VRFY((H5Pset_data_transform(plist_id, "x") >= 0), "Set data transform expression succeeded"); + + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, correct_buf) >= 0), + "Dataset write succeeded"); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + + /* Verify space allocation status */ + plist_id = H5Dget_create_plist(dset_id); + VRFY((plist_id >= 0), "H5Dget_create_plist succeeded"); + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + } + + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + dset_id = H5Dopen2(group_id, READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + sel_dims[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; + sel_dims[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS; + + /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ + flat_dims[0] = sel_dims[0] * sel_dims[1]; + + memspace = H5Screate_simple(1, flat_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + /* + * Each process defines the dataset selection in the file and reads + * it to the selection in memory + */ + count[0] = 1; + count[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_NCOLS / + (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; + stride[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS; + block[1] = (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NCOLS; + start[0] = ((hsize_t)mpi_rank * (hsize_t)READ_UNSHARED_TRANSFORMED_FILTERED_CHUNKS_CH_NROWS * count[0]); + start[1] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + /* Create property list for data transform */ + plist_id = H5Pcopy(dxpl_id); + VRFY((plist_id >= 0), "DXPL copy succeeded"); + + /* Set data transform expression */ + VRFY((H5Pset_data_transform(plist_id, "x") >= 0), "Set data transform expression succeeded"); + + read_buf_size = flat_dims[0] * sizeof(*read_buf); + + read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, read_buf) >= 0), + "Dataset read succeeded"); + + global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != global_buf), "HDcalloc succeeded"); + + /* Collect each piece of data from all ranks into a global buffer on all ranks */ + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + recvcounts[i] = (int)flat_dims[0]; + + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * flat_dims[0]); + + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, recvcounts, + displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); + + VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + if (displs) + HDfree(displs); + if (recvcounts) + HDfree(recvcounts); + if (global_buf) + HDfree(global_buf); + if (read_buf) + HDfree(read_buf); + if (correct_buf) + HDfree(correct_buf); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); + VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + return; +} + +/* + * Tests parallel read of filtered data in the case where + * the dataset has 3 dimensions and each process reads from + * each "page" in the 3rd dimension. However, no chunk on a + * given "page" is read from by more than one process. + * + * The MAINPROCESS rank will first write out all of the + * data to the dataset. Then, each rank reads a part of + * each "page" of the dataset and contributes its piece to a + * global buffer that is checked for consistency. + * + * Programmer: Jordan Henderson + * 05/16/2018 + */ +static void +test_read_3d_filtered_dataset_no_overlap_same_pages(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) +{ + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + C_DATATYPE *global_buf = NULL; + hsize_t dataset_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t chunk_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t sel_dims[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t start[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t stride[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t count[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t block[READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS]; + hsize_t flat_dims[1]; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; + + if (MAINPROCESS) + HDputs("Testing read from unshared filtered chunks on the same pages in 3D dataset"); + + dataset_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NROWS; + dataset_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS; + dataset_dims[2] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DEPTH; + + /* Setup the buffer for writing and for comparison */ + correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + + correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); + + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] * dataset_dims[1])) + + (i / (dataset_dims[0] * dataset_dims[1]))); + + if (MAINPROCESS) { + plist_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((plist_id >= 0), "FAPL creation succeeded"); + + VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), + "Set libver bounds succeeded"); + + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + filespace = + H5Screate_simple(READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + /* Create chunked dataset */ + chunk_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; + chunk_dims[2] = 1; + + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_DIMS, chunk_dims) >= + 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_NAME, + HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + } + + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + dset_id = H5Dopen2(group_id, READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + sel_dims[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; + sel_dims[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS; + sel_dims[2] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_DEPTH; + + /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ + flat_dims[0] = sel_dims[0] * sel_dims[1] * sel_dims[2]; + + memspace = H5Screate_simple(1, flat_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + /* + * Each process defines the dataset selection in the file and + * reads it to the selection in memory + */ + count[0] = 1; + count[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_NCOLS / + (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; + count[2] = (hsize_t)mpi_size; + stride[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; + stride[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; + stride[2] = 1; + block[0] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS; + block[1] = (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NCOLS; + block[2] = 1; + start[0] = ((hsize_t)mpi_rank * (hsize_t)READ_UNSHARED_FILTERED_CHUNKS_3D_SAME_PAGE_CH_NROWS * count[0]); + start[1] = 0; + start[2] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + read_buf_size = flat_dims[0] * sizeof(*read_buf); + + read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + + global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != global_buf), "HDcalloc succeeded"); + + /* Collect each piece of data from all ranks into a global buffer on all ranks */ + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + recvcounts[i] = (int)flat_dims[0]; + + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * flat_dims[0]); + + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, recvcounts, + displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); + + VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + if (displs) + HDfree(displs); + if (recvcounts) + HDfree(recvcounts); + if (global_buf) + HDfree(global_buf); + if (read_buf) + HDfree(read_buf); + if (correct_buf) + HDfree(correct_buf); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + return; +} + +/* + * Tests parallel read of filtered data in the case where + * the dataset has 3 dimensions and each process reads from + * each "page" in the 3rd dimension. Further, each chunk in + * each "page" is read from equally by all processes. + * + * The MAINPROCESS rank will first write out all of the + * data to the dataset. Then, each rank reads part of each + * chunk of each "page" and contributes its pieces to a + * global buffer that is checked for consistency. + * + * Programmer: Jordan Henderson + * 05/16/2018 + */ +static void +test_read_3d_filtered_dataset_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) +{ + MPI_Datatype vector_type; + MPI_Datatype resized_vector_type; + C_DATATYPE * read_buf = NULL; + C_DATATYPE * correct_buf = NULL; + C_DATATYPE * global_buf = NULL; + hsize_t dataset_dims[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t chunk_dims[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t sel_dims[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t start[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t stride[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t count[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t block[READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS]; + hsize_t flat_dims[1]; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + + if (MAINPROCESS) + HDputs("Testing read from shared filtered chunks in 3D dataset"); + + dataset_dims[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_NROWS; + dataset_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_NCOLS; + dataset_dims[2] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_DEPTH; + + /* Setup the buffer for writing and for comparison */ + correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + + correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); + + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + /* Add the Column Index */ + correct_buf[i] = (C_DATATYPE)( + (i % (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * READ_SHARED_FILTERED_CHUNKS_3D_NCOLS)) + + /* Add the Row Index */ + + ((i % (hsize_t)(mpi_size * READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * + READ_SHARED_FILTERED_CHUNKS_3D_NCOLS)) / + (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * READ_SHARED_FILTERED_CHUNKS_3D_NCOLS)) + + /* Add the amount that gets added when a rank moves down to its next + section vertically in the dataset */ + + ((hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * READ_SHARED_FILTERED_CHUNKS_3D_NCOLS) * + (i / (hsize_t)(mpi_size * READ_SHARED_FILTERED_CHUNKS_3D_DEPTH * + READ_SHARED_FILTERED_CHUNKS_3D_NCOLS)))); + + if (MAINPROCESS) { + plist_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((plist_id >= 0), "FAPL creation succeeded"); + + VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), + "Set libver bounds succeeded"); + + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + filespace = H5Screate_simple(READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + /* Create chunked dataset */ + chunk_dims[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; + chunk_dims[2] = 1; + + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, READ_SHARED_FILTERED_CHUNKS_3D_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, READ_SHARED_FILTERED_CHUNKS_3D_DATASET_NAME, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + } + + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + dset_id = H5Dopen2(group_id, READ_SHARED_FILTERED_CHUNKS_3D_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + sel_dims[0] = (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_NROWS / mpi_size); + sel_dims[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_NCOLS; + sel_dims[2] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_DEPTH; + + /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ + flat_dims[0] = sel_dims[0] * sel_dims[1] * sel_dims[2]; + + memspace = H5Screate_simple(1, flat_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + /* + * Each process defines the dataset selection in the file and + * reads it to the selection in memory + */ + count[0] = (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_NROWS / READ_SHARED_FILTERED_CHUNKS_3D_CH_NROWS); + count[1] = (hsize_t)(READ_SHARED_FILTERED_CHUNKS_3D_NCOLS / READ_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS); + count[2] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_DEPTH; + stride[0] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NROWS; + stride[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; + stride[2] = 1; + block[0] = 1; + block[1] = (hsize_t)READ_SHARED_FILTERED_CHUNKS_3D_CH_NCOLS; + block[2] = 1; + start[0] = (hsize_t)mpi_rank; + start[1] = 0; + start[2] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + read_buf_size = flat_dims[0] * sizeof(*read_buf); + + read_buf = (C_DATATYPE *)HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + + global_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != global_buf), "HDcalloc succeeded"); + + { + size_t run_length = + (size_t)(READ_SHARED_FILTERED_CHUNKS_3D_NCOLS * READ_SHARED_FILTERED_CHUNKS_3D_DEPTH); + size_t num_blocks = (size_t)(READ_SHARED_FILTERED_CHUNKS_3D_NROWS / mpi_size); + + /* + * Due to the nature of 3-dimensional reading, create an MPI vector type that allows each + * rank to write to the nth position of the global data buffer, where n is the rank number. + */ + VRFY( + (MPI_SUCCESS == MPI_Type_vector((int)num_blocks, (int)run_length, + (int)(mpi_size * (int)run_length), C_DATATYPE_MPI, &vector_type)), + "MPI_Type_vector succeeded"); + VRFY((MPI_SUCCESS == MPI_Type_commit(&vector_type)), "MPI_Type_commit succeeded"); + + /* + * Resize the type to allow interleaving, + * so make it "run_length" MPI_LONGs wide + */ + VRFY((MPI_SUCCESS == MPI_Type_create_resized(vector_type, 0, (MPI_Aint)(run_length * sizeof(long)), + &resized_vector_type)), + "MPI_Type_create_resized"); + VRFY((MPI_SUCCESS == MPI_Type_commit(&resized_vector_type)), "MPI_Type_commit succeeded"); + } + + VRFY((MPI_SUCCESS == MPI_Allgather(read_buf, (int)flat_dims[0], C_DATATYPE_MPI, global_buf, 1, + resized_vector_type, comm)), + "MPI_Allgatherv succeeded"); + + VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + VRFY((MPI_SUCCESS == MPI_Type_free(&vector_type)), "MPI_Type_free succeeded"); + VRFY((MPI_SUCCESS == MPI_Type_free(&resized_vector_type)), "MPI_Type_free succeeded"); + + if (global_buf) + HDfree(global_buf); + if (read_buf) + HDfree(read_buf); + if (correct_buf) + HDfree(correct_buf); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + return; +} + +/* + * Tests parallel read of filtered data to unshared + * chunks using a compound datatype which doesn't + * require a datatype conversion. + * + * The MAINPROCESS rank will first write out all of the + * data to the dataset. Then, each rank reads a part of + * the dataset and contributes its piece to a global + * buffer that is checked for consistency. + * + * Programmer: Jordan Henderson + * 05/17/2018 + */ +static void +test_read_cmpd_filtered_dataset_no_conversion_unshared(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) +{ + COMPOUND_C_DATATYPE *read_buf = NULL; + COMPOUND_C_DATATYPE *correct_buf = NULL; + COMPOUND_C_DATATYPE *global_buf = NULL; + hsize_t dataset_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t chunk_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t sel_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t start[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t stride[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t count[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t block[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t flat_dims[1]; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID, + memtype = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; + + if (MAINPROCESS) + HDputs("Testing read from unshared filtered chunks in Compound Datatype dataset without Datatype " + "conversion"); + + /* SZIP and ScaleOffset filters don't support compound types */ + if (filter_id == H5Z_FILTER_SZIP || filter_id == H5Z_FILTER_SCALEOFFSET) { + if (MAINPROCESS) + SKIPPED(); + return; + } + + dataset_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_NROWS; + dataset_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_NCOLS; + + /* Setup the buffer for writing and for comparison */ + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + + correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); + + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { + correct_buf[i].field1 = (short)((i % dataset_dims[1]) + (i / dataset_dims[1])); + + correct_buf[i].field2 = (int)((i % dataset_dims[1]) + (i / dataset_dims[1])); + + correct_buf[i].field3 = (long)((i % dataset_dims[1]) + (i / dataset_dims[1])); + } + + /* Create the compound type for memory. */ + memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); + VRFY((memtype >= 0), "Datatype creation succeeded"); + + VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), + "Datatype insertion succeeded"); + + if (MAINPROCESS) { + plist_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((plist_id >= 0), "FAPL creation succeeded"); + + VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), + "Set libver bounds succeeded"); + + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + filespace = H5Screate_simple(READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, + dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + /* Create chunked dataset */ + chunk_dims[0] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; + chunk_dims[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; + + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, + chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_NAME, + memtype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + VRFY((H5Dwrite(dset_id, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + } + + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + dset_id = + H5Dopen2(group_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + sel_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; + sel_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + + /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ + flat_dims[0] = sel_dims[0] * sel_dims[1]; + + memspace = H5Screate_simple(1, flat_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + /* + * Each process defines the dataset selection in the file and + * reads it to the selection in memory + */ + count[0] = 1; + count[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + stride[0] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; + stride[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; + block[0] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; + block[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; + start[0] = 0; + start[1] = ((hsize_t)mpi_rank * READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS); + + if (VERBOSE_MED) { + HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + read_buf_size = flat_dims[0] * sizeof(*read_buf); + + read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + VRFY((H5Dread(dset_id, memtype, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); + + global_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != global_buf), "HDcalloc succeeded"); + + /* Collect each piece of data from all ranks into a global buffer on all ranks */ + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + recvcounts[i] = (int)(flat_dims[0] * sizeof(*read_buf)); + + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * flat_dims[0] * sizeof(*read_buf)); + + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)(flat_dims[0] * sizeof(COMPOUND_C_DATATYPE)), MPI_BYTE, + global_buf, recvcounts, displs, MPI_BYTE, comm)), + "MPI_Allgatherv succeeded"); + + VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + if (displs) + HDfree(displs); + if (recvcounts) + HDfree(recvcounts); + if (global_buf) + HDfree(global_buf); + if (read_buf) + HDfree(read_buf); + if (correct_buf) + HDfree(correct_buf); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); + VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + return; +} + +/* + * Tests parallel read of filtered data from shared + * chunks using a compound datatype which doesn't + * require a datatype conversion. + * + * The MAINPROCESS rank will first write out all of the + * data to the dataset. Then, each rank reads a part of + * each chunk of the dataset and contributes its piece + * to a global buffer that is checked for consistency. + * + * Programmer: Jordan Henderson + * 05/17/2018 + */ +static void +test_read_cmpd_filtered_dataset_no_conversion_shared(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) +{ + COMPOUND_C_DATATYPE *read_buf = NULL; + COMPOUND_C_DATATYPE *correct_buf = NULL; + COMPOUND_C_DATATYPE *global_buf = NULL; + hsize_t dataset_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t chunk_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t sel_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t start[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t stride[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t count[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t block[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t flat_dims[1]; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID, + memtype = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; + + if (MAINPROCESS) + HDputs("Testing read from shared filtered chunks in Compound Datatype dataset without Datatype " + "conversion"); + + /* SZIP and ScaleOffset filters don't support compound types */ + if (filter_id == H5Z_FILTER_SZIP || filter_id == H5Z_FILTER_SCALEOFFSET) { + if (MAINPROCESS) + SKIPPED(); + return; + } + + dataset_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_NROWS; + dataset_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_NCOLS; + + /* Setup the buffer for writing and for comparison */ + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + + correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); + + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { + correct_buf[i].field1 = + (short)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + + correct_buf[i].field2 = + (int)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + + correct_buf[i].field3 = + (long)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + } + + /* Create the compound type for memory. */ + memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); + VRFY((memtype >= 0), "Datatype creation succeeded"); + + VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), + "Datatype insertion succeeded"); + + if (MAINPROCESS) { + plist_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((plist_id >= 0), "FAPL creation succeeded"); + + VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), + "Set libver bounds succeeded"); + + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + filespace = H5Screate_simple(READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, + dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + /* Create chunked dataset */ + chunk_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; + + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, + chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_NAME, + memtype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + VRFY((H5Dwrite(dset_id, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + } + + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + dset_id = + H5Dopen2(group_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + sel_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; + sel_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; + + /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ + flat_dims[0] = sel_dims[0] * sel_dims[1]; + + memspace = H5Screate_simple(1, flat_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + /* + * Each process defines the dataset selection in the file and + * reads it to the selection in memory + */ + count[0] = 1; + count[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; + stride[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS; + stride[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; + block[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; + block[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; + start[0] = (hsize_t)mpi_rank; + start[1] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + read_buf_size = flat_dims[0] * sizeof(*read_buf); + + read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + VRFY((H5Dread(dset_id, memtype, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); + + global_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != global_buf), "HDcalloc succeeded"); + + /* Collect each piece of data from all ranks into a global buffer on all ranks */ + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + recvcounts[i] = (int)(flat_dims[0] * sizeof(*read_buf)); + + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * flat_dims[0] * sizeof(*read_buf)); + + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)(flat_dims[0] * sizeof(COMPOUND_C_DATATYPE)), MPI_BYTE, + global_buf, recvcounts, displs, MPI_BYTE, comm)), + "MPI_Allgatherv succeeded"); + + VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + if (displs) + HDfree(displs); + if (recvcounts) + HDfree(recvcounts); + if (global_buf) + HDfree(global_buf); + if (read_buf) + HDfree(read_buf); + if (correct_buf) + HDfree(correct_buf); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); + VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + return; +} + +/* + * Tests parallel read of filtered data from unshared + * chunks using a compound datatype which requires a + * datatype conversion. + * + * The MAINPROCESS rank will first write out all of the + * data to the dataset. Then, each rank reads a part of + * the dataset and contributes its piece to a global + * buffer that is checked for consistency. + * + * Programmer: Jordan Henderson + * 05/17/2018 + */ +static void +test_read_cmpd_filtered_dataset_type_conversion_unshared(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) +{ + COMPOUND_C_DATATYPE *read_buf = NULL; + COMPOUND_C_DATATYPE *correct_buf = NULL; + COMPOUND_C_DATATYPE *global_buf = NULL; + hsize_t dataset_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t chunk_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t sel_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t start[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t stride[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t count[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t block[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; + hsize_t flat_dims[1]; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t filetype = H5I_INVALID_HID, memtype = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; + + if (MAINPROCESS) + HDputs("Testing read from unshared filtered chunks in Compound Datatype dataset with Datatype " + "conversion"); + + /* SZIP and ScaleOffset filters don't support compound types */ + if (filter_id == H5Z_FILTER_SZIP || filter_id == H5Z_FILTER_SCALEOFFSET) { + if (MAINPROCESS) + SKIPPED(); + return; + } + + dataset_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_NROWS; + dataset_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_NCOLS; + + /* Setup the buffer for writing and for comparison */ + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + + correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); + + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { + correct_buf[i].field1 = (short)((i % dataset_dims[1]) + (i / dataset_dims[1])); + + correct_buf[i].field2 = (int)((i % dataset_dims[1]) + (i / dataset_dims[1])); + + correct_buf[i].field3 = (long)((i % dataset_dims[1]) + (i / dataset_dims[1])); + } + + /* Create the compound type for memory. */ + memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); + VRFY((memtype >= 0), "Datatype creation succeeded"); + + VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), + "Datatype insertion succeeded"); + + /* Create the compound type for file. */ + filetype = H5Tcreate(H5T_COMPOUND, 32); + VRFY((filetype >= 0), "Datatype creation succeeded"); + + VRFY((H5Tinsert(filetype, "ShortData", 0, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((H5Tinsert(filetype, "IntData", 8, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((H5Tinsert(filetype, "LongData", 16, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + + if (MAINPROCESS) { + plist_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((plist_id >= 0), "FAPL creation succeeded"); + + VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), + "Set libver bounds succeeded"); + + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + filespace = H5Screate_simple(READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, + dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + /* Create chunked dataset */ + chunk_dims[0] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; + chunk_dims[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; + + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, + chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_NAME, + filetype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + VRFY((H5Dwrite(dset_id, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + } + + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + dset_id = + H5Dopen2(group_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + sel_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; + sel_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + + /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ + flat_dims[0] = sel_dims[0] * sel_dims[1]; + + memspace = H5Screate_simple(1, flat_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + /* + * Each process defines the dataset selection in the file and + * reads it to the selection in memory + */ + count[0] = 1; + count[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + stride[0] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; + stride[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; + block[0] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; + block[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; + start[0] = 0; + start[1] = ((hsize_t)mpi_rank * READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS); + + if (VERBOSE_MED) { + HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + read_buf_size = flat_dims[0] * sizeof(*read_buf); + + read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + VRFY((H5Dread(dset_id, memtype, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); + + global_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != global_buf), "HDcalloc succeeded"); + + /* Collect each piece of data from all ranks into a global buffer on all ranks */ + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + recvcounts[i] = (int)(flat_dims[0] * sizeof(*read_buf)); + + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * flat_dims[0] * sizeof(*read_buf)); + + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)(flat_dims[0] * sizeof(COMPOUND_C_DATATYPE)), MPI_BYTE, + global_buf, recvcounts, displs, MPI_BYTE, comm)), + "MPI_Allgatherv succeeded"); + + VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + if (displs) + HDfree(displs); + if (recvcounts) + HDfree(recvcounts); + if (global_buf) + HDfree(global_buf); + if (read_buf) + HDfree(read_buf); + if (correct_buf) + HDfree(correct_buf); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); + VRFY((H5Tclose(filetype) >= 0), "File datatype close succeeded"); + VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + return; +} + +/* + * Tests parallel read of filtered data from shared + * chunks using a compound datatype which requires + * a datatype conversion. + * + * The MAINPROCESS rank will first write out all of the + * data to the dataset. Then, each rank reads a part of + * each chunk of the dataset and contributes its pieces + * to a global buffer that is checked for consistency. + * + * Programmer: Jordan Henderson + * 05/17/2018 + */ +static void +test_read_cmpd_filtered_dataset_type_conversion_shared(const char *parent_group, H5Z_filter_t filter_id, + hid_t fapl_id, hid_t dcpl_id, hid_t dxpl_id) +{ + COMPOUND_C_DATATYPE *read_buf = NULL; + COMPOUND_C_DATATYPE *correct_buf = NULL; + COMPOUND_C_DATATYPE *global_buf = NULL; + hsize_t dataset_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t chunk_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t sel_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t start[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t stride[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t count[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t block[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; + hsize_t flat_dims[1]; + size_t i, read_buf_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t filetype = H5I_INVALID_HID, memtype = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; + + if (MAINPROCESS) + HDputs( + "Testing read from shared filtered chunks in Compound Datatype dataset with Datatype conversion"); + + /* SZIP and ScaleOffset filters don't support compound types */ + if (filter_id == H5Z_FILTER_SZIP || filter_id == H5Z_FILTER_SCALEOFFSET) { + if (MAINPROCESS) + SKIPPED(); + return; + } + + dataset_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_NROWS; + dataset_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_NCOLS; + + /* Setup the buffer for writing and for comparison */ + correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + + correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); + + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { + correct_buf[i].field1 = + (short)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + + correct_buf[i].field2 = + (int)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + + correct_buf[i].field3 = + (long)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + + (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + } + + /* Create the compound type for memory. */ + memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); + VRFY((memtype >= 0), "Datatype creation succeeded"); + + VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), + "Datatype insertion succeeded"); + VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), + "Datatype insertion succeeded"); + + /* Create the compound type for file. */ + filetype = H5Tcreate(H5T_COMPOUND, 32); + VRFY((filetype >= 0), "Datatype creation succeeded"); + + VRFY((H5Tinsert(filetype, "ShortData", 0, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((H5Tinsert(filetype, "IntData", 8, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((H5Tinsert(filetype, "LongData", 16, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + + if (MAINPROCESS) { + plist_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((plist_id >= 0), "FAPL creation succeeded"); + + VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), + "Set libver bounds succeeded"); + + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + filespace = H5Screate_simple(READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, + dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + /* Create chunked dataset */ + chunk_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS; + chunk_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; + + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, + chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_NAME, + filetype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + VRFY((H5Dwrite(dset_id, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + } + + file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + dset_id = + H5Dopen2(group_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + sel_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; + sel_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; + + /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ + flat_dims[0] = sel_dims[0] * sel_dims[1]; + + memspace = H5Screate_simple(1, flat_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + /* + * Each process defines the dataset selection in the file and + * reads it to the selection in memory + */ + count[0] = 1; + count[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; + stride[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS; + stride[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; + block[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; + block[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; + start[0] = (hsize_t)mpi_rank; + start[1] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + read_buf_size = flat_dims[0] * sizeof(*read_buf); + + read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + VRFY((H5Dread(dset_id, memtype, memspace, filespace, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); + + global_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != global_buf), "HDcalloc succeeded"); + + /* Collect each piece of data from all ranks into a global buffer on all ranks */ + recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + recvcounts[i] = (int)(flat_dims[0] * sizeof(*read_buf)); + + displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) + displs[i] = (int)(i * flat_dims[0] * sizeof(*read_buf)); + + VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)(flat_dims[0] * sizeof(COMPOUND_C_DATATYPE)), MPI_BYTE, + global_buf, recvcounts, displs, MPI_BYTE, comm)), + "MPI_Allgatherv succeeded"); + + VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + if (displs) + HDfree(displs); + if (recvcounts) + HDfree(recvcounts); + if (global_buf) + HDfree(global_buf); + if (read_buf) + HDfree(read_buf); + if (correct_buf) + HDfree(correct_buf); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); + VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + return; +} + +/* + * Tests write of filtered data to a dataset + * by a single process. After the write has + * succeeded, the dataset is closed and then + * re-opened in parallel and read by all + * processes to ensure data correctness. + * + * Programmer: Jordan Henderson + * 08/03/2017 + */ +static void +test_write_serial_read_parallel(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) +{ + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_SERIAL_READ_PARALLEL_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_SERIAL_READ_PARALLEL_DATASET_DIMS]; + size_t i, data_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; + + if (MAINPROCESS) + HDputs("Testing write file serially; read file in parallel"); + + dataset_dims[0] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_NROWS; + dataset_dims[1] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_NCOLS; + dataset_dims[2] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_DEPTH; + + /* Write the file on the MAINPROCESS rank */ + if (MAINPROCESS) { + /* Set up file access property list */ + plist_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((plist_id >= 0), "FAPL creation succeeded"); + + VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), + "Set libver bounds succeeded"); + + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + chunk_dims[0] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_CH_NCOLS; + chunk_dims[2] = 1; + + filespace = H5Screate_simple(WRITE_SERIAL_READ_PARALLEL_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, WRITE_SERIAL_READ_PARALLEL_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, WRITE_SERIAL_READ_PARALLEL_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + data_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*data); + + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); + + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); + + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + if (data) + HDfree(data); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + } + + correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + + correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); + + read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + correct_buf[i] = (long)i; + + /* All ranks open the file and verify their "portion" of the dataset is correct */ + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + dset_id = H5Dopen2(group_id, WRITE_SERIAL_READ_PARALLEL_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + + VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + + if (correct_buf) + HDfree(correct_buf); + if (read_buf) + HDfree(read_buf); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + return; +} + +#ifdef H5_HAVE_PARALLEL_FILTERED_WRITES +/* + * Tests parallel write of filtered data + * to a dataset. After the write has + * succeeded, the dataset is closed and + * then re-opened and read by a single + * process to ensure data correctness. + * + * Programmer: Jordan Henderson + * 08/03/2017 + */ +static void +test_write_parallel_read_serial(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, + hid_t dcpl_id, hid_t dxpl_id) +{ + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + hsize_t dataset_dims[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; + hsize_t sel_dims[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; + hsize_t count[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; + hsize_t stride[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; + hsize_t block[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; + hsize_t offset[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; + size_t i, data_size, correct_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; + + if (MAINPROCESS) + HDputs("Testing write file in parallel; read serially"); + + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); + + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); + + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_NROWS; + dataset_dims[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_NCOLS; + dataset_dims[2] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_DEPTH; + chunk_dims[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NCOLS; + chunk_dims[2] = 1; + sel_dims[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS; + sel_dims[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_NCOLS; + sel_dims[2] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_DEPTH; + + filespace = H5Screate_simple(WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); + + memspace = H5Screate_simple(WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS, sel_dims, NULL); + VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, WRITE_PARALLEL_READ_SERIAL_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = 1; + count[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_NCOLS / (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NCOLS; + count[2] = (hsize_t)mpi_size; + stride[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS; + stride[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NCOLS; + stride[2] = 1; + block[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS; + block[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NCOLS; + block[2] = 1; + offset[0] = ((hsize_t)mpi_rank * (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS * count[0]); + offset[1] = 0; + offset[2] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE + " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE " ], offset[ %" PRIuHSIZE + ", %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE + " ]\n", + mpi_rank, count[0], count[1], count[2], stride[0], stride[1], stride[2], offset[0], + offset[1], offset[2], block[0], block[1], block[2]); + HDfflush(stdout); + } + + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sel_dims[2] * sizeof(*data); + + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); - correct_buf[i].field2 = (int)((i % dataset_dims[1]) + (i / dataset_dims[1])); + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); - correct_buf[i].field3 = (long)((i % dataset_dims[1]) + (i / dataset_dims[1])); - } + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - /* Create the compound type for memory. */ - memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); - VRFY((memtype >= 0), "Datatype creation succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); - VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), - "Datatype insertion succeeded"); + if (data) + HDfree(data); + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); if (MAINPROCESS) { plist_id = H5Pcreate(H5P_FILE_ACCESS); @@ -5153,783 +7215,794 @@ test_read_cmpd_filtered_dataset_no_conversion_unshared(void) VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); - /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, - dataset_dims, NULL); - VRFY((filespace >= 0), "File dataspace creation succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - /* Create chunked dataset */ - chunk_dims[0] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; - chunk_dims[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; + dset_id = H5Dopen2(group_id, WRITE_PARALLEL_READ_SERIAL_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); - VRFY((H5Pset_chunk(plist_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_DIMS, - chunk_dims) >= 0), - "Chunk size set"); + correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); - /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); - dset_id = H5Dcreate2(file_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_NAME, - memtype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset creation succeeded"); + for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) + correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] * dataset_dims[1])) + + (i / (dataset_dims[0] * dataset_dims[1]))); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, read_buf) >= 0), + "Dataset read succeeded"); - VRFY((H5Dwrite(dset_id, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), - "Dataset write succeeded"); + VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + + HDfree(correct_buf); + HDfree(read_buf); } - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + return; +} - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); +/* + * Tests that causing chunks to continually grow and shrink + * by writing random data followed by zeroed-out data (and + * thus controlling the compression ratio) does not cause + * problems. + * + * Programmer: Jordan Henderson + * 06/04/2018 + */ +static void +test_shrinking_growing_chunks(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, hid_t dcpl_id, + hid_t dxpl_id) +{ + double *data = NULL; + double *read_buf = NULL; + hsize_t dataset_dims[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; + hsize_t start[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; + hsize_t stride[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; + hsize_t count[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; + hsize_t block[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; + size_t i, data_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID, memspace = H5I_INVALID_HID; - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + if (MAINPROCESS) + HDputs("Testing continually shrinking/growing chunks"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); - - dset_id = - H5Dopen2(file_id, "/" READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - sel_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; - sel_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_NCOLS; + chunk_dims[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NCOLS; + sel_dims[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS; + sel_dims[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_NCOLS; - /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ - flat_dims[0] = sel_dims[0] * sel_dims[1]; + filespace = H5Screate_simple(SHRINKING_GROWING_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(1, flat_dims, NULL); + memspace = H5Screate_simple(SHRINKING_GROWING_CHUNKS_DATASET_DIMS, sel_dims, NULL); VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - /* Select hyperslab in the file */ - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); + + VRFY((H5Pset_chunk(plist_id, SHRINKING_GROWING_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); + + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + dset_id = H5Dcreate2(group_id, SHRINKING_GROWING_CHUNKS_DATASET_NAME, H5T_NATIVE_DOUBLE, filespace, + H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); /* - * Each process defines the dataset selection in the file and - * reads it to the selection in memory + * Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file */ count[0] = 1; - count[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_ENTRIES_PER_PROC; - stride[0] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; - stride[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; - block[0] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NROWS; - block[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS; - start[0] = 0; - start[1] = ((hsize_t)mpi_rank * READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_UNSHARED_CH_NCOLS); + count[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_NCOLS / (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NCOLS; + stride[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS; + block[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NCOLS; + start[0] = ((hsize_t)mpi_rank * (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS * count[0]); + start[1] = 0; if (VERBOSE_MED) { - HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE " ]\n", mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); HDfflush(stdout); } + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((dset_id >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + data_size = sel_dims[0] * sel_dims[1] * sizeof(double); - read_buf_size = flat_dims[0] * sizeof(*read_buf); + data = (double *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); - read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, read_buf_size); + read_buf = (double *)HDcalloc(1, data_size); VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dread(dset_id, memtype, memspace, filespace, plist_id, read_buf) >= 0), "Dataset read succeeded"); - - global_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != global_buf), "HDcalloc succeeded"); - - /* Collect each piece of data from all ranks into a global buffer on all ranks */ - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); + for (i = 0; i < SHRINKING_GROWING_CHUNKS_NLOOPS; i++) { + /* Continually write random float data, followed by zeroed-out data */ + if (i % 2) + HDmemset(data, 0, data_size); + else { + size_t j; + for (j = 0; j < data_size / sizeof(*data); j++) { + data[j] = (rand() / (double)(RAND_MAX / (double)1.0L)); + } + } - for (i = 0; i < (size_t)mpi_size; i++) - recvcounts[i] = (int)(flat_dims[0] * sizeof(*read_buf)); + VRFY((H5Dwrite(dset_id, H5T_NATIVE_DOUBLE, memspace, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * flat_dims[0] * sizeof(*read_buf)); + if (i % 2) { + HDmemset(read_buf, 255, data_size); + } + else { + HDmemset(read_buf, 0, data_size); + } - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)(flat_dims[0] * sizeof(COMPOUND_C_DATATYPE)), MPI_BYTE, - global_buf, recvcounts, displs, MPI_BYTE, comm)), - "MPI_Allgatherv succeeded"); + VRFY((H5Dread(dset_id, H5T_NATIVE_DOUBLE, memspace, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + VRFY((0 == HDmemcmp(read_buf, data, data_size)), "data verification succeeded"); + } - if (displs) - HDfree(displs); - if (recvcounts) - HDfree(recvcounts); - if (global_buf) - HDfree(global_buf); if (read_buf) HDfree(read_buf); - if (correct_buf) - HDfree(correct_buf); + if (data) + HDfree(data); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel read of filtered data from shared - * chunks using a compound datatype which doesn't - * require a datatype conversion. - * - * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads a part of - * each chunk of the dataset and contributes its piece - * to a global buffer that is checked for consistency. + * Tests that filtered and unfiltered partial edge chunks can be + * written to and read from correctly in parallel when only one MPI + * rank writes to a particular partial edge chunk in the dataset. * - * Programmer: Jordan Henderson - * 05/17/2018 + * The dataset contains partial edge chunks in the second dimension. + * Each MPI rank selects a hyperslab in the shape of a single chunk + * that is offset to cover the whole edge chunk and part of the + * full chunk next to the edge chunk. */ static void -test_read_cmpd_filtered_dataset_no_conversion_shared(void) +test_edge_chunks_no_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, hid_t dcpl_id, + hid_t dxpl_id) { - COMPOUND_C_DATATYPE *read_buf = NULL; - COMPOUND_C_DATATYPE *correct_buf = NULL; - COMPOUND_C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t chunk_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t sel_dims[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t start[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t stride[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t count[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t block[READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id, dset_id, plist_id, memtype; - hid_t filespace, memspace; - int * recvcounts = NULL; - int * displs = NULL; + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + hsize_t dataset_dims[WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t start[WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t stride[WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t count[WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t block[WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + size_t i, data_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing read from shared filtered chunks in Compound Datatype dataset without Datatype " - "conversion"); - - CHECK_CUR_FILTER_AVAIL(); - - dataset_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_NROWS; - dataset_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_NCOLS; - - /* Setup the buffer for writing and for comparison */ - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); - - correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); + HDputs("Testing write to unshared filtered edge chunks"); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { - correct_buf[i].field1 = - (short)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); - correct_buf[i].field2 = - (int)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - correct_buf[i].field3 = - (long)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); - } + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + sel_dims[0] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NROWS; + sel_dims[1] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + + filespace = H5Screate_simple(WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); - /* Create the compound type for memory. */ - memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); - VRFY((memtype >= 0), "Datatype creation succeeded"); + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), - "Datatype insertion succeeded"); + VRFY((H5Pset_chunk(plist_id, WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); - if (MAINPROCESS) { - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + dset_id = H5Dcreate2(group_id, WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, - dataset_dims, NULL); - VRFY((filespace >= 0), "File dataspace creation succeeded"); + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = 1; + count[1] = 1; + stride[0] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NROWS; + block[1] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + start[0] = ((hsize_t)mpi_rank * (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NROWS); + start[1] = + (hsize_t)(WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_NCOLS - WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS); - /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - VRFY((H5Pset_chunk(plist_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_DIMS, - chunk_dims) >= 0), - "Chunk size set"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); - dset_id = H5Dcreate2(file_id, READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_NAME, - memtype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset creation succeeded"); + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + read_buf = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Dwrite(dset_id, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), - "Dataset write succeeded"); + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); - } + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, (mpi_size > 1) ? SOME_CHUNKS_WRITTEN : ALL_CHUNKS_WRITTEN); - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + /* Verify the correct data was written */ + dset_id = H5Dopen2(group_id, WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + VRFY((0 == HDmemcmp(read_buf, data, data_size)), "Data verification succeeded"); - dset_id = - H5Dopen2(file_id, "/" READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - sel_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; - sel_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; + /* Repeat the previous, but set option to not filter partial edge chunks */ + if (MAINPROCESS) + HDputs("Testing write to unshared unfiltered edge chunks"); - /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ - flat_dims[0] = sel_dims[0] * sel_dims[1]; + H5Pset_chunk_opts(plist_id, H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS); - memspace = H5Screate_simple(1, flat_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + dset_id = H5Dcreate2(group_id, WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME2, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); - /* Select hyperslab in the file */ - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - /* - * Each process defines the dataset selection in the file and - * reads it to the selection in memory + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file */ count[0] = 1; - count[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_ENTRIES_PER_PROC; - stride[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS; - stride[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; - block[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; - block[1] = READ_COMPOUND_FILTERED_CHUNKS_NO_CONVERSION_SHARED_CH_NCOLS; - start[0] = (hsize_t)mpi_rank; - start[1] = 0; + count[1] = 1; + stride[0] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NROWS; + block[1] = (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + start[0] = ((hsize_t)mpi_rank * (hsize_t)WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NROWS); + start[1] = + (hsize_t)(WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_NCOLS - WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS); if (VERBOSE_MED) { - HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE " ]\n", mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); HDfflush(stdout); } + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - read_buf_size = flat_dims[0] * sizeof(*read_buf); - - read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, read_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); - - VRFY((H5Dread(dset_id, memtype, memspace, filespace, plist_id, read_buf) >= 0), "Dataset read succeeded"); - - global_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != global_buf), "HDcalloc succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - /* Collect each piece of data from all ranks into a global buffer on all ranks */ - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, (mpi_size > 1) ? SOME_CHUNKS_WRITTEN : ALL_CHUNKS_WRITTEN); - for (i = 0; i < (size_t)mpi_size; i++) - recvcounts[i] = (int)(flat_dims[0] * sizeof(*read_buf)); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + /* Verify the correct data was written */ + dset_id = H5Dopen2(group_id, WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME2, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * flat_dims[0] * sizeof(*read_buf)); + HDmemset(read_buf, 255, data_size); - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)(flat_dims[0] * sizeof(COMPOUND_C_DATATYPE)), MPI_BYTE, - global_buf, recvcounts, displs, MPI_BYTE, comm)), - "MPI_Allgatherv succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + VRFY((0 == HDmemcmp(read_buf, data, data_size)), "Data verification succeeded"); - if (displs) - HDfree(displs); - if (recvcounts) - HDfree(recvcounts); - if (global_buf) - HDfree(global_buf); + if (data) + HDfree(data); if (read_buf) HDfree(read_buf); - if (correct_buf) - HDfree(correct_buf); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel read of filtered data from unshared - * chunks using a compound datatype which requires a - * datatype conversion. - * - * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads a part of - * the dataset and contributes its piece to a global - * buffer that is checked for consistency. + * Tests that filtered and unfiltered partial edge chunks can be + * written to and read from correctly in parallel when every MPI + * rank writes to every partial edge chunk in the dataset. * - * Programmer: Jordan Henderson - * 05/17/2018 + * The dataset contains partial edge chunks in the second dimension. + * Each MPI rank selects a hyperslab in the shape of one row of each + * chunk that is offset in the second dimension to cover the whole + * edge chunk and part of the full chunk next to the edge chunk. */ static void -test_read_cmpd_filtered_dataset_type_conversion_unshared(void) +test_edge_chunks_overlap(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, hid_t dcpl_id, + hid_t dxpl_id) { - COMPOUND_C_DATATYPE *read_buf = NULL; - COMPOUND_C_DATATYPE *correct_buf = NULL; - COMPOUND_C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t chunk_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t sel_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t start[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t stride[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t count[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t block[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS]; - hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1, filetype = -1, memtype = -1; - hid_t filespace = -1, memspace = -1; - int * recvcounts = NULL; - int * displs = NULL; + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + hsize_t dataset_dims[WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t chunk_dims[WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t sel_dims[WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t start[WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t stride[WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t count[WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + hsize_t block[WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS]; + size_t i, data_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing read from unshared filtered chunks in Compound Datatype dataset with Datatype " - "conversion"); - - CHECK_CUR_FILTER_AVAIL(); - - dataset_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_NROWS; - dataset_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_NCOLS; - - /* Setup the buffer for writing and for comparison */ - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); - - correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); + HDputs("Testing write to shared filtered edge chunks"); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { - correct_buf[i].field1 = (short)((i % dataset_dims[1]) + (i / dataset_dims[1])); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); - correct_buf[i].field2 = (int)((i % dataset_dims[1]) + (i / dataset_dims[1])); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - correct_buf[i].field3 = (long)((i % dataset_dims[1]) + (i / dataset_dims[1])); - } + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_NROWS; + dataset_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_NCOLS; + chunk_dims[0] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NROWS; + chunk_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; + sel_dims[1] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; - /* Create the compound type for memory. */ - memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); - VRFY((memtype >= 0), "Datatype creation succeeded"); + filespace = H5Screate_simple(WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); - VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), - "Datatype insertion succeeded"); + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - /* Create the compound type for file. */ - filetype = H5Tcreate(H5T_COMPOUND, 32); - VRFY((filetype >= 0), "Datatype creation succeeded"); + VRFY((H5Pset_chunk(plist_id, WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), + "Chunk size set"); - VRFY((H5Tinsert(filetype, "ShortData", 0, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - VRFY((H5Tinsert(filetype, "IntData", 8, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - VRFY((H5Tinsert(filetype, "LongData", 16, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - if (MAINPROCESS) { - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + dset_id = H5Dcreate2(group_id, WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = + (hsize_t)(WRITE_SHARED_FILTERED_EDGE_CHUNKS_NROWS / WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NROWS); + count[1] = 1; + stride[0] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)1; + block[1] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + start[0] = (hsize_t)mpi_rank; + start[1] = + (hsize_t)(WRITE_SHARED_FILTERED_EDGE_CHUNKS_NCOLS - WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS); - /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, - dataset_dims, NULL); - VRFY((filespace >= 0), "File dataspace creation succeeded"); + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } - /* Create chunked dataset */ - chunk_dims[0] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; - chunk_dims[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - VRFY((H5Pset_chunk(plist_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_DIMS, - chunk_dims) >= 0), - "Chunk size set"); + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); - /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); - dset_id = H5Dcreate2(file_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_NAME, - filetype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset creation succeeded"); + read_buf = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); - VRFY((H5Dwrite(dset_id, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), - "Dataset write succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); - } + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + /* Verify the correct data was written */ + dset_id = H5Dopen2(group_id, WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + VRFY((0 == HDmemcmp(read_buf, data, data_size)), "Data verification succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - dset_id = H5Dopen2(file_id, "/" READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_DATASET_NAME, - H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + /* Repeat the previous, but set option to not filter partial edge chunks */ + if (MAINPROCESS) + HDputs("Testing write to shared unfiltered edge chunks"); - sel_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; - sel_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; + H5Pset_chunk_opts(plist_id, H5D_CHUNK_DONT_FILTER_PARTIAL_CHUNKS); - /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ - flat_dims[0] = sel_dims[0] * sel_dims[1]; + dset_id = H5Dcreate2(group_id, WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME2, HDF5_DATATYPE_NAME, + filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); - memspace = H5Screate_simple(1, flat_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - /* Select hyperslab in the file */ - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - /* - * Each process defines the dataset selection in the file and - * reads it to the selection in memory + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file */ - count[0] = 1; - count[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_ENTRIES_PER_PROC; - stride[0] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; - stride[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; - block[0] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NROWS; - block[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS; - start[0] = 0; - start[1] = ((hsize_t)mpi_rank * READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_UNSHARED_CH_NCOLS); + count[0] = + (hsize_t)(WRITE_SHARED_FILTERED_EDGE_CHUNKS_NROWS / WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NROWS); + count[1] = 1; + stride[0] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NROWS; + stride[1] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + block[0] = (hsize_t)1; + block[1] = (hsize_t)WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS; + start[0] = (hsize_t)mpi_rank; + start[1] = + (hsize_t)(WRITE_SHARED_FILTERED_EDGE_CHUNKS_NCOLS - WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS); if (VERBOSE_MED) { - HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE " ]\n", mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); HDfflush(stdout); } + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - read_buf_size = flat_dims[0] * sizeof(*read_buf); - - read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, read_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); - - VRFY((H5Dread(dset_id, memtype, memspace, filespace, plist_id, read_buf) >= 0), "Dataset read succeeded"); - - global_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != global_buf), "HDcalloc succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - /* Collect each piece of data from all ranks into a global buffer on all ranks */ - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); - for (i = 0; i < (size_t)mpi_size; i++) - recvcounts[i] = (int)(flat_dims[0] * sizeof(*read_buf)); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + /* Verify the correct data was written */ + dset_id = H5Dopen2(group_id, WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME2, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * flat_dims[0] * sizeof(*read_buf)); + HDmemset(read_buf, 255, data_size); - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)(flat_dims[0] * sizeof(COMPOUND_C_DATATYPE)), MPI_BYTE, - global_buf, recvcounts, displs, MPI_BYTE, comm)), - "MPI_Allgatherv succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + VRFY((0 == HDmemcmp(read_buf, data, data_size)), "Data verification succeeded"); - if (displs) - HDfree(displs); - if (recvcounts) - HDfree(recvcounts); - if (global_buf) - HDfree(global_buf); + if (data) + HDfree(data); if (read_buf) HDfree(read_buf); - if (correct_buf) - HDfree(correct_buf); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Tclose(filetype) >= 0), "File datatype close succeeded"); - VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests parallel read of filtered data from shared - * chunks using a compound datatype which requires - * a datatype conversion. - * - * The MAINPROCESS rank will first write out all of the - * data to the dataset. Then, each rank reads a part of - * each chunk of the dataset and contributes its pieces - * to a global buffer that is checked for consistency. + * Tests that filtered and unfiltered partial edge chunks can be + * written to and read from correctly in parallel when only one + * MPI rank writes to a particular edge chunk in the dataset and + * only performs a partial write to the edge chunk. * - * Programmer: Jordan Henderson - * 05/17/2018 + * The dataset contains partial edge chunks in the second dimension. + * Each MPI rank selects a hyperslab in the shape of part of a single + * edge chunk and writes to just a portion of the edge chunk. */ static void -test_read_cmpd_filtered_dataset_type_conversion_shared(void) +test_edge_chunks_partial_write(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, hid_t dcpl_id, + hid_t dxpl_id) { - COMPOUND_C_DATATYPE *read_buf = NULL; - COMPOUND_C_DATATYPE *correct_buf = NULL; - COMPOUND_C_DATATYPE *global_buf = NULL; - hsize_t dataset_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t chunk_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t sel_dims[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t start[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t stride[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t count[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t block[READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS]; - hsize_t flat_dims[1]; - size_t i, read_buf_size, correct_buf_size; - hid_t file_id, dset_id, plist_id, filetype, memtype; - hid_t filespace, memspace; - int * recvcounts = NULL; - int * displs = NULL; + /* TODO */ +} + +/* + * Tests that the parallel compression feature correctly handles + * writing fill values to a dataset and reading fill values from + * unallocated parts of a dataset. + */ +static void +test_fill_values(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, hid_t dcpl_id, + hid_t dxpl_id) +{ + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *correct_buf = NULL; + C_DATATYPE fill_value; + hsize_t dataset_dims[FILL_VALUES_TEST_DATASET_DIMS]; + hsize_t chunk_dims[FILL_VALUES_TEST_DATASET_DIMS]; + hsize_t sel_dims[FILL_VALUES_TEST_DATASET_DIMS]; + hsize_t start[FILL_VALUES_TEST_DATASET_DIMS]; + hsize_t stride[FILL_VALUES_TEST_DATASET_DIMS]; + hsize_t count[FILL_VALUES_TEST_DATASET_DIMS]; + hsize_t block[FILL_VALUES_TEST_DATASET_DIMS]; + size_t i, data_size, read_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; if (MAINPROCESS) - HDputs( - "Testing read from shared filtered chunks in Compound Datatype dataset with Datatype conversion"); + HDputs("Testing fill values"); - CHECK_CUR_FILTER_AVAIL(); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "Test file open succeeded"); - dataset_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_NROWS; - dataset_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_NCOLS; + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); - /* Setup the buffer for writing and for comparison */ - correct_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*correct_buf); + /* Create the dataspace for the dataset */ + dataset_dims[0] = (hsize_t)FILL_VALUES_TEST_NROWS; + dataset_dims[1] = (hsize_t)FILL_VALUES_TEST_NCOLS; + chunk_dims[0] = (hsize_t)FILL_VALUES_TEST_CH_NROWS; + chunk_dims[1] = (hsize_t)FILL_VALUES_TEST_CH_NCOLS; + sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; + sel_dims[1] = (hsize_t)FILL_VALUES_TEST_CH_NCOLS * (hsize_t)DIM1_SCALE_FACTOR; - correct_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); + filespace = H5Screate_simple(FILL_VALUES_TEST_DATASET_DIMS, dataset_dims, NULL); + VRFY((filespace >= 0), "File dataspace creation succeeded"); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) { - correct_buf[i].field1 = - (short)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + /* Create chunked dataset */ + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - correct_buf[i].field2 = - (int)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); + VRFY((H5Pset_chunk(plist_id, FILL_VALUES_TEST_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); - correct_buf[i].field3 = - (long)((dataset_dims[1] * (i / ((hsize_t)mpi_size * dataset_dims[1]))) + (i % dataset_dims[1]) + - (((i % ((hsize_t)mpi_size * dataset_dims[1])) / dataset_dims[1]) % dataset_dims[1])); - } + /* Add test filter to the pipeline */ + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - /* Create the compound type for memory. */ - memtype = H5Tcreate(H5T_COMPOUND, sizeof(COMPOUND_C_DATATYPE)); - VRFY((memtype >= 0), "Datatype creation succeeded"); + /* Set a fill value */ + fill_value = FILL_VALUES_TEST_FILL_VAL; + VRFY((H5Pset_fill_value(plist_id, HDF5_DATATYPE_NAME, &fill_value) >= 0), "Fill Value set"); - VRFY((H5Tinsert(memtype, "ShortData", HOFFSET(COMPOUND_C_DATATYPE, field1), H5T_NATIVE_SHORT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "IntData", HOFFSET(COMPOUND_C_DATATYPE, field2), H5T_NATIVE_INT) >= 0), - "Datatype insertion succeeded"); - VRFY((H5Tinsert(memtype, "LongData", HOFFSET(COMPOUND_C_DATATYPE, field3), H5T_NATIVE_LONG) >= 0), - "Datatype insertion succeeded"); + dset_id = H5Dcreate2(group_id, FILL_VALUES_TEST_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, + plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); - /* Create the compound type for file. */ - filetype = H5Tcreate(H5T_COMPOUND, 32); - VRFY((filetype >= 0), "Datatype creation succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - VRFY((H5Tinsert(filetype, "ShortData", 0, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - VRFY((H5Tinsert(filetype, "IntData", 8, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); - VRFY((H5Tinsert(filetype, "LongData", 16, H5T_STD_I64BE) >= 0), "Datatype insertion succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - if (MAINPROCESS) { - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + /* Allocate buffer for reading entire dataset */ + read_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*read_buf); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + read_buf = HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + correct_buf = HDcalloc(1, read_buf_size); + VRFY((NULL != correct_buf), "HDcalloc succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + /* Read entire dataset and verify that the fill value is returned */ + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - /* Create the dataspace for the dataset */ - filespace = H5Screate_simple(READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, - dataset_dims, NULL); - VRFY((filespace >= 0), "File dataspace creation succeeded"); + for (i = 0; i < read_buf_size / sizeof(*read_buf); i++) + correct_buf[i] = FILL_VALUES_TEST_FILL_VAL; - /* Create chunked dataset */ - chunk_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS; - chunk_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; + VRFY((0 == HDmemcmp(read_buf, correct_buf, read_buf_size)), "Data verification succeeded"); - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + /* + * Write to part of the first chunk in the dataset with + * all ranks, then read the whole dataset and ensure that + * the fill value is returned for the unwritten part of + * the chunk, as well as for the rest of the dataset that + * hasn't been written to yet. + */ + count[0] = 1; + count[1] = 1; + stride[0] = (hsize_t)FILL_VALUES_TEST_CH_NROWS; + stride[1] = (hsize_t)FILL_VALUES_TEST_CH_NCOLS; + block[0] = 1; + block[1] = (hsize_t)(FILL_VALUES_TEST_CH_NCOLS - 1); + start[0] = (hsize_t)mpi_rank; + start[1] = 0; - VRFY((H5Pset_chunk(plist_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_DIMS, - chunk_dims) >= 0), - "Chunk size set"); + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } - /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - dset_id = H5Dcreate2(file_id, READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_NAME, - filetype, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset creation succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); - VRFY((H5Dwrite(dset_id, memtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, correct_buf) >= 0), - "Dataset write succeeded"); + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); - } + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDONLY, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + /* Verify correct data was written */ + dset_id = H5Dopen2(group_id, FILL_VALUES_TEST_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - dset_id = - H5Dopen2(file_id, "/" READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + /* + * Each MPI rank communicates their written piece of data + * into each other rank's correctness-checking buffer + */ + recvcounts = HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); - sel_dims[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; - sel_dims[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; + displs = HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); - /* Setup one-dimensional memory dataspace for reading the dataset data into a contiguous buffer */ - flat_dims[0] = sel_dims[0] * sel_dims[1]; + for (i = 0; i < (size_t)mpi_size; i++) { + recvcounts[i] = (int)(count[1] * block[1]); + displs[i] = (int)(i * dataset_dims[1]); + } - memspace = H5Screate_simple(1, flat_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); + VRFY((MPI_SUCCESS == MPI_Allgatherv(data, recvcounts[mpi_rank], C_DATATYPE_MPI, correct_buf, recvcounts, + displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); - /* Select hyperslab in the file */ - filespace = H5Dget_space(dset_id); - VRFY((filespace >= 0), "File dataspace retrieval succeeded"); + VRFY((0 == HDmemcmp(read_buf, correct_buf, read_buf_size)), "Data verification succeeded"); /* - * Each process defines the dataset selection in the file and - * reads it to the selection in memory + * Write to whole dataset and ensure fill value isn't returned + * after reading whole dataset back */ - count[0] = 1; - count[1] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_ENTRIES_PER_PROC; - stride[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS; - stride[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; - block[0] = (hsize_t)READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NROWS / (hsize_t)mpi_size; - block[1] = READ_COMPOUND_FILTERED_CHUNKS_TYPE_CONVERSION_SHARED_CH_NCOLS; - start[0] = (hsize_t)mpi_rank; + + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = (hsize_t)FILL_VALUES_TEST_NROWS / (hsize_t)FILL_VALUES_TEST_CH_NROWS; + count[1] = (hsize_t)FILL_VALUES_TEST_NCOLS / (hsize_t)FILL_VALUES_TEST_CH_NCOLS; + stride[0] = (hsize_t)FILL_VALUES_TEST_CH_NROWS; + stride[1] = (hsize_t)FILL_VALUES_TEST_CH_NCOLS; + block[0] = (hsize_t)FILL_VALUES_TEST_CH_NROWS / (hsize_t)mpi_size; + block[1] = (hsize_t)FILL_VALUES_TEST_CH_NCOLS; + start[0] = (hsize_t)mpi_rank * block[0]; start[1] = 0; if (VERBOSE_MED) { - HDprintf("Process %d is reading with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE " ]\n", mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); @@ -5939,304 +8012,290 @@ test_read_cmpd_filtered_dataset_type_conversion_shared(void) VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset read */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - read_buf_size = flat_dims[0] * sizeof(*read_buf); - - read_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, read_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); - - VRFY((H5Dread(dset_id, memtype, memspace, filespace, plist_id, read_buf) >= 0), "Dataset read succeeded"); - - global_buf = (COMPOUND_C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != global_buf), "HDcalloc succeeded"); - - /* Collect each piece of data from all ranks into a global buffer on all ranks */ - recvcounts = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); - VRFY((NULL != recvcounts), "HDcalloc succeeded"); - - for (i = 0; i < (size_t)mpi_size; i++) - recvcounts[i] = (int)(flat_dims[0] * sizeof(*read_buf)); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - displs = (int *)HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); - VRFY((NULL != displs), "HDcalloc succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); - for (i = 0; i < (size_t)mpi_size; i++) - displs[i] = (int)(i * flat_dims[0] * sizeof(*read_buf)); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((MPI_SUCCESS == MPI_Allgatherv(read_buf, (int)(flat_dims[0] * sizeof(COMPOUND_C_DATATYPE)), MPI_BYTE, - global_buf, recvcounts, displs, MPI_BYTE, comm)), - "MPI_Allgatherv succeeded"); + /* Verify correct data was written */ + dset_id = H5Dopen2(group_id, FILL_VALUES_TEST_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((0 == HDmemcmp(global_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - if (displs) - HDfree(displs); - if (recvcounts) - HDfree(recvcounts); - if (global_buf) - HDfree(global_buf); - if (read_buf) - HDfree(read_buf); - if (correct_buf) - HDfree(correct_buf); + for (i = 0; i < read_buf_size / sizeof(*read_buf); i++) + VRFY((read_buf[i] != FILL_VALUES_TEST_FILL_VAL), "Data verification succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Tclose(memtype) >= 0), "Memory datatype close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); - - return; -} - -/* - * Tests write of filtered data to a dataset - * by a single process. After the write has - * succeeded, the dataset is closed and then - * re-opened in parallel and read by all - * processes to ensure data correctness. - * - * Programmer: Jordan Henderson - * 08/03/2017 - */ -static void -test_write_serial_read_parallel(void) -{ - C_DATATYPE *data = NULL; - C_DATATYPE *read_buf = NULL; - C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_SERIAL_READ_PARALLEL_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_SERIAL_READ_PARALLEL_DATASET_DIMS]; - size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1; - if (MAINPROCESS) - HDputs("Testing write file serially; read file in parallel"); + /******************************************************************** + * Set the fill time to H5D_FILL_TIME_ALLOC and repeat the previous * + ********************************************************************/ - CHECK_CUR_FILTER_AVAIL(); - - dataset_dims[0] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_NROWS; - dataset_dims[1] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_NCOLS; - dataset_dims[2] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_DEPTH; + VRFY((H5Pset_fill_time(plist_id, H5D_FILL_TIME_ALLOC) >= 0), "H5Pset_fill_time succeeded"); - /* Write the file on the MAINPROCESS rank */ - if (MAINPROCESS) { - /* Set up file access property list */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + dset_id = H5Dcreate2(group_id, FILL_VALUES_TEST_DATASET_NAME2, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, + plist_id, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + /* Read entire dataset and verify that the fill value is returned */ + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - /* Create the dataspace for the dataset */ - chunk_dims[0] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_SERIAL_READ_PARALLEL_CH_NCOLS; - chunk_dims[2] = 1; + for (i = 0; i < read_buf_size / sizeof(*read_buf); i++) + correct_buf[i] = FILL_VALUES_TEST_FILL_VAL; - filespace = H5Screate_simple(WRITE_SERIAL_READ_PARALLEL_DATASET_DIMS, dataset_dims, NULL); - VRFY((filespace >= 0), "File dataspace creation succeeded"); + VRFY((0 == HDmemcmp(read_buf, correct_buf, read_buf_size)), "Data verification succeeded"); - /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + /* + * Write to part of the first chunk in the dataset with + * all ranks, then read the whole dataset and ensure that + * the fill value is returned for the unwritten part of + * the chunk, as well as for the rest of the dataset that + * hasn't been written to yet. + */ + count[0] = 1; + count[1] = 1; + stride[0] = (hsize_t)FILL_VALUES_TEST_CH_NROWS; + stride[1] = (hsize_t)FILL_VALUES_TEST_CH_NCOLS; + block[0] = 1; + block[1] = (hsize_t)(FILL_VALUES_TEST_CH_NCOLS - 1); + start[0] = (hsize_t)mpi_rank; + start[1] = 0; - VRFY((H5Pset_chunk(plist_id, WRITE_SERIAL_READ_PARALLEL_DATASET_DIMS, chunk_dims) >= 0), - "Chunk size set"); + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } - /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + /* Select hyperslab in the file */ + filespace = H5Dget_space(dset_id); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - dset_id = H5Dcreate2(file_id, WRITE_SERIAL_READ_PARALLEL_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, - H5P_DEFAULT, plist_id, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset creation succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); - data_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*data); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - data = (C_DATATYPE *)HDcalloc(1, data_size); - VRFY((NULL != data), "HDcalloc succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); - for (i = 0; i < data_size / sizeof(*data); i++) - data[i] = (C_DATATYPE)GEN_DATA(i); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) >= 0), - "Dataset write succeeded"); + /* Verify correct data was written */ + dset_id = H5Dopen2(group_id, FILL_VALUES_TEST_DATASET_NAME2, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - if (data) - HDfree(data); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + for (i = 0; i < (size_t)mpi_size; i++) { + recvcounts[i] = (int)(count[1] * block[1]); + displs[i] = (int)(i * dataset_dims[1]); } - correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + /* + * Each MPI rank communicates their written piece of data + * into each other rank's correctness-checking buffer + */ + VRFY((MPI_SUCCESS == MPI_Allgatherv(data, recvcounts[mpi_rank], C_DATATYPE_MPI, correct_buf, recvcounts, + displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); - correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); + VRFY((0 == HDmemcmp(read_buf, correct_buf, read_buf_size)), "Data verification succeeded"); - read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); + /* + * Write to whole dataset and ensure fill value isn't returned + * after reading whole dataset back + */ - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (long)i; + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = (hsize_t)FILL_VALUES_TEST_NROWS / (hsize_t)FILL_VALUES_TEST_CH_NROWS; + count[1] = (hsize_t)FILL_VALUES_TEST_NCOLS / (hsize_t)FILL_VALUES_TEST_CH_NCOLS; + stride[0] = (hsize_t)FILL_VALUES_TEST_CH_NROWS; + stride[1] = (hsize_t)FILL_VALUES_TEST_CH_NCOLS; + block[0] = (hsize_t)FILL_VALUES_TEST_CH_NROWS / (hsize_t)mpi_size; + block[1] = (hsize_t)FILL_VALUES_TEST_CH_NCOLS; + start[0] = (hsize_t)mpi_rank * block[0]; + start[1] = 0; - /* All ranks open the file and verify their "portion" of the dataset is correct */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - dset_id = H5Dopen2(file_id, "/" WRITE_SERIAL_READ_PARALLEL_DATASET_NAME, H5P_DEFAULT); + /* Verify correct data was written */ + dset_id = H5Dopen2(group_id, FILL_VALUES_TEST_DATASET_NAME2, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset open succeeded"); - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, plist_id, read_buf) >= 0), + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), "Dataset read succeeded"); - VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + for (i = 0; i < read_buf_size / sizeof(*read_buf); i++) + VRFY((read_buf[i] != FILL_VALUES_TEST_FILL_VAL), "Data verification succeeded"); - if (correct_buf) - HDfree(correct_buf); + if (displs) + HDfree(displs); + if (recvcounts) + HDfree(recvcounts); + if (data) + HDfree(data); if (read_buf) HDfree(read_buf); + if (correct_buf) + HDfree(correct_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } -#if MPI_VERSION >= 3 /* - * Tests parallel write of filtered data - * to a dataset. After the write has - * succeeded, the dataset is closed and - * then re-opened and read by a single - * process to ensure data correctness. - * - * Programmer: Jordan Henderson - * 08/03/2017 + * Tests that the parallel compression feature can handle + * an undefined fill value. Nothing is verified in this + * test since the fill value isn't defined. */ static void -test_write_parallel_read_serial(void) +test_fill_value_undefined(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, hid_t dcpl_id, + hid_t dxpl_id) { - C_DATATYPE *data = NULL; - C_DATATYPE *read_buf = NULL; - C_DATATYPE *correct_buf = NULL; - hsize_t dataset_dims[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; - hsize_t chunk_dims[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; - hsize_t sel_dims[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; - hsize_t count[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; - hsize_t stride[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; - hsize_t block[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; - hsize_t offset[WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS]; - size_t i, data_size, correct_buf_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + H5D_alloc_time_t alloc_time; + C_DATATYPE * data = NULL; + C_DATATYPE * read_buf = NULL; + hsize_t dataset_dims[FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS]; + hsize_t chunk_dims[FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS]; + hsize_t sel_dims[FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS]; + hsize_t start[FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS]; + hsize_t stride[FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS]; + hsize_t count[FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS]; + hsize_t block[FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS]; + size_t i, data_size, read_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; if (MAINPROCESS) - HDputs("Testing write file in parallel; read serially"); + HDputs("Testing undefined fill value"); - CHECK_CUR_FILTER_AVAIL(); + VRFY((H5Pget_alloc_time(dcpl_id, &alloc_time) >= 0), "H5Pget_alloc_time succeeded"); - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_NROWS; - dataset_dims[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_NCOLS; - dataset_dims[2] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_DEPTH; - chunk_dims[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS; - chunk_dims[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NCOLS; - chunk_dims[2] = 1; - sel_dims[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS; - sel_dims[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_NCOLS; - sel_dims[2] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_DEPTH; + dataset_dims[0] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_NROWS; + dataset_dims[1] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_NCOLS; + chunk_dims[0] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NROWS; + chunk_dims[1] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NCOLS; + sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; + sel_dims[1] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NCOLS * (hsize_t)DIM1_SCALE_FACTOR; - filespace = H5Screate_simple(WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS, sel_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, WRITE_PARALLEL_READ_SERIAL_DATASET_DIMS, chunk_dims) >= 0), - "Chunk size set"); + VRFY((H5Pset_chunk(plist_id, FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); - dset_id = H5Dcreate2(file_id, WRITE_PARALLEL_READ_SERIAL_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, + /* Set an undefined fill value */ + VRFY((H5Pset_fill_value(plist_id, HDF5_DATATYPE_NAME, NULL) >= 0), "Fill Value set"); + + dset_id = H5Dcreate2(group_id, FILL_VALUE_UNDEFINED_TEST_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - /* Each process defines the dataset selection in memory and writes - * it to the hyperslab in the file + /* Allocate buffer for reading entire dataset */ + read_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*read_buf); + + read_buf = HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + /* + * Read entire dataset - nothing to verify since there's no fill value. + * If not using early space allocation, the read should fail since storage + * isn't allocated yet and no fill value is defined. */ - count[0] = 1; - count[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_NCOLS / (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NCOLS; - count[2] = (hsize_t)mpi_size; - stride[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS; - stride[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NCOLS; - stride[2] = 1; - block[0] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS; - block[1] = (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NCOLS; - block[2] = 1; - offset[0] = ((hsize_t)mpi_rank * (hsize_t)WRITE_PARALLEL_READ_SERIAL_CH_NROWS * count[0]); - offset[1] = 0; - offset[2] = 0; + if (alloc_time == H5D_ALLOC_TIME_EARLY) { + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + } + else { + H5E_BEGIN_TRY + { + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) < 0), + "Dataset read succeeded"); + } + H5E_END_TRY; + } + + /* + * Write to part of the first chunk in the dataset with + * all ranks, then read the whole dataset. Don't verify + * anything since there's no fill value defined. + */ + count[0] = 1; + count[1] = 1; + stride[0] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NROWS; + stride[1] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NCOLS; + block[0] = 1; + block[1] = (hsize_t)(FILL_VALUE_UNDEFINED_TEST_CH_NCOLS - 1); + start[0] = (hsize_t)mpi_rank; + start[1] = 0; if (VERBOSE_MED) { - HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE - " ], stride[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE " ], offset[ %" PRIuHSIZE - ", %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE - " ]\n", - mpi_rank, count[0], count[1], count[2], stride[0], stride[1], stride[2], offset[0], - offset[1], offset[2], block[0], block[1], block[2]); + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); HDfflush(stdout); } @@ -6244,11 +8303,11 @@ test_write_parallel_read_serial(void) filespace = H5Dget_space(dset_id); VRFY((filespace >= 0), "File dataspace retrieval succeeded"); - VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, stride, count, block) >= 0), + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); /* Fill data buffer */ - data_size = sel_dims[0] * sel_dims[1] * sel_dims[2] * sizeof(*data); + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); data = (C_DATATYPE *)HDcalloc(1, data_size); VRFY((NULL != data), "HDcalloc succeeded"); @@ -6256,150 +8315,204 @@ test_write_parallel_read_serial(void) for (i = 0; i < data_size / sizeof(*data); i++) data[i] = (C_DATATYPE)GEN_DATA(i); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); - - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); - - VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, memspace, filespace, plist_id, data) >= 0), + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), "Dataset write succeeded"); - if (data) - HDfree(data); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); - VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); - - if (MAINPROCESS) { - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + dset_id = H5Dopen2(group_id, FILL_VALUE_UNDEFINED_TEST_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); - VRFY((file_id >= 0), "Test file open succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + /* + * Write to whole dataset and ensure data is correct + * after reading whole dataset back + */ - dset_id = H5Dopen2(file_id, "/" WRITE_PARALLEL_READ_SERIAL_DATASET_NAME, H5P_DEFAULT); - VRFY((dset_id >= 0), "Dataset open succeeded"); + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_NROWS / (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NROWS; + count[1] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_NCOLS / (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NCOLS; + stride[0] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NROWS; + stride[1] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NCOLS; + block[0] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NROWS / (hsize_t)mpi_size; + block[1] = (hsize_t)FILL_VALUE_UNDEFINED_TEST_CH_NCOLS; + start[0] = (hsize_t)mpi_rank * block[0]; + start[1] = 0; - correct_buf_size = dataset_dims[0] * dataset_dims[1] * dataset_dims[2] * sizeof(*correct_buf); + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); + } - correct_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != correct_buf), "HDcalloc succeeded"); + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); - read_buf = (C_DATATYPE *)HDcalloc(1, correct_buf_size); - VRFY((NULL != read_buf), "HDcalloc succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - for (i = 0; i < correct_buf_size / sizeof(*correct_buf); i++) - correct_buf[i] = (C_DATATYPE)((i % (dataset_dims[0] * dataset_dims[1])) + - (i / (dataset_dims[0] * dataset_dims[1]))); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); - VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, H5P_DEFAULT, read_buf) >= 0), - "Dataset read succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((0 == HDmemcmp(read_buf, correct_buf, correct_buf_size)), "Data verification succeeded"); + /* Verify correct data was written */ + dset_id = H5Dopen2(group_id, FILL_VALUE_UNDEFINED_TEST_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); - VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); - HDfree(correct_buf); + if (data) + HDfree(data); + if (read_buf) HDfree(read_buf); - } + + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); + VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; } /* - * Tests that causing chunks to continually grow and shrink - * by writing random data followed by zeroed-out data (and - * thus controlling the compression ratio) does not cause - * problems. - * - * Programmer: Jordan Henderson - * 06/04/2018 + * Tests that the parallel compression feature correctly handles + * avoiding writing fill values to a dataset when the fill time + * is set as H5D_FILL_TIME_NEVER. */ static void -test_shrinking_growing_chunks(void) +test_fill_time_never(const char *parent_group, H5Z_filter_t filter_id, hid_t fapl_id, hid_t dcpl_id, + hid_t dxpl_id) { - double *data = NULL; - hsize_t dataset_dims[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; - hsize_t chunk_dims[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; - hsize_t sel_dims[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; - hsize_t start[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; - hsize_t stride[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; - hsize_t count[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; - hsize_t block[SHRINKING_GROWING_CHUNKS_DATASET_DIMS]; - size_t i, data_size; - hid_t file_id = -1, dset_id = -1, plist_id = -1; - hid_t filespace = -1, memspace = -1; + C_DATATYPE *data = NULL; + C_DATATYPE *read_buf = NULL; + C_DATATYPE *fill_buf = NULL; + C_DATATYPE fill_value; + hsize_t dataset_dims[FILL_TIME_NEVER_TEST_DATASET_DIMS]; + hsize_t chunk_dims[FILL_TIME_NEVER_TEST_DATASET_DIMS]; + hsize_t sel_dims[FILL_TIME_NEVER_TEST_DATASET_DIMS]; + hsize_t start[FILL_TIME_NEVER_TEST_DATASET_DIMS]; + hsize_t stride[FILL_TIME_NEVER_TEST_DATASET_DIMS]; + hsize_t count[FILL_TIME_NEVER_TEST_DATASET_DIMS]; + hsize_t block[FILL_TIME_NEVER_TEST_DATASET_DIMS]; + size_t i, data_size, read_buf_size; + hid_t file_id = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, plist_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; + int * recvcounts = NULL; + int * displs = NULL; if (MAINPROCESS) - HDputs("Testing continually shrinking/growing chunks"); - - CHECK_CUR_FILTER_AVAIL(); + HDputs("Testing fill time H5D_FILL_TIME_NEVER"); - /* Set up file access property list with parallel I/O access */ - plist_id = H5Pcreate(H5P_FILE_ACCESS); - VRFY((plist_id >= 0), "FAPL creation succeeded"); + /* + * Only run this test when incremental file space allocation is + * used, as HDF5's chunk allocation code always writes fill values + * when filters are in the pipeline, but parallel compression does + * incremental file space allocation differently. + */ + { + H5D_alloc_time_t alloc_time; - VRFY((H5Pset_fapl_mpio(plist_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + VRFY((H5Pget_alloc_time(dcpl_id, &alloc_time) >= 0), "H5Pget_alloc_time succeeded"); - VRFY((H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + if (alloc_time != H5D_ALLOC_TIME_INCR) { + if (MAINPROCESS) + SKIPPED(); + return; + } + } - file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, plist_id); + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); VRFY((file_id >= 0), "Test file open succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "FAPL close succeeded"); + group_id = H5Gopen2(file_id, parent_group, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gopen2 succeeded"); /* Create the dataspace for the dataset */ - dataset_dims[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_NROWS; - dataset_dims[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_NCOLS; - chunk_dims[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS; - chunk_dims[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NCOLS; - sel_dims[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS; - sel_dims[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_NCOLS; + dataset_dims[0] = (hsize_t)FILL_TIME_NEVER_TEST_NROWS; + dataset_dims[1] = (hsize_t)FILL_TIME_NEVER_TEST_NCOLS; + chunk_dims[0] = (hsize_t)FILL_TIME_NEVER_TEST_CH_NROWS; + chunk_dims[1] = (hsize_t)FILL_TIME_NEVER_TEST_CH_NCOLS; + sel_dims[0] = (hsize_t)DIM0_SCALE_FACTOR; + sel_dims[1] = (hsize_t)FILL_TIME_NEVER_TEST_CH_NCOLS * (hsize_t)DIM1_SCALE_FACTOR; - filespace = H5Screate_simple(SHRINKING_GROWING_CHUNKS_DATASET_DIMS, dataset_dims, NULL); + filespace = H5Screate_simple(FILL_TIME_NEVER_TEST_DATASET_DIMS, dataset_dims, NULL); VRFY((filespace >= 0), "File dataspace creation succeeded"); - memspace = H5Screate_simple(SHRINKING_GROWING_CHUNKS_DATASET_DIMS, sel_dims, NULL); - VRFY((memspace >= 0), "Memory dataspace creation succeeded"); - /* Create chunked dataset */ - plist_id = H5Pcreate(H5P_DATASET_CREATE); - VRFY((plist_id >= 0), "DCPL creation succeeded"); + plist_id = H5Pcopy(dcpl_id); + VRFY((plist_id >= 0), "DCPL copy succeeded"); - VRFY((H5Pset_chunk(plist_id, SHRINKING_GROWING_CHUNKS_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); + VRFY((H5Pset_chunk(plist_id, FILL_TIME_NEVER_TEST_DATASET_DIMS, chunk_dims) >= 0), "Chunk size set"); /* Add test filter to the pipeline */ - VRFY((set_dcpl_filter(plist_id) >= 0), "Filter set"); + VRFY((set_dcpl_filter(plist_id, filter_id, NULL) >= 0), "Filter set"); + + /* Set a fill value */ + fill_value = FILL_VALUES_TEST_FILL_VAL; + VRFY((H5Pset_fill_value(plist_id, HDF5_DATATYPE_NAME, &fill_value) >= 0), "Fill Value set"); - dset_id = H5Dcreate2(file_id, SHRINKING_GROWING_CHUNKS_DATASET_NAME, H5T_NATIVE_DOUBLE, filespace, + /* Set fill time of 'never' */ + VRFY((H5Pset_fill_time(plist_id, H5D_FILL_TIME_NEVER) >= 0), "H5Pset_fill_time succeeded"); + + dset_id = H5Dcreate2(group_id, FILL_TIME_NEVER_TEST_DATASET_NAME, HDF5_DATATYPE_NAME, filespace, H5P_DEFAULT, plist_id, H5P_DEFAULT); VRFY((dset_id >= 0), "Dataset creation succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, DATASET_JUST_CREATED); + VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); + /* Allocate buffer for reading entire dataset */ + read_buf_size = dataset_dims[0] * dataset_dims[1] * sizeof(*read_buf); + + read_buf = HDcalloc(1, read_buf_size); + VRFY((NULL != read_buf), "HDcalloc succeeded"); + + fill_buf = HDcalloc(1, read_buf_size); + VRFY((NULL != fill_buf), "HDcalloc succeeded"); + + /* Read entire dataset and verify that the fill value isn't returned */ + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + + for (i = 0; i < read_buf_size / sizeof(*read_buf); i++) + fill_buf[i] = FILL_TIME_NEVER_TEST_FILL_VAL; + /* - * Each process defines the dataset selection in memory and writes - * it to the hyperslab in the file + * It should be very unlikely for the dataset's random + * values to all be the fill value, so this should be + * a safe comparison in theory. + */ + VRFY((0 != HDmemcmp(read_buf, fill_buf, read_buf_size)), "Data verification succeeded"); + + /* + * Write to part of the first chunk in the dataset with + * all ranks, then read the whole dataset and ensure that + * the fill value isn't returned for the unwritten part of + * the chunk, as well as for the rest of the dataset that + * hasn't been written to yet. */ count[0] = 1; - count[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_NCOLS / (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NCOLS; - stride[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS; - stride[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NCOLS; - block[0] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS; - block[1] = (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NCOLS; - start[0] = ((hsize_t)mpi_rank * (hsize_t)SHRINKING_GROWING_CHUNKS_CH_NROWS * count[0]); + count[1] = 1; + stride[0] = (hsize_t)FILL_TIME_NEVER_TEST_CH_NROWS; + stride[1] = (hsize_t)FILL_TIME_NEVER_TEST_CH_NCOLS; + block[0] = 1; + block[1] = (hsize_t)(FILL_TIME_NEVER_TEST_CH_NCOLS - 1); + start[0] = (hsize_t)mpi_rank; start[1] = 0; if (VERBOSE_MED) { @@ -6412,44 +8525,122 @@ test_shrinking_growing_chunks(void) /* Select hyperslab in the file */ filespace = H5Dget_space(dset_id); - VRFY((dset_id >= 0), "File dataspace retrieval succeeded"); + VRFY((filespace >= 0), "File dataspace retrieval succeeded"); VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), "Hyperslab selection succeeded"); - /* Create property list for collective dataset write */ - plist_id = H5Pcreate(H5P_DATASET_XFER); - VRFY((plist_id >= 0), "DXPL creation succeeded"); + /* Fill data buffer */ + data_size = sel_dims[0] * sel_dims[1] * sizeof(*data); - VRFY((H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE) >= 0), "Set DXPL MPIO succeeded"); + data = (C_DATATYPE *)HDcalloc(1, data_size); + VRFY((NULL != data), "HDcalloc succeeded"); - data_size = sel_dims[0] * sel_dims[1] * sizeof(double); + for (i = 0; i < data_size / sizeof(*data); i++) + data[i] = (C_DATATYPE)GEN_DATA(i); - data = (double *)HDcalloc(1, data_size); - VRFY((NULL != data), "HDcalloc succeeded"); + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); - for (i = 0; i < SHRINKING_GROWING_CHUNKS_NLOOPS; i++) { - /* Continually write random float data, followed by zeroed-out data */ - if ((i % 2)) - HDmemset(data, 0, data_size); - else { - size_t j; - for (j = 0; j < data_size / sizeof(*data); j++) { - data[j] = (float)(rand() / (double)(RAND_MAX / (double)1.0L)); - } - } + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, SOME_CHUNKS_WRITTEN); - VRFY((H5Dwrite(dset_id, H5T_NATIVE_DOUBLE, memspace, filespace, plist_id, data) >= 0), - "Dataset write succeeded"); + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + + /* Verify correct data was written */ + dset_id = H5Dopen2(group_id, FILL_TIME_NEVER_TEST_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + + /* + * Each MPI rank communicates their written piece of data + * into each other rank's correctness-checking buffer + */ + recvcounts = HDcalloc(1, (size_t)mpi_size * sizeof(*recvcounts)); + VRFY((NULL != recvcounts), "HDcalloc succeeded"); + + displs = HDcalloc(1, (size_t)mpi_size * sizeof(*displs)); + VRFY((NULL != displs), "HDcalloc succeeded"); + + for (i = 0; i < (size_t)mpi_size; i++) { + recvcounts[i] = (int)(count[1] * block[1]); + displs[i] = (int)(i * dataset_dims[1]); + } + + VRFY((MPI_SUCCESS == MPI_Allgatherv(data, recvcounts[mpi_rank], C_DATATYPE_MPI, fill_buf, recvcounts, + displs, C_DATATYPE_MPI, comm)), + "MPI_Allgatherv succeeded"); + + /* + * It should be very unlikely for the dataset's random + * values to all be the fill value, so this should be + * a safe comparison in theory. + */ + VRFY((0 != HDmemcmp(read_buf, fill_buf, read_buf_size)), "Data verification succeeded"); + + /* + * Write to whole dataset and ensure fill value isn't returned + * after reading whole dataset back + */ + + /* Each process defines the dataset selection in memory and writes + * it to the hyperslab in the file + */ + count[0] = (hsize_t)FILL_TIME_NEVER_TEST_NROWS / (hsize_t)FILL_TIME_NEVER_TEST_CH_NROWS; + count[1] = (hsize_t)FILL_TIME_NEVER_TEST_NCOLS / (hsize_t)FILL_TIME_NEVER_TEST_CH_NCOLS; + stride[0] = (hsize_t)FILL_TIME_NEVER_TEST_CH_NROWS; + stride[1] = (hsize_t)FILL_TIME_NEVER_TEST_CH_NCOLS; + block[0] = (hsize_t)FILL_TIME_NEVER_TEST_CH_NROWS / (hsize_t)mpi_size; + block[1] = (hsize_t)FILL_TIME_NEVER_TEST_CH_NCOLS; + start[0] = (hsize_t)mpi_rank * block[0]; + start[1] = 0; + + if (VERBOSE_MED) { + HDprintf("Process %d is writing with count[ %" PRIuHSIZE ", %" PRIuHSIZE " ], stride[ %" PRIuHSIZE + ", %" PRIuHSIZE " ], start[ %" PRIuHSIZE ", %" PRIuHSIZE " ], block size[ %" PRIuHSIZE + ", %" PRIuHSIZE " ]\n", + mpi_rank, count[0], count[1], stride[0], stride[1], start[0], start[1], block[0], block[1]); + HDfflush(stdout); } + VRFY((H5Sselect_hyperslab(filespace, H5S_SELECT_SET, start, stride, count, block) >= 0), + "Hyperslab selection succeeded"); + + VRFY((H5Dwrite(dset_id, HDF5_DATATYPE_NAME, H5S_BLOCK, filespace, dxpl_id, data) >= 0), + "Dataset write succeeded"); + + /* Verify space allocation status */ + verify_space_alloc_status(dset_id, plist_id, ALL_CHUNKS_WRITTEN); + + VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); + + /* Verify correct data was written */ + dset_id = H5Dopen2(group_id, FILL_TIME_NEVER_TEST_DATASET_NAME, H5P_DEFAULT); + VRFY((dset_id >= 0), "Dataset open succeeded"); + + VRFY((H5Dread(dset_id, HDF5_DATATYPE_NAME, H5S_ALL, H5S_ALL, dxpl_id, read_buf) >= 0), + "Dataset read succeeded"); + + for (i = 0; i < read_buf_size / sizeof(*read_buf); i++) + VRFY((read_buf[i] != FILL_TIME_NEVER_TEST_FILL_VAL), "Data verification succeeded"); + + if (displs) + HDfree(displs); + if (recvcounts) + HDfree(recvcounts); if (data) HDfree(data); + if (read_buf) + HDfree(read_buf); + if (fill_buf) + HDfree(fill_buf); + VRFY((H5Pclose(plist_id) >= 0), "DCPL close succeeded"); VRFY((H5Dclose(dset_id) >= 0), "Dataset close succeeded"); VRFY((H5Sclose(filespace) >= 0), "File dataspace close succeeded"); - VRFY((H5Sclose(memspace) >= 0), "Memory dataspace close succeeded"); - VRFY((H5Pclose(plist_id) >= 0), "DXPL close succeeded"); + VRFY((H5Gclose(group_id) >= 0), "Group close succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); return; @@ -6459,8 +8650,14 @@ test_shrinking_growing_chunks(void) int main(int argc, char **argv) { - size_t i; - hid_t file_id = -1, fapl = -1; + size_t cur_filter_idx = 0; + size_t num_filters = 0; + hid_t file_id = H5I_INVALID_HID; + hid_t fcpl_id = H5I_INVALID_HID; + hid_t group_id = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + hid_t dxpl_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; int mpi_code; /* Initialize MPI */ @@ -6487,7 +8684,7 @@ main(int argc, char **argv) if (MAINPROCESS) { HDprintf("==========================\n"); - HDprintf("Parallel Filters tests\n"); + HDprintf(" Parallel Filters tests\n"); HDprintf("==========================\n\n"); } @@ -6496,72 +8693,161 @@ main(int argc, char **argv) TestAlarmOn(); - /* Create test file */ - fapl = H5Pcreate(H5P_FILE_ACCESS); - VRFY((fapl >= 0), "FAPL creation succeeded"); - - VRFY((H5Pset_fapl_mpio(fapl, comm, info) >= 0), "Set FAPL MPIO succeeded"); - - VRFY((H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); - - VRFY((h5_fixname(FILENAME[0], fapl, filenames[0], sizeof(filenames[0])) != NULL), - "Test file name created"); + num_filters = ARRAY_SIZE(filterIDs); - file_id = H5Fcreate(filenames[0], H5F_ACC_TRUNC, H5P_DEFAULT, fapl); - VRFY((file_id >= 0), "Test file creation succeeded"); + /* Set up file access property list with parallel I/O access, + * collective metadata reads/writes and the latest library + * version bounds */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((fapl_id >= 0), "FAPL creation succeeded"); - VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + VRFY((H5Pset_fapl_mpio(fapl_id, comm, info) >= 0), "Set FAPL MPIO succeeded"); + VRFY((H5Pset_all_coll_metadata_ops(fapl_id, TRUE) >= 0), "H5Pset_all_coll_metadata_ops succeeded"); + VRFY((H5Pset_coll_metadata_write(fapl_id, TRUE) >= 0), "H5Pset_coll_metadata_write succeeded"); - for (i = 0; i < ARRAY_SIZE(tests); i++) { - if (MPI_SUCCESS == (mpi_code = MPI_Barrier(comm))) { - (*tests[i])(); - } - else { - if (MAINPROCESS) - MESG("MPI_Barrier failed"); - nerrors++; - } - } + VRFY((H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), + "Set libver bounds succeeded"); /* - * Increment the filter index to switch to the checksum filter - * and re-run the tests. + * Set up Paged and Persistent Free Space Management */ - cur_filter_idx++; - - h5_clean_files(FILENAME, fapl); - - fapl = H5Pcreate(H5P_FILE_ACCESS); - VRFY((fapl >= 0), "FAPL creation succeeded"); + fcpl_id = H5Pcreate(H5P_FILE_CREATE); + VRFY((fcpl_id >= 0), "FCPL creation succeeded"); - VRFY((H5Pset_fapl_mpio(fapl, comm, info) >= 0), "Set FAPL MPIO succeeded"); + VRFY((H5Pset_file_space_strategy(fcpl_id, H5F_FSPACE_STRATEGY_PAGE, TRUE, 1) >= 0), + "H5Pset_file_space_strategy succeeded"); - VRFY((H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) >= 0), - "Set libver bounds succeeded"); + VRFY((h5_fixname(FILENAME[0], fapl_id, filenames[0], sizeof(filenames[0])) != NULL), + "Test file name created"); - file_id = H5Fcreate(filenames[0], H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + file_id = H5Fcreate(filenames[0], H5F_ACC_TRUNC, fcpl_id, fapl_id); VRFY((file_id >= 0), "Test file creation succeeded"); VRFY((H5Fclose(file_id) >= 0), "File close succeeded"); + file_id = H5I_INVALID_HID; - if (MAINPROCESS) { - HDprintf("\n=================================================================\n"); - HDprintf("Re-running Parallel Filters tests with Fletcher32 checksum filter\n"); - HDprintf("=================================================================\n\n"); - } - - for (i = 0; i < ARRAY_SIZE(tests); i++) { - if (MPI_SUCCESS == (mpi_code = MPI_Barrier(comm))) { - (*tests[i])(); - } - else { - if (MAINPROCESS) - MESG("MPI_Barrier failed"); - nerrors++; + /* Create property list for collective dataset write */ + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_id >= 0), "DXPL creation succeeded"); + + VRFY((H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE) >= 0), "H5Pset_dxpl_mpio succeeded"); + + /* Create DCPL for dataset creation */ + dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + VRFY((dcpl_id >= 0), "DCPL creation succeeded"); + + /* Run tests with all available filters */ + for (cur_filter_idx = 0; cur_filter_idx < num_filters; cur_filter_idx++) { + H5FD_mpio_chunk_opt_t chunk_opt; + H5Z_filter_t cur_filter = filterIDs[cur_filter_idx]; + + /* Run tests with both linked-chunk and multi-chunk I/O */ + for (chunk_opt = H5FD_MPIO_CHUNK_ONE_IO; chunk_opt <= H5FD_MPIO_CHUNK_MULTI_IO; chunk_opt++) { + H5D_alloc_time_t space_alloc_time; + + /* Run tests with all available space allocation times */ + for (space_alloc_time = H5D_ALLOC_TIME_EARLY; space_alloc_time <= H5D_ALLOC_TIME_INCR; + space_alloc_time++) { + const char *alloc_time; + unsigned filter_config; + htri_t filter_avail; + size_t i; + char group_name[512]; + + switch (space_alloc_time) { + case H5D_ALLOC_TIME_EARLY: + alloc_time = "Early"; + break; + case H5D_ALLOC_TIME_LATE: + alloc_time = "Late"; + break; + case H5D_ALLOC_TIME_INCR: + alloc_time = "Incremental"; + break; + default: + alloc_time = "Unknown"; + } + + if (MAINPROCESS) + HDprintf("== Running tests with filter '%s' using '%s' and '%s' allocation time ==\n\n", + filterNames[cur_filter_idx], + H5FD_MPIO_CHUNK_ONE_IO == chunk_opt ? "Linked-Chunk I/O" : "Multi-Chunk I/O", + alloc_time); + + /* Make sure current filter is available before testing with it */ + filter_avail = H5Zfilter_avail(cur_filter); + VRFY((filter_avail >= 0), "H5Zfilter_avail succeeded"); + + if (!filter_avail) { + if (MAINPROCESS) + HDprintf(" ** SKIPPED tests with filter '%s' - filter unavailable **\n\n", + filterNames[cur_filter_idx]); + continue; + } + + /* Get the current filter's info */ + VRFY((H5Zget_filter_info(cur_filter, &filter_config) >= 0), "H5Zget_filter_info succeeded"); + + /* Determine if filter is encode-enabled */ + if (0 == (filter_config & H5Z_FILTER_CONFIG_ENCODE_ENABLED)) { + if (MAINPROCESS) + HDprintf(" ** SKIPPED tests with filter '%s' - filter not encode-enabled **\n\n", + filterNames[cur_filter_idx]); + continue; + } + + /* Set space allocation time */ + VRFY((H5Pset_alloc_time(dcpl_id, space_alloc_time) >= 0), "H5Pset_alloc_time succeeded"); + + /* Set chunk I/O optimization method */ + VRFY((H5Pset_dxpl_mpio_chunk_opt(dxpl_id, chunk_opt) >= 0), + "H5Pset_dxpl_mpio_chunk_opt succeeded"); + + /* Create a group to hold all the datasets for this combination + * of filter and chunk optimization mode. Then, close the file + * again since some tests may need to open the file in a special + * way, like on rank 0 only */ + file_id = H5Fopen(filenames[0], H5F_ACC_RDWR, fapl_id); + VRFY((file_id >= 0), "H5Fopen succeeded"); + + HDsnprintf(group_name, sizeof(group_name), "%s_%s_%s", filterNames[cur_filter_idx], + H5FD_MPIO_CHUNK_ONE_IO == chunk_opt ? "linked-chunk-io" : "multi-chunk-io", + alloc_time); + + group_id = H5Gcreate2(file_id, group_name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((group_id >= 0), "H5Gcreate2 succeeded"); + + VRFY((H5Gclose(group_id) >= 0), "H5Gclose failed"); + group_id = H5I_INVALID_HID; + + VRFY((H5Fclose(file_id) >= 0), "H5Fclose succeeded"); + file_id = H5I_INVALID_HID; + + for (i = 0; i < ARRAY_SIZE(tests); i++) { + test_func func = tests[i]; + + if (MPI_SUCCESS == (mpi_code = MPI_Barrier(comm))) { + func(group_name, cur_filter, fapl_id, dcpl_id, dxpl_id); + } + else { + if (MAINPROCESS) + MESG("MPI_Barrier failed"); + nerrors++; + } + } + + if (MAINPROCESS) + HDputs(""); + } } } + VRFY((H5Pclose(dcpl_id) >= 0), "DCPL close succeeded"); + dcpl_id = H5I_INVALID_HID; + + VRFY((H5Pclose(dxpl_id) >= 0), "DXPL close succeeded"); + dxpl_id = H5I_INVALID_HID; + if (nerrors) goto exit; @@ -6575,7 +8861,21 @@ exit: TestAlarmOff(); - h5_clean_files(FILENAME, fapl); + h5_clean_files(FILENAME, fapl_id); + fapl_id = H5I_INVALID_HID; + + if (dcpl_id >= 0) + VRFY((H5Pclose(dcpl_id) >= 0), "H5Pclose succeeded"); + if (dxpl_id >= 0) + VRFY((H5Pclose(dxpl_id) >= 0), "H5Pclose succeeded"); + if (fapl_id >= 0) + VRFY((H5Pclose(fapl_id) >= 0), "H5Pclose succeeded"); + if (fcpl_id >= 0) + VRFY((H5Pclose(fcpl_id) >= 0), "H5Pclose succeeded"); + if (group_id >= 0) + VRFY((H5Gclose(group_id) >= 0), "H5Gclose succeeded"); + if (file_id >= 0) + VRFY((H5Fclose(file_id) >= 0), "H5Fclose succeeded"); H5close(); diff --git a/testpar/t_filters_parallel.h b/testpar/t_filters_parallel.h index 7eb34ed..800604c 100644 --- a/testpar/t_filters_parallel.h +++ b/testpar/t_filters_parallel.h @@ -30,23 +30,23 @@ #include "stdlib.h" #include "testpar.h" +#define ARRAY_SIZE(a) sizeof(a) / sizeof(a[0]) + /* Used to load other filters than GZIP */ /* #define DYNAMIC_FILTER */ /* Uncomment and define the fields below to use a dynamically loaded filter */ + +#ifdef DYNAMIC_FILTER #define FILTER_NUM_CDVALUES 1 const unsigned int cd_values[FILTER_NUM_CDVALUES] = {0}; -H5Z_filter_t filter_id; -unsigned int flags = 0; -size_t cd_nelmts = FILTER_NUM_CDVALUES; - -/* Utility Macros */ -#define STRINGIFY(type) #type +unsigned int flags = 0; +size_t cd_nelmts = FILTER_NUM_CDVALUES; +#endif /* Common defines for all tests */ -#define C_DATATYPE long -#define C_DATATYPE_MPI MPI_LONG -#define COMPOUND_C_DATATYPE cmpd_filtered_t -#define C_DATATYPE_STR(type) STRINGIFY(type) -#define HDF5_DATATYPE_NAME H5T_NATIVE_LONG +#define C_DATATYPE long +#define C_DATATYPE_MPI MPI_LONG +#define COMPOUND_C_DATATYPE cmpd_filtered_t +#define HDF5_DATATYPE_NAME H5T_NATIVE_LONG /* Macro used to generate data for datasets for later verification */ #define GEN_DATA(i) INCREMENTAL_DATA(i) @@ -59,7 +59,7 @@ size_t cd_nelmts = FILTER_NUM_CDVALUES; #define RANK_DATA(i) \ (mpi_rank) /* Generates test data to visibly show which rank wrote to which parts of the dataset */ -#define DEFAULT_DEFLATE_LEVEL 6 +#define DEFAULT_DEFLATE_LEVEL 9 #define DIM0_SCALE_FACTOR 4 #define DIM1_SCALE_FACTOR 2 @@ -89,6 +89,14 @@ typedef struct { #define WRITE_UNSHARED_FILTERED_CHUNKS_CH_NROWS (WRITE_UNSHARED_FILTERED_CHUNKS_NROWS / mpi_size) #define WRITE_UNSHARED_FILTERED_CHUNKS_CH_NCOLS (WRITE_UNSHARED_FILTERED_CHUNKS_NCOLS / mpi_size) +/* Defines for the unshared filtered chunks partial write test */ +#define WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_NAME "unshared_filtered_chunks_partial_write" +#define WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_DATASET_DIMS 2 +#define WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_NROWS (mpi_size * DIM0_SCALE_FACTOR) +#define WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_NCOLS (mpi_size * DIM1_SCALE_FACTOR) +#define WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NROWS (DIM0_SCALE_FACTOR) +#define WRITE_UNSHARED_FILTERED_CHUNKS_PARTIAL_CH_NCOLS (DIM1_SCALE_FACTOR) + /* Defines for the shared filtered chunks write test */ #define WRITE_SHARED_FILTERED_CHUNKS_DATASET_NAME "shared_filtered_chunks_write" #define WRITE_SHARED_FILTERED_CHUNKS_DATASET_DIMS 2 @@ -97,6 +105,42 @@ typedef struct { #define WRITE_SHARED_FILTERED_CHUNKS_NROWS (WRITE_SHARED_FILTERED_CHUNKS_CH_NROWS * DIM0_SCALE_FACTOR) #define WRITE_SHARED_FILTERED_CHUNKS_NCOLS (WRITE_SHARED_FILTERED_CHUNKS_CH_NCOLS * DIM1_SCALE_FACTOR) +/* Defines for the unshared filtered chunks w/ single unlim. dimension write test */ +#define WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_NAME "unshared_filtered_chunks_single_unlim_dim_write" +#define WRITE_UNSHARED_ONE_UNLIM_DIM_DATASET_DIMS 2 +#define WRITE_UNSHARED_ONE_UNLIM_DIM_NROWS (mpi_size * DIM0_SCALE_FACTOR) +#define WRITE_UNSHARED_ONE_UNLIM_DIM_NCOLS (mpi_size * DIM1_SCALE_FACTOR) +#define WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NROWS (WRITE_UNSHARED_ONE_UNLIM_DIM_NROWS / mpi_size) +#define WRITE_UNSHARED_ONE_UNLIM_DIM_CH_NCOLS (WRITE_UNSHARED_ONE_UNLIM_DIM_NCOLS / mpi_size) +#define WRITE_UNSHARED_ONE_UNLIM_DIM_NLOOPS 5 + +/* Defines for the shared filtered chunks w/ single unlim. dimension write test */ +#define WRITE_SHARED_ONE_UNLIM_DIM_DATASET_NAME "shared_filtered_chunks_single_unlim_dim_write" +#define WRITE_SHARED_ONE_UNLIM_DIM_DATASET_DIMS 2 +#define WRITE_SHARED_ONE_UNLIM_DIM_CH_NROWS (mpi_size) +#define WRITE_SHARED_ONE_UNLIM_DIM_CH_NCOLS (mpi_size) +#define WRITE_SHARED_ONE_UNLIM_DIM_NROWS (WRITE_SHARED_ONE_UNLIM_DIM_CH_NROWS * DIM0_SCALE_FACTOR) +#define WRITE_SHARED_ONE_UNLIM_DIM_NCOLS (WRITE_SHARED_ONE_UNLIM_DIM_CH_NCOLS * DIM1_SCALE_FACTOR) +#define WRITE_SHARED_ONE_UNLIM_DIM_NLOOPS 5 + +/* Defines for the unshared filtered chunks w/ two unlim. dimension write test */ +#define WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_NAME "unshared_filtered_chunks_two_unlim_dim_write" +#define WRITE_UNSHARED_TWO_UNLIM_DIM_DATASET_DIMS 2 +#define WRITE_UNSHARED_TWO_UNLIM_DIM_NROWS (mpi_size * DIM0_SCALE_FACTOR) +#define WRITE_UNSHARED_TWO_UNLIM_DIM_NCOLS (DIM1_SCALE_FACTOR) +#define WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NROWS (DIM0_SCALE_FACTOR) +#define WRITE_UNSHARED_TWO_UNLIM_DIM_CH_NCOLS (DIM1_SCALE_FACTOR) +#define WRITE_UNSHARED_TWO_UNLIM_DIM_NLOOPS 5 + +/* Defines for the shared filtered chunks w/ two unlim. dimension write test */ +#define WRITE_SHARED_TWO_UNLIM_DIM_DATASET_NAME "shared_filtered_chunks_two_unlim_dim_write" +#define WRITE_SHARED_TWO_UNLIM_DIM_DATASET_DIMS 2 +#define WRITE_SHARED_TWO_UNLIM_DIM_CH_NROWS (mpi_size) +#define WRITE_SHARED_TWO_UNLIM_DIM_CH_NCOLS (mpi_size) +#define WRITE_SHARED_TWO_UNLIM_DIM_NROWS (mpi_size) +#define WRITE_SHARED_TWO_UNLIM_DIM_NCOLS (mpi_size) +#define WRITE_SHARED_TWO_UNLIM_DIM_NLOOPS 5 + /* Defines for the filtered chunks write test where a process has no selection */ #define WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_NAME "single_no_selection_filtered_chunks_write" #define WRITE_SINGLE_NO_SELECTION_FILTERED_CHUNKS_DATASET_DIMS 2 @@ -403,4 +447,53 @@ typedef struct { #define SHRINKING_GROWING_CHUNKS_CH_NCOLS (SHRINKING_GROWING_CHUNKS_NCOLS / mpi_size) #define SHRINKING_GROWING_CHUNKS_NLOOPS 20 +/* Defines for the unshared filtered edge chunks write test */ +#define WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME "unshared_filtered_edge_chunks_write" +#define WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME2 "unshared_filtered_edge_chunks_no_filter_write" +#define WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS 2 +#define WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NROWS (DIM0_SCALE_FACTOR) +#define WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS (DIM1_SCALE_FACTOR) +#define WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_NROWS (mpi_size * DIM0_SCALE_FACTOR) +#define WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_NCOLS \ + (mpi_size * DIM1_SCALE_FACTOR) + (WRITE_UNSHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS - 1) + +/* Defines for the shared filtered edge chunks write test */ +#define WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME "shared_filtered_edge_chunks_write" +#define WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_NAME2 "shared_filtered_edge_chunks_no_filter_write" +#define WRITE_SHARED_FILTERED_EDGE_CHUNKS_DATASET_DIMS 2 +#define WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NROWS (mpi_size) +#define WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS (mpi_size) +#define WRITE_SHARED_FILTERED_EDGE_CHUNKS_NROWS \ + (WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NROWS * DIM0_SCALE_FACTOR) +#define WRITE_SHARED_FILTERED_EDGE_CHUNKS_NCOLS \ + ((WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS * DIM1_SCALE_FACTOR) + \ + (WRITE_SHARED_FILTERED_EDGE_CHUNKS_CH_NCOLS - 1)) + +/* Defines for the fill values test */ +#define FILL_VALUES_TEST_DATASET_NAME "fill_value_test" +#define FILL_VALUES_TEST_DATASET_NAME2 "fill_value_alloc_test" +#define FILL_VALUES_TEST_DATASET_DIMS 2 +#define FILL_VALUES_TEST_FILL_VAL (-1) +#define FILL_VALUES_TEST_CH_NROWS (mpi_size) +#define FILL_VALUES_TEST_CH_NCOLS (mpi_size + 1) +#define FILL_VALUES_TEST_NROWS (FILL_VALUES_TEST_CH_NROWS * DIM0_SCALE_FACTOR) +#define FILL_VALUES_TEST_NCOLS (FILL_VALUES_TEST_CH_NCOLS * DIM1_SCALE_FACTOR) + +/* Defines for the undefined fill value test */ +#define FILL_VALUE_UNDEFINED_TEST_DATASET_NAME "fill_value_undefined_test" +#define FILL_VALUE_UNDEFINED_TEST_DATASET_DIMS 2 +#define FILL_VALUE_UNDEFINED_TEST_CH_NROWS (mpi_size) +#define FILL_VALUE_UNDEFINED_TEST_CH_NCOLS (mpi_size + 1) +#define FILL_VALUE_UNDEFINED_TEST_NROWS (FILL_VALUE_UNDEFINED_TEST_CH_NROWS * DIM0_SCALE_FACTOR) +#define FILL_VALUE_UNDEFINED_TEST_NCOLS (FILL_VALUE_UNDEFINED_TEST_CH_NCOLS * DIM1_SCALE_FACTOR) + +/* Defines for the fill time of 'never' test */ +#define FILL_TIME_NEVER_TEST_DATASET_NAME "fill_time_never_test" +#define FILL_TIME_NEVER_TEST_DATASET_DIMS 2 +#define FILL_TIME_NEVER_TEST_FILL_VAL (-1) +#define FILL_TIME_NEVER_TEST_CH_NROWS (mpi_size) +#define FILL_TIME_NEVER_TEST_CH_NCOLS (mpi_size + 1) +#define FILL_TIME_NEVER_TEST_NROWS (FILL_TIME_NEVER_TEST_CH_NROWS * DIM0_SCALE_FACTOR) +#define FILL_TIME_NEVER_TEST_NCOLS (FILL_TIME_NEVER_TEST_CH_NCOLS * DIM1_SCALE_FACTOR) + #endif /* TEST_PARALLEL_FILTERS_H_ */ diff --git a/testpar/testphdf5.h b/testpar/testphdf5.h index c692287..16f45d3 100644 --- a/testpar/testphdf5.h +++ b/testpar/testphdf5.h @@ -186,10 +186,6 @@ enum H5TEST_COLL_CHUNK_API { #define TEST_NOT_SIMPLE_OR_SCALAR_DATASPACES 0x010 #define TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_COMPACT 0x020 #define TEST_NOT_CONTIGUOUS_OR_CHUNKED_DATASET_EXTERNAL 0x040 -#define TEST_FILTERS 0x080 -/* TEST_FILTERS will take place of this after supporting mpio + filter for - * H5Dcreate and H5Dwrite */ -#define TEST_FILTERS_READ 0x100 /* Don't erase these lines, they are put here for debugging purposes */ /* -- cgit v0.12 From 2db4c770f83a35d1ec013114c483c5120b509fdc Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Thu, 24 Feb 2022 10:59:15 -0600 Subject: Fix release check version logic (#1438) --- src/H5.c | 19 +++++++++---------- test/CMakeTests.cmake | 3 ++- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/H5.c b/src/H5.c index d4fca9a..4c449d0 100644 --- a/src/H5.c +++ b/src/H5.c @@ -70,9 +70,9 @@ static int H5__mpi_delete_cb(MPI_Comm comm, int keyval, void *attr_val, int *fla /* Library Private Variables */ /*****************************/ -/* Library incompatible release versions */ -const unsigned VERS_RELEASE_EXCEPTIONS[] = {0}; -const unsigned VERS_RELEASE_EXCEPTIONS_SIZE = 0; +/* Library incompatible release versions, develop releases are incompatible by design */ +const unsigned VERS_RELEASE_EXCEPTIONS[] = {0, 1}; +const unsigned VERS_RELEASE_EXCEPTIONS_SIZE = 2; /* statically initialize block for pthread_once call used in initializing */ /* the first global mutex */ @@ -954,6 +954,7 @@ H5check_version(unsigned majnum, unsigned minnum, unsigned relnum) static int checked = 0; /* If we've already checked the version info */ static unsigned int disable_version_check = 0; /* Set if the version check should be disabled */ static const char * version_mismatch_warning = VERSION_MISMATCH_WARNING; + static const char * release_mismatch_warning = RELEASE_MISMATCH_WARNING; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API_NOINIT_NOERR_NOFS @@ -974,10 +975,7 @@ H5check_version(unsigned majnum, unsigned minnum, unsigned relnum) } /* H5_VERS_MAJOR and H5_VERS_MINOR must match */ - /* Cast relnum to int to avoid warning for unsigned < 0 comparison - * in first release versions */ - if (H5_VERS_MAJOR != majnum || H5_VERS_MINOR != minnum || H5_VERS_RELEASE > (int)relnum) { - + if (H5_VERS_MAJOR != majnum || H5_VERS_MINOR != minnum) { switch (disable_version_check) { case 0: HDfprintf(stderr, "%s%s", version_mismatch_warning, @@ -1012,9 +1010,10 @@ H5check_version(unsigned majnum, unsigned minnum, unsigned relnum) break; } /* end switch */ - } /* end if (H5_VERS_MAJOR != majnum || H5_VERS_MINOR != minnum || H5_VERS_RELEASE > relnum) */ + } /* end if (H5_VERS_MAJOR != majnum || H5_VERS_MINOR != minnum) */ /* H5_VERS_RELEASE should be compatible, we will only add checks for exceptions */ + /* Library develop release versions are incompatible by design */ if (H5_VERS_RELEASE != relnum) { for (unsigned i = 0; i < VERS_RELEASE_EXCEPTIONS_SIZE; i++) { /* Check for incompatible headers or incompatible library */ @@ -1022,7 +1021,7 @@ H5check_version(unsigned majnum, unsigned minnum, unsigned relnum) switch (disable_version_check) { case 0: HDfprintf( - stderr, "%s%s", version_mismatch_warning, + stderr, "%s%s", release_mismatch_warning, "You can, at your own risk, disable this warning by setting the environment\n" "variable 'HDF5_DISABLE_VERSION_CHECK' to a value of '1'.\n" "Setting it to 2 or higher will suppress the warning messages totally.\n"); @@ -1041,7 +1040,7 @@ H5check_version(unsigned majnum, unsigned minnum, unsigned relnum) "%s'HDF5_DISABLE_VERSION_CHECK' " "environment variable is set to %d, application will\n" "continue at your own risk.\n", - version_mismatch_warning, disable_version_check); + release_mismatch_warning, disable_version_check); /* Mention the versions we are referring to */ HDfprintf(stderr, "Headers are %u.%u.%u, library is %u.%u.%u\n", majnum, minnum, relnum, (unsigned)H5_VERS_MAJOR, (unsigned)H5_VERS_MINOR, diff --git a/test/CMakeTests.cmake b/test/CMakeTests.cmake index b2ed8e8..4e828f8 100644 --- a/test/CMakeTests.cmake +++ b/test/CMakeTests.cmake @@ -649,10 +649,11 @@ set_tests_properties (H5TEST-tcheck_version-minor PROPERTIES WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST WILL_FAIL "true" ) -# release + 1 should pass +# release + 1 should pass on non-develop branches add_test (NAME H5TEST-tcheck_version-release COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ "-tr") set_tests_properties (H5TEST-tcheck_version-release PROPERTIES WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST + WILL_FAIL "true" ) ############################################################################## -- cgit v0.12 From e19e7e3faae303a4a63b7df002abe1fe31348290 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Fri, 25 Feb 2022 14:31:55 -0600 Subject: Add release note for #1438 (#1457) * Add release note for #1438 * fix typos. --- config/cmake/scripts/HDF5config.cmake | 2 +- release_docs/RELEASE.txt | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/config/cmake/scripts/HDF5config.cmake b/config/cmake/scripts/HDF5config.cmake index 2f5af77..1448e81 100644 --- a/config/cmake/scripts/HDF5config.cmake +++ b/config/cmake/scripts/HDF5config.cmake @@ -68,7 +68,7 @@ endif () # build generator must be defined if (NOT DEFINED BUILD_GENERATOR) - message (FATAL_ERROR "BUILD_GENERATOR must be defined - Unix, VS2019, VS201964, VS2017, or VS201764, VS2015, VS201564") + message (FATAL_ERROR "BUILD_GENERATOR must be defined - Unix, VS2019, VS201964, VS2017, VS201764, VS2015, VS201564") endif () ################################################################### diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index a165433..ab4ec2a 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -1129,6 +1129,23 @@ Bug Fixes since HDF5-1.12.0 release =================================== Library ------- + - Issue #1436 identified a problem with the H5_VERS_RELEASE check in the + H5check_version function. + + Investigating the original fix, #812, we discovered some inconsistencies + with a new block added to check H5_VERS_RELEASE for incompatibilities. + This new block was not using the new warning text dealing with the + H5_VERS_RELEASE check and would cause the warning to be duplicated. + + By removing the H5_VERS_RELEASE argument in the first check for + H5_VERS_MAJOR and H5_VERS_MINOR, the second check would only check + the H5_VERS_RELEASE for incompatible release versions. This adheres + to the statement that except for the develop branch, all release versions + in a major.minor maintenance branch should be compatible. The prerequisite + is that an application will not use any APIs not present in all release versions. + + (ADB - 2022/02/24, #1438) + - Fixed an H5Pget_filter_by_id1/2() assert w/ out of range filter IDs Both H5Pget_filter_by_id1 and 2 did not range check the filter ID, which -- cgit v0.12 From f46e5db184b22a2a53cf901f23c7561e596f39ca Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Fri, 25 Feb 2022 14:36:34 -0600 Subject: Amend RELEASE.txt with note about parallel compression testing assertion failure (#1458) --- release_docs/RELEASE.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index ab4ec2a..2346e2f 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -922,6 +922,15 @@ New Features * Improved coverage of regression testing for the feature + NOTE: we are aware of and working on a fix for an assertion failure that + occurs during regression testing of the feature. It looks similar to: + + hdf5/src/H5ACmpio.c:1089: H5AC__log_moved_entry: Assertion `!entry_dirty' failed. + + so far, this assertion failure has only occurred during testing and has + not been seen in any production uses, but until this bug is fixed the + feature should be considered unstable. + (JTH - 2022/2/23) Fortran Library: -- cgit v0.12 From 80954e82723086e573af37b31e85d0f3d562e9b3 Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Sun, 27 Feb 2022 23:15:38 -0600 Subject: Fix for parallel compression examples on Windows (#1459) --- examples/ph5_filtered_writes.c | 30 +++++++++++++++++++----------- examples/ph5_filtered_writes_no_sel.c | 4 ++++ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/examples/ph5_filtered_writes.c b/examples/ph5_filtered_writes.c index 8b55528..0f399dd 100644 --- a/examples/ph5_filtered_writes.c +++ b/examples/ph5_filtered_writes.c @@ -43,6 +43,10 @@ #define HDF5_DATATYPE H5T_NATIVE_INT typedef int C_DATATYPE; +#ifndef PATH_MAX +#define PATH_MAX 512 +#endif + /* Global variables */ int mpi_rank, mpi_size; @@ -234,16 +238,16 @@ write_dataset_no_overlap(hid_t file_id, hid_t dxpl_id) static void write_dataset_overlap(hid_t file_id, hid_t dxpl_id) { - C_DATATYPE data[mpi_size][EXAMPLE_DSET_CHUNK_DIM_SIZE]; - hsize_t dataset_dims[EXAMPLE_DSET_DIMS]; - hsize_t chunk_dims[EXAMPLE_DSET_DIMS]; - hsize_t start[EXAMPLE_DSET_DIMS]; - hsize_t stride[EXAMPLE_DSET_DIMS]; - hsize_t count[EXAMPLE_DSET_DIMS]; - size_t i, j; - hid_t dset_id = H5I_INVALID_HID; - hid_t dcpl_id = H5I_INVALID_HID; - hid_t file_dataspace = H5I_INVALID_HID; + C_DATATYPE *data = NULL; + hsize_t dataset_dims[EXAMPLE_DSET_DIMS]; + hsize_t chunk_dims[EXAMPLE_DSET_DIMS]; + hsize_t start[EXAMPLE_DSET_DIMS]; + hsize_t stride[EXAMPLE_DSET_DIMS]; + hsize_t count[EXAMPLE_DSET_DIMS]; + size_t i, j; + hid_t dset_id = H5I_INVALID_HID; + hid_t dcpl_id = H5I_INVALID_HID; + hid_t file_dataspace = H5I_INVALID_HID; /* * ------------------------------------ @@ -324,7 +328,9 @@ write_dataset_overlap(hid_t file_id, hid_t dxpl_id) * -------------------------------------- */ - fill_databuf(start, count, stride, &data[0][0]); + data = malloc(mpi_size * EXAMPLE_DSET_CHUNK_DIM_SIZE * sizeof(C_DATATYPE)); + + fill_databuf(start, count, stride, data); /* * --------------------------------- @@ -334,6 +340,8 @@ write_dataset_overlap(hid_t file_id, hid_t dxpl_id) H5Dwrite(dset_id, HDF5_DATATYPE, H5S_BLOCK, file_dataspace, dxpl_id, data); + free(data); + /* * -------------- * Close HDF5 IDs diff --git a/examples/ph5_filtered_writes_no_sel.c b/examples/ph5_filtered_writes_no_sel.c index 14c68c8..2e29838 100644 --- a/examples/ph5_filtered_writes_no_sel.c +++ b/examples/ph5_filtered_writes_no_sel.c @@ -43,6 +43,10 @@ #define HDF5_DATATYPE H5T_NATIVE_INT typedef int C_DATATYPE; +#ifndef PATH_MAX +#define PATH_MAX 512 +#endif + /* Global variables */ int mpi_rank, mpi_size; -- cgit v0.12 From 8e0b427bf2a22871318e1c3ca76717344ec0019f Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Tue, 1 Mar 2022 12:30:49 -0600 Subject: Fix metadata cache bug when resizing a pinned/protected entry (v2) (#1463) --- release_docs/RELEASE.txt | 26 +++++++++++------ src/H5AC.c | 73 ++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 84 insertions(+), 15 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 2346e2f..1fd6f2d 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -922,15 +922,6 @@ New Features * Improved coverage of regression testing for the feature - NOTE: we are aware of and working on a fix for an assertion failure that - occurs during regression testing of the feature. It looks similar to: - - hdf5/src/H5ACmpio.c:1089: H5AC__log_moved_entry: Assertion `!entry_dirty' failed. - - so far, this assertion failure has only occurred during testing and has - not been seen in any production uses, but until this bug is fixed the - feature should be considered unstable. - (JTH - 2022/2/23) Fortran Library: @@ -1138,6 +1129,23 @@ Bug Fixes since HDF5-1.12.0 release =================================== Library ------- + - Fixed a metadata cache bug when resizing a pinned/protected cache entry + + When resizing a pinned/protected cache entry, the metadata + cache code previously would wait until after resizing the + entry to attempt to log the newly-dirtied entry. This would + cause H5C_resize_entry to mark the entry as dirty and make + H5AC_resize_entry think that it doesn't need to add the + newly-dirtied entry to the dirty entries skiplist. + + Thus, a subsequent H5AC__log_moved_entry would think it + needs to allocate a new entry for insertion into the dirty + entry skip list, since the entry doesn't exist on that list. + This causes an assertion failure, as the code to allocate a + new entry assumes that the entry is not dirty. + + (JRM - 2022/02/28) + - Issue #1436 identified a problem with the H5_VERS_RELEASE check in the H5check_version function. diff --git a/src/H5AC.c b/src/H5AC.c index 47d3a65..ac28a8c 100644 --- a/src/H5AC.c +++ b/src/H5AC.c @@ -1440,21 +1440,82 @@ H5AC_resize_entry(void *thing, size_t new_size) cache_ptr = entry_ptr->cache_ptr; HDassert(cache_ptr); - /* Resize the entry */ - if (H5C_resize_entry(thing, new_size) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTRESIZE, FAIL, "can't resize entry") - #ifdef H5_HAVE_PARALLEL - { + /* Log the generation of dirty bytes of metadata iff: + * + * 1) The entry is clean on entry, and this resize will dirty it + * (i.e. the current and new sizes are different), and + * + * 2) This is a parallel computation -- which it is if the aux_ptr + * is non-null. + * + * A few points to note about this section of the code: + * + * 1) This call must occur before the call to H5C_resize_entry() since + * H5AC__log_dirtied_entry() expects the target entry to be clean + * on entry. + * + * 2) This code has some basic issues in terms of the number of bytes + * added to the dirty bytes count. + * + * First, it adds the initial entry size to aux_ptr->dirty_bytes, + * not the final size. Note that this code used to use the final + * size, but code to support this has been removed from + * H5AC__log_dirtied_entry() for reasons unknown since I wrote this + * code. + * + * As long as all ranks do the same thing here, this probably doesn't + * matter much, although it will delay initiation of sync points. + * + * A more interesting point is that this code will not increment + * aux_ptr->dirty_bytes if a dirty entry is resized. At first glance + * this seems major, as particularly with the older file formats, + * resizes can be quite large. However, this is probably not an + * issue either, since such resizes will be accompanied by large + * amounts of dirty metadata creation in other areas -- which will + * cause aux_ptr->dirty_bytes to be incremented. + * + * The bottom line is that this code is probably OK, but the above + * points should be kept in mind. + * + * One final observation: This comment is occasioned by a bug caused + * by moving the call to H5AC__log_dirtied_entry() after the call to + * H5C_resize_entry(), and then only calling H5AC__log_dirtied_entry() + * if entry_ptr->is_dirty was false. + * + * Since H5C_resize_entry() marks the target entry dirty unless there + * is not change in size, this had the effect of not calling + * H5AC__log_dirtied_entry() when it should be, and corrupting + * the cleaned and dirtied lists used by rank 0 in the parallel + * version of the metadata cache. + * + * The point here is that you should be very careful when working with + * this code, and not modify it unless you fully understand it. + * + * JRM -- 2/28/22 + */ + + if ((!entry_ptr->is_dirty) && (entry_ptr->size != new_size)) { + + /* the entry is clean, and will be marked dirty in the resize + * operation. + */ H5AC_aux_t *aux_ptr; aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); - if ((!entry_ptr->is_dirty) && (NULL != aux_ptr)) + + if (NULL != aux_ptr) { + if (H5AC__log_dirtied_entry(entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "can't log dirtied entry") + } } #endif /* H5_HAVE_PARALLEL */ + /* Resize the entry */ + if (H5C_resize_entry(thing, new_size) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTRESIZE, FAIL, "can't resize entry") + done: /* If currently logging, generate a message */ if (cache_ptr != NULL && cache_ptr->log_info != NULL) -- cgit v0.12 From b7d8177404e0800eec7262ed051fff9b22ae1af8 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Tue, 1 Mar 2022 13:33:18 -0600 Subject: Remove targets PERMISSIONS setting (#1462) --- src/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 017b5d6..da99dc6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1329,7 +1329,6 @@ if (HDF5_EXPORTED_TARGETS) RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT libraries FRAMEWORK DESTINATION ${HDF5_INSTALL_FWRK_DIR} COMPONENT libraries INCLUDES DESTINATION include - PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE ) endif () -- cgit v0.12 From c72904c6b3ff04a48209c8e26dcce3007e2ddb40 Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Tue, 1 Mar 2022 14:19:15 -0600 Subject: fixed buf definition in h5dfill_f (#1465) * H5Lexists docs: Removed reference to 1.8.16 since the change is the 1.8.x releases, HDFFV-11289 * H5Oget_info_by_name, name can be any object, not just a group * fixed buf definition in h5dfill_f --- fortran/src/H5Dff.F90 | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/fortran/src/H5Dff.F90 b/fortran/src/H5Dff.F90 index 655cf1c..6b77a8c 100644 --- a/fortran/src/H5Dff.F90 +++ b/fortran/src/H5Dff.F90 @@ -1625,8 +1625,7 @@ CONTAINS ! Inputs: ! fill_value - fill value ! space_id - memory space selection identifier -! buf - data buffer iin memory ro apply selection to -! - of k-th dimension of the buf array +! buf - memory buffer containing the selection to be filled ! Outputs: ! hdferr: - error code ! Success: 0 @@ -1676,8 +1675,7 @@ CONTAINS ! Inputs: ! fill_value - fill value ! space_id - memory space selection identifier -! buf - data buffer iin memory ro apply selection to -! - of k-th dimension of the buf array +! buf - memory buffer containing the selection to be filled ! Outputs: ! hdferr: - error code ! Success: 0 @@ -1724,8 +1722,7 @@ CONTAINS ! Inputs: ! fill_value - fill value ! space_id - memory space selection identifier - ! buf - data buffer iin memory ro apply selection to - ! - of k-th dimension of the buf array + ! buf - memory buffer containing the selection to be filled ! Outputs: ! hdferr: - error code ! Success: 0 @@ -1799,8 +1796,7 @@ CONTAINS ! Inputs: ! fill_value - fill value ! space_id - memory space selection identifier -! buf - data buffer iin memory ro apply selection to -! - of k-th dimension of the buf array +! buf - memory buffer containing the selection to be filled ! Outputs: ! hdferr: - error code ! Success: 0 -- cgit v0.12 From ebf89bb231cd5b41180e58facae485f819cc3cea Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Tue, 1 Mar 2022 14:19:43 -0600 Subject: Fix install config and java test name conflict (#1467) --- config/cmake_ext_mod/HDFMacros.cmake | 2 +- java/test/TestH5OcopyOld.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/config/cmake_ext_mod/HDFMacros.cmake b/config/cmake_ext_mod/HDFMacros.cmake index 09492e7..0f9b367 100644 --- a/config/cmake_ext_mod/HDFMacros.cmake +++ b/config/cmake_ext_mod/HDFMacros.cmake @@ -356,7 +356,7 @@ macro (HDF_DIR_PATHS package_prefix) endif () endif () if (NOT ${package_prefix}_INSTALL_CMAKE_DIR) - set (${package_prefix}_INSTALL_CMAKE_DIR share/cmake) + set (${package_prefix}_INSTALL_CMAKE_DIR cmake) endif () # Always use full RPATH, i.e. don't skip the full RPATH for the build tree diff --git a/java/test/TestH5OcopyOld.java b/java/test/TestH5OcopyOld.java index b360cd0..e547051 100644 --- a/java/test/TestH5OcopyOld.java +++ b/java/test/TestH5OcopyOld.java @@ -32,7 +32,7 @@ import org.junit.rules.TestName; public class TestH5OcopyOld { @Rule public TestName testname = new TestName(); - private static final String FILENAME = "testRefsattribute.h5"; + private static final String FILENAME = "testRefsattributeO.h5"; private static final int DIM_X = 4; private static final int DIM_Y = 6; long H5fid = HDF5Constants.H5I_INVALID_HID; @@ -140,7 +140,6 @@ public class TestH5OcopyOld { byte[] dset_data = new byte[16]; long attribute_id = HDF5Constants.H5I_INVALID_HID; - try { rbuf0 = H5.H5Rcreate(H5fid, "/G1", HDF5Constants.H5R_OBJECT, -1); rbuf1 = H5.H5Rcreate(H5fid, "DS2", HDF5Constants.H5R_OBJECT, -1); -- cgit v0.12 From 94d01f537796effac71b04c11285a51dd6243327 Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Tue, 1 Mar 2022 14:20:34 -0600 Subject: Update version to 1.13.2-1 after 1.13.1 release; add new Makefile.in files to MANIFEST (#1460) --- MANIFEST | 4 ++++ README.md | 2 +- c++/src/cpp_doc_config | 2 +- config/cmake/scripts/HDF5config.cmake | 2 +- configure.ac | 2 +- java/src/hdf/hdf5lib/H5.java | 4 ++-- java/test/TestH5.java | 4 ++-- release_docs/RELEASE.txt | 2 +- src/H5.c | 4 ++-- src/H5public.h | 4 ++-- .../testfiles/h5repack_layout.h5-plugin_version_test.ddl | 14 +++++++------- 11 files changed, 24 insertions(+), 20 deletions(-) diff --git a/MANIFEST b/MANIFEST index 37b84d4..a7ab9c7 100644 --- a/MANIFEST +++ b/MANIFEST @@ -3891,3 +3891,7 @@ ./utils/Makefile.in ./utils/mirror_vfd/Makefile.in ./utils/test/Makefile.in +./utils/tools/Makefile.in +./utils/tools/h5dwalk/Makefile.in +./utils/tools/test/Makefile.in +./utils/tools/test/h5dwalk/Makefile.in diff --git a/README.md b/README.md index ce92342..7e0c281 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -HDF5 version 1.13.1-1 currently under development +HDF5 version 1.13.2-1 currently under development ![HDF5 Logo](doxygen/img/HDF5.png) diff --git a/c++/src/cpp_doc_config b/c++/src/cpp_doc_config index 3eb7645..1e26187 100644 --- a/c++/src/cpp_doc_config +++ b/c++/src/cpp_doc_config @@ -38,7 +38,7 @@ PROJECT_NAME = # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "1.13.1-1, currently under development" +PROJECT_NUMBER = "1.13.2-1, currently under development" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/config/cmake/scripts/HDF5config.cmake b/config/cmake/scripts/HDF5config.cmake index 1448e81..9a3ead2 100644 --- a/config/cmake/scripts/HDF5config.cmake +++ b/config/cmake/scripts/HDF5config.cmake @@ -37,7 +37,7 @@ cmake_minimum_required (VERSION 3.12) # CTEST_SOURCE_NAME - source folder ############################################################################## -set (CTEST_SOURCE_VERSION "1.13.1") +set (CTEST_SOURCE_VERSION "1.13.2") set (CTEST_SOURCE_VERSEXT "-1") ############################################################################## diff --git a/configure.ac b/configure.ac index 55468bf..0ad2228 100644 --- a/configure.ac +++ b/configure.ac @@ -23,7 +23,7 @@ AC_PREREQ([2.69]) ## NOTE: Do not forget to change the version number here when we do a ## release!!! ## -AC_INIT([HDF5], [1.13.1-1], [help@hdfgroup.org]) +AC_INIT([HDF5], [1.13.2-1], [help@hdfgroup.org]) AC_CONFIG_SRCDIR([src/H5.c]) AC_CONFIG_HEADERS([src/H5config.h]) diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index 49a539b..3180ede 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -216,7 +216,7 @@ import hdf.hdf5lib.structs.H5O_token_t; * exception handlers to print out the HDF-5 error stack. *
* - * @version HDF5 1.13.1
+ * @version HDF5 1.13.2
* See also: hdf.hdf5lib.HDFArray
* hdf.hdf5lib.HDF5Constants
* hdf.hdf5lib.HDF5CDataTypes
@@ -239,7 +239,7 @@ public class H5 implements java.io.Serializable { * * Make sure to update the versions number when a different library is used. */ - public final static int LIB_VERSION[] = { 1, 13, 1 }; + public final static int LIB_VERSION[] = { 1, 13, 2 }; /** * add system property to load library by path diff --git a/java/test/TestH5.java b/java/test/TestH5.java index 1f81f09..168f28a 100644 --- a/java/test/TestH5.java +++ b/java/test/TestH5.java @@ -287,7 +287,7 @@ public class TestH5 { */ @Test public void testH5get_libversion() { - int libversion[] = { 1, 13, 1 }; + int libversion[] = { 1, 13, 2 }; try { H5.H5get_libversion(libversion); @@ -326,7 +326,7 @@ public class TestH5 { */ @Test public void testH5check_version() { - int majnum = 1, minnum = 13, relnum = 1; + int majnum = 1, minnum = 13, relnum = 2; try { H5.H5check_version(majnum, minnum, relnum); diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 1fd6f2d..045d594 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -1,4 +1,4 @@ -HDF5 version 1.13.1-1 currently under development +HDF5 version 1.13.2-1 currently under development ================================================================================ diff --git a/src/H5.c b/src/H5.c index 4c449d0..1585dd3 100644 --- a/src/H5.c +++ b/src/H5.c @@ -71,8 +71,8 @@ static int H5__mpi_delete_cb(MPI_Comm comm, int keyval, void *attr_val, int *fla /*****************************/ /* Library incompatible release versions, develop releases are incompatible by design */ -const unsigned VERS_RELEASE_EXCEPTIONS[] = {0, 1}; -const unsigned VERS_RELEASE_EXCEPTIONS_SIZE = 2; +const unsigned VERS_RELEASE_EXCEPTIONS[] = {0, 1, 2}; +const unsigned VERS_RELEASE_EXCEPTIONS_SIZE = 3; /* statically initialize block for pthread_once call used in initializing */ /* the first global mutex */ diff --git a/src/H5public.h b/src/H5public.h index 037501b..3f9848a 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -83,7 +83,7 @@ /** * For tweaks, bug-fixes, or development */ -#define H5_VERS_RELEASE 1 +#define H5_VERS_RELEASE 2 /** * For pre-releases like \c snap0. Empty string for official releases. */ @@ -91,7 +91,7 @@ /** * Full version string */ -#define H5_VERS_INFO "HDF5 library version: 1.13.1-1" +#define H5_VERS_INFO "HDF5 library version: 1.13.2-1" #define H5check() H5check_version(H5_VERS_MAJOR, H5_VERS_MINOR, H5_VERS_RELEASE) diff --git a/tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl b/tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl index 15ae813..0992a0c 100644 --- a/tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl +++ b/tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl @@ -11,7 +11,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 1 } + PARAMS { 9 1 13 2 } } } FILLVALUE { @@ -33,7 +33,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 1 } + PARAMS { 9 1 13 2 } } } FILLVALUE { @@ -55,7 +55,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 1 } + PARAMS { 9 1 13 2 } } } FILLVALUE { @@ -77,7 +77,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 1 } + PARAMS { 9 1 13 2 } } } FILLVALUE { @@ -99,7 +99,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 1 } + PARAMS { 9 1 13 2 } } } FILLVALUE { @@ -121,7 +121,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 1 } + PARAMS { 9 1 13 2 } } } FILLVALUE { @@ -143,7 +143,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 1 } + PARAMS { 9 1 13 2 } } } FILLVALUE { -- cgit v0.12 From 7c18723404757cf96175df68c402cbc77ad0665e Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Tue, 1 Mar 2022 14:48:12 -0600 Subject: Create HISTORY-1_13.txt and update release.txt (#1466) --- MANIFEST | 1 + release_docs/HISTORY-1_13.txt | 1758 +++++++++++++++++++++++++++++++++++++++++ release_docs/RELEASE.txt | 1435 +-------------------------------- 3 files changed, 1775 insertions(+), 1419 deletions(-) create mode 100644 release_docs/HISTORY-1_13.txt diff --git a/MANIFEST b/MANIFEST index a7ab9c7..bc967be 100644 --- a/MANIFEST +++ b/MANIFEST @@ -620,6 +620,7 @@ ./release_docs/HISTORY-1_8.txt ./release_docs/HISTORY-1_10.txt ./release_docs/HISTORY-1_12.txt +./release_docs/HISTORY-1_13.txt ./release_docs/INSTALL ./release_docs/INSTALL_CMake.txt ./release_docs/INSTALL_Cygwin.txt diff --git a/release_docs/HISTORY-1_13.txt b/release_docs/HISTORY-1_13.txt new file mode 100644 index 0000000..06be75b --- /dev/null +++ b/release_docs/HISTORY-1_13.txt @@ -0,0 +1,1758 @@ +HDF5 History +============ + +This file contains development history of the HDF5 1.13 releases from +the develop branch + +01. Release Information for hdf5-1.13.0 + +[Search on the string '%%%%' for section breaks of each release.] + +%%%%1.13.0%%%% + +HDF5 version 1.13.0 released on 2021-12-01 +================================================================================ + + +INTRODUCTION +============ + +This document describes the differences between this release and the previous +HDF5 release. It contains information on the platforms tested and known +problems in this release. For more details check the HISTORY*.txt files in the +HDF5 source. + +Note that documentation in the links below will be updated at the time of each +final release. + +Links to HDF5 documentation can be found on The HDF5 web page: + + https://portal.hdfgroup.org/display/HDF5/HDF5 + +The official HDF5 releases can be obtained from: + + https://www.hdfgroup.org/downloads/hdf5/ + +Changes from Release to Release and New Features in the HDF5-1.13.x release series +can be found at: + + https://portal.hdfgroup.org/display/HDF5/HDF5+Application+Developer%27s+Guide + +If you have any questions or comments, please send them to the HDF Help Desk: + + help@hdfgroup.org + + +CONTENTS +======== + +- New Features +- New platforms, languages and compilers tested +- Bug Fixes since HDF5-1.12.0 +- Platforms Tested +- Known Problems +- CMake vs. Autotools installations + + +New Features +============ + + Configuration: + ------------- + - Added new configure option to support building parallel tools. + See Tools below (autotools - CMake): + --enable-parallel-tools HDF5_BUILD_PARALLEL_TOOLS + + (RAW - 2021/10/25) + + - Added new configure options to enable dimension scales APIs (H5DS*) to + use new object references with the native VOL connector (aka native HDF5 + library). New references are always used for non-native terminal VOL + connectors (e.g., DAOS). + + Autotools --enable-dimension-scales-with-new-ref + CMake HDF5_DIMENSION_SCALES_NEW_REF=ON + + (EIP - 2021/10/25, HDFFV-11180) + + - Refactored the utils folder. + + Added subfolder test and moved the 'swmr_check_compat_vfd.c file' + from test into utils/test. Deleted the duplicate swmr_check_compat_vfd.c + file in hl/tools/h5watch folder. Also fixed vfd check options. + + (ADB - 2021/10/18) + + - Changed autotools and CMake configurations to derive both + compilation warnings-as-errors and warnings-only-warn configurations + from the same files, 'config/*/*error*'. Removed redundant files + 'config/*/*noerror*'. + + (DCY - 2021/09/29) + + - Added new option to control the build of High-Level tools + that default ON/enabled. + + Add configure options (autotools - CMake): + --enable-hltools HDF5_BUILD_HL_TOOLS + + Disabling this option prevents building the gif tool which + contains the following CVEs: + HDFFV-10592 CVE-2018-17433 + HDFFV-10593 CVE-2018-17436 + HDFFV-11048 CVE-2020-10809 + + (ADB - 2021/09/16, HDFFV-11266) + + - Adds C++ Autotools configuration file for Intel + + * Checks for icpc as the compiler + * Sets std=c++11 + * Copies most non-warning flags from intel-flags + + (DER - 2021/06/02) + + - Adds C++ Autotools configuration file for PGI + + * Checks for pgc++ as the compiler name (was: pgCC) + * Sets -std=c++11 + * Other options basically match new C options (below) + + (DER - 2021/06/02) + + - Updates PGI C options + + * -Minform set to warn (was: inform) to suppress spurious messages + * Sets -gopt -O2 as debug options + * Sets -O4 as 'high optimization' option + * Sets -O0 as 'no optimization' option + * Removes specific settings for PGI 9 and 10 + + (DER - 2021/06/02) + + - A C++11-compliant compiler is now required to build the C++ wrappers + + CMAKE_CXX_STANDARD is now set to 11 when building with CMake and + -std=c++11 is added when building with clang/gcc via the Autotools. + + (DER - 2021/05/27) + + - CMake will now run the shell script tests in test/ by default + + The test directory includes several shell script tests that previously + were not run by CMake. These are now run by default. TEST_SHELL_SCRIPTS + has been set to ON and SH_PROGRAM has been set to bash (some test + scripts use bash-isms). Platforms without bash (e.g., Windows) will + ignore the script tests. + + (DER - 2021/05/23) + + - Removed unused HDF5_ENABLE_HSIZET option from CMake + + This has been unused for some time and has no effect. + + (DER - 2021/05/23) + + - CMake no longer builds the C++ library by default + + HDF5_BUILD_CPP_LIB now defaults to OFF, which is in line with the + Autotools build defaults. + + (DER - 2021/04/20) + + - Removal of pre-VS2015 work-arounds + + HDF5 now requires Visual Studio 2015 or greater, so old work-around + code and definitions have been removed, including: + + * + * snprintf and vsnprintf + * llround, llroundf, lround, lroundf, round, roundf + * strtoll and strtoull + * va_copy + * struct timespec + + (DER - 2021/03/22) + + - Add CMake variable HDF5_LIB_INFIX + + This infix is added to all library names after 'hdf5'. + e.g. the infix '_openmpi' results in the library name 'libhdf5_openmpi.so' + This name is used in packages on debian based systems. + (see https://packages.debian.org/jessie/amd64/libhdf5-openmpi-8/filelist) + + (barcode - 2021/03/22) + + - On macOS, Universal Binaries can now be built, allowing native execution on + both Intel and Apple Silicon (ARM) based Macs. + + To do so, set CMAKE_OSX_ARCHITECTURES="x86_64;arm64" + + (SAM - 2021/02/07, github-311) + + - Added a configure-time option to control certain compiler warnings + diagnostics + + A new configure-time option was added that allows some compiler warnings + diagnostics to have the default operation. This is mainly intended for + library developers and currently only works for gcc 10 and above. The + diagnostics flags apply to C, C++ and Fortran compilers and will appear + in "H5 C Flags", H5 C++ Flags" and H5 Fortran Flags, respectively. They + will NOT be exported to h5cc, etc. + + The default is OFF, which will disable the warnings URL and color attributes + for the warnings output. ON will not add the flags and allow default behavior. + + Autotools: --enable-diags + + CMake: HDF5_ENABLE_BUILD_DIAGS + + (ADB - 2021/02/05, HDFFV-11213) + + - CMake option to build the HDF filter plugins project as an external project + + The HDF filter plugins project is a collection of registered compression + filters that can be dynamically loaded when needed to access data stored + in a hdf5 file. This CMake-only option allows the plugins to be built and + distributed with the hdf5 library and tools. Like the options for szip and + zlib, either a tgz file or a git repository can be specified for the source. + + The option was refactored to use the CMake FetchContent process. This allows + more control over the filter targets, but required external project command + options to be moved to a CMake include file, HDF5PluginCache.cmake. Also + enabled the filter examples to be used as tests for operation of the + filter plugins. + + (ADB - 2020/12/10, OESS-98) + + - FreeBSD Autotools configuration now defaults to 'cc' and 'c++' compilers + + On FreeBSD, the autotools defaulted to 'gcc' as the C compiler and did + not process C++ options. Since FreeBSD 10, the default compiler has + been clang (via 'cc'). + + The default compilers have been set to 'cc' for C and 'c++' for C++, + which will pick up clang and clang++ respectively on FreeBSD 10+. + Additionally, clang options are now set correctly for both C and C++ + and g++ options will now be set if that compiler is being used (an + omission from the former functionality). + + (DER - 2020/11/28, HDFFV-11193) + + - Fixed POSIX problems when building w/ gcc on Solaris + + When building on Solaris using gcc, the POSIX symbols were not + being set correctly, which could lead to issues like clock_gettime() + not being found. + + The standard is now set to gnu99 when building with gcc on Solaris, + which allows POSIX things to be #defined and linked correctly. This + differs slightly from the gcc norm, where we set the standard to c99 + and manually set POSIX #define symbols. + + (DER - 2020/11/25, HDFFV-11191) + + - Added a configure-time option to consider certain compiler warnings + as errors + + A new configure-time option was added that converts some compiler warnings + to errors. This is mainly intended for library developers and currently + only works for gcc and clang. The warnings that are considered errors + will appear in the generated libhdf5.settings file. These warnings apply + to C and C++ code and will appear in "H5 C Flags" and H5 C++ Flags", + respectively. They will NOT be exported to h5cc, etc. + + The default is OFF. Building with this option may fail when compiling + on operating systems and with compiler versions not commonly used by + the library developers. Compilation may also fail when headers not + under the control of the library developers (e.g., mpi.h, hdfs.h) raise + warnings. + + Autotools: --enable-warnings-as-errors + + CMake: HDF5_ENABLE_WARNINGS_AS_ERRORS + + (DER - 2020/11/23, HDFFV-11189) + + - Autotools and CMake target added to produce doxygen generated documentation + + The default is OFF or disabled. + Autoconf option is '--enable-doxygen' + autotools make target is 'doxygen' and will build all doxygen targets + CMake configure option is 'HDF5_BUILD_DOC'. + CMake target is 'doxygen' for all available doxygen targets + CMake target is 'hdf5lib_doc' for the src subdirectory + + (ADB - 2020/11/03) + + - CMake option to use MSVC naming conventions with MinGW + + HDF5_MSVC_NAMING_CONVENTION option enable to use MSVC naming conventions + when using a MinGW toolchain + + (xan - 2020/10/30) + + - CMake option to statically link gcc libs with MinGW + + HDF5_MINGW_STATIC_GCC_LIBS allows to statically link libg/libstdc++ + with the MinGW toolchain + + (xan - 2020/10/30) + + - CMake option to build the HDF filter plugins project as an external project + + The HDF filter plugins project is a collection of registered compression + filters that can be dynamically loaded when needed to access data stored + in a hdf5 file. This CMake-only option allows the plugins to be built and + distributed with the hdf5 library and tools. Like the options for szip and + zlib, either a tgz file or a git repository can be specified for the source. + + The necessary options are (see the INSTALL_CMake.txt file): + HDF5_ENABLE_PLUGIN_SUPPORT + PLUGIN_TGZ_NAME or PLUGIN_GIT_URL + There are more options necessary for various filters and the plugin project + documents should be referenced. + + (ADB - 2020/09/27, OESS-98) + + - Added CMake option to format source files + + HDF5_ENABLE_FORMATTERS option will enable creation of targets using the + pattern - HDF5_*_SRC_FORMAT - where * corresponds to the source folder + or tool folder. All sources can be formatted by executing the format target; + make format + + (ADB - 2020/08/24) + + - Add file locking configure and CMake options + + HDF5 1.10.0 introduced a file locking scheme, primarily to help + enforce SWMR setup. Formerly, the only user-level control of the scheme + was via the HDF5_USE_FILE_LOCKING environment variable. + + This change introduces configure-time options that control whether + or not file locking will be used and whether or not the library + ignores errors when locking has been disabled on the file system + (useful on some HPC Lustre installations). + + In both the Autotools and CMake, the settings have the effect of changing + the default property list settings (see the H5Pset/get_file_locking() + entry, below). + + The yes/no/best-effort file locking configure setting has also been + added to the libhdf5.settings file. + + Autotools: + + An --enable-file-locking=(yes|no|best-effort) option has been added. + + yes: Use file locking. + no: Do not use file locking. + best-effort: Use file locking and ignore "disabled" errors. + + CMake: + + Two self-explanatory options have been added: + + HDF5_USE_FILE_LOCKING + HDF5_IGNORE_DISABLED_FILE_LOCKS + + Setting both of these to ON is the equivalent to the Autotools' + best-effort setting. + + NOTE: + The precedence order of the various file locking control mechanisms is: + + 1) HDF5_USE_FILE_LOCKING environment variable (highest) + + 2) H5Pset_file_locking() + + 3) configure/CMake options (which set the property list defaults) + + 4) library defaults (currently best-effort) + + (DER - 2020/07/30, HDFFV-11092) + + - CMake option to link the generated Fortran MOD files into the include + directory. + + The Fortran generation of MOD files by a Fortran compile can produce + different binary files between SHARED and STATIC compiles with different + compilers and/or different platforms. Note that it has been found that + different versions of Fortran compilers will produce incompatible MOD + files. Currently, CMake will locate these MOD files in subfolders of + the include directory and add that path to the Fortran library target + in the CMake config file, which can be used by the CMake find library + process. For other build systems using the binary from a CMake install, + a new CMake configuration can be used to copy the pre-chosen version + of the Fortran MOD files into the install include directory. + + The default will depend on the configuration of + BUILD_STATIC_LIBS and BUILD_SHARED_LIBS: + YES YES Default to SHARED + YES NO Default to STATIC + NO YES Default to SHARED + NO NO Default to SHARED + The defaults can be overridden by setting the config option + HDF5_INSTALL_MOD_FORTRAN to one of NO, SHARED, or STATIC + + (ADB - 2020/07/09, HDFFV-11116) + + - CMake option to use AEC (open source SZip) library instead of SZip + + The open source AEC library is a replacement library for SZip. In + order to use it for hdf5 the libaec CMake source was changed to add + "-fPIC" and exclude test files. Autotools does not build the + compression libraries within hdf5 builds. New option USE_LIBAEC is + required to compensate for the different files produced by AEC build. + + (ADB - 2020/04/22, OESS-65) + + - CMake ConfigureChecks.cmake file now uses CHECK_STRUCT_HAS_MEMBER + + Some handcrafted tests in HDFTests.c has been removed and the CMake + CHECK_STRUCT_HAS_MEMBER module has been used. + + (ADB - 2020/03/24, TRILAB-24) + + - Both build systems use same set of warnings flags + + GNU C, C++ and gfortran warnings flags were moved to files in a config + sub-folder named gnu-warnings. Flags that only are available for a specific + version of the compiler are in files named with that version. + Clang C warnings flags were moved to files in a config sub-folder + named clang-warnings. + Intel C, Fortran warnings flags were moved to files in a config sub-folder + named intel-warnings. + + There are flags in named "error-xxx" files with warnings that may + be promoted to errors. Some source files may still need fixes. + + There are also pairs of files named "developer-xxx" and "no-developer-xxx" + that are chosen by the CMake option:HDF5_ENABLE_DEV_WARNINGS or the + configure option:--enable-developer-warnings. + + In addition, CMake no longer applies these warnings for examples. + + (ADB - 2020/03/24, TRILAB-192) + + - Added test script for file size compare + + If CMake minimum version is at least 3.14, the fileCompareTest.cmake + script will compare file sizes. + + (ADB - 2020/02/24, HDFFV-11036) + + - Update CMake minimum version to 3.12 + + Updated CMake minimum version to 3.12 and added version checks + for Windows features. + + (ADB - 2020/02/05, TRILABS-142) + + - Fixed CMake include properties for Fortran libraries + + Corrected the library properties for Fortran to use the + correct path for the Fortran module files. + + (ADB - 2020/02/04, HDFFV-11012) + + - Added common warnings files for gnu and intel + + Added warnings files to use one common set of flags + during configure for both autotools and CMake build + systems. The initial implementation only affects a + general set of flags for gnu and intel compilers. + + (ADB - 2020/01/17) + + - Added new options to CMake for control of testing + + Added CMake options (default ON); + HDF5_TEST_SERIAL AND/OR HDF5_TEST_PARALLEL + combined with: + HDF5_TEST_TOOLS + HDF5_TEST_EXAMPLES + HDF5_TEST_SWMR + HDF5_TEST_FORTRAN + HDF5_TEST_CPP + HDF5_TEST_JAVA + + (ADB - 2020/01/15, HDFFV-11001) + + - Added Clang sanitizers to CMake for analyzer support if compiler is clang. + + Added CMake code and files to execute the Clang sanitizers if + HDF5_ENABLE_SANITIZERS is enabled and the USE_SANITIZER option + is set to one of the following: + Address + Memory + MemoryWithOrigins + Undefined + Thread + Leak + 'Address;Undefined' + + (ADB - 2019/12/12, TRILAB-135) + + - Update CMake for VS2019 support + + CMake added support for VS2019 in version 3.15. Changes to the CMake + generator setting required changes to scripts. Also updated version + references in CMake files as necessary. + + (ADB - 2019/11/18, HDFFV-10962) + + - Update CMake options to match new autotools options + + Add configure options (autotools - CMake): + enable-asserts HDF5_ENABLE_ASSERTS + enable-symbols HDF5_ENABLE_SYMBOLS + enable-profiling HDF5_ENABLE_PROFILING + enable-optimization HDF5_ENABLE_OPTIMIZATION + In addition NDEBUG is no longer forced defined and relies on the CMake + process. + + (ADB - 2019/10/07, HDFFV-100901, HDFFV-10637, TRILAB-97) + + + Library: + -------- + - Adds new file driver-level memory copy operation for + "ctl" callback and updates compact dataset I/O routines + to utilize it + + When accessing an HDF5 file with a file driver that uses + memory allocated in special ways (e.g., without standard + library's `malloc`), a crash could be observed when HDF5 + tries to perform `memcpy` operations on such a memory + region. + + These changes add a new H5FD_FEAT_MEMMANAGE VFD feature + flag, which, if specified as supported by a VFD, will + inform HDF5 that the VFD either uses special memory + management routines or wishes to perform memory management + in a specific way. Therefore, this flag instructs HDF5 to + ask the file driver to perform memory management for + certain operations. + + These changes also introduce a new "ctl" callback + operation identified by the H5FD_CTL__MEM_COPY op code. + This operation simply asks a VFD to perform a memory copy. + The arguments to this operation are passed to the "ctl" + callback's "input" parameter as a pointer to a struct + defined as: + + struct H5FD_ctl_memcpy_args_t { + void * dstbuf; /**< Destination buffer */ + hsize_t dst_off; /**< Offset within destination buffer */ + const void *srcbuf; /**< Source buffer */ + hsize_t src_off; /**< Offset within source buffer */ + size_t len; /**< Length of data to copy from source buffer */ + } H5FD_ctl_memcpy_args_t; + + Further, HDF5's compact dataset I/O routines were + identified as a problematic area that could cause a crash + for VFDs that make use of special memory management. Those + I/O routines were therefore updated to make use of this new + "ctl" callback operation in order to ask the underlying + file driver to correctly handle memory copies. + + (JTH - 2021/09/28) + + - Adds new "ctl" callback to VFD H5FD_class_t structure + with the following prototype: + + herr_t (*ctl)(H5FD_t *file, uint64_t op_code, + uint64_t flags, const void *input, + void **output); + + This newly-added "ctl" callback allows Virtual File + Drivers to intercept and handle arbitrary operations + identified by an operation code. Its parameters are + as follows: + + `file` [in] - A pointer to the file to be operated on + `op_code` [in] - The operation code identifying the + operation to be performed + `flags` [in] - Flags governing the behavior of the + operation performed (see H5FDpublic.h + for a list of valid flags) + `input` [in] - A pointer to arguments passed to the + VFD performing the operation + `output` [out] - A pointer for the receiving VFD to + use for output from the operation + + (JRM - 2021/08/16) + + - Change how the release part of version, in major.minor.release is checked + for compatibility + + The HDF5 library uses a function, H5check_version, to check that + the version defined in the header files, which is used to compile an + application is compatible with the version codified in the library, which + the application loads at runtime. This previously required an exact match + or the library would print a warning, dump the build settings and then + abort or continue. An environment variable controlled the logic. + + Now the function first checks that the library release version, in + major.minor.release, is not older than the version in the headers. + Secondly, if the release version is different, it checks if either + the library version or the header version is in the exception list, in + which case the release part of version, in major.minor.release, must + be exact. An environment variable still controls the logic. + + (ADB - 2021/07/27) + + - gcc warning suppression macros were moved out of H5public.h + + The HDF5 library uses a set of macros to suppress warnings on gcc. + These warnings were originally located in H5public.h so that the + multi VFD (which only uses public headers) could also make use of them + but internal macros should not be publicly exposed like this. + + These macros have now been moved to H5private.h. Pending future multi + VFD refactoring, the macros have been duplicated in H5FDmulti.c to + suppress the format string warnings there. + + (DER - 2021/06/03) + + - H5Gcreate1() now rejects size_hint parameters larger than UINT32_MAX + + The size_hint value is ultimately stored in a uint32_t struct field, + so specifying a value larger than this on a 64-bit machine can cause + undefined behavior including crashing the system. + + The documentation for this API call was also incorrect, stating that + passing a negative value would cause the library to use a default + value. Instead, passing a "negative" value actually passes a very large + value, which is probably not what the user intends and can cause + crashes on 64-bit systems. + + The Doxygen documentation has been updated and passing values larger + than UINT32_MAX for size_hint will now produce a normal HDF5 error. + + (DER - 2021/04/29, HDFFV-11241) + + + - H5Pset_fapl_log() no longer crashes when passed an invalid fapl ID + + When passed an invalid fapl ID, H5Pset_fapl_log() would usually + segfault when attempting to free an uninitialized pointer in the error + handling code. This behavior is more common in release builds or + when the memory sanitization checks were not selected as a build + option. + + The pointer is now correctly initialized and the API call now + produces a normal HDF5 error when fed an invalid fapl ID. + + (DER - 2021/04/28, HDFFV-11240) + + - Fixes a segfault when H5Pset_mdc_log_options() is called multiple times + + The call incorrectly attempts to free an internal copy of the previous + log location string, which causes a segfault. This only happens + when the call is invoked multiple times on the same property list. + On the first call to a given fapl, the log location is set to NULL so + the segfault does not occur. + + The string is now handled properly and the segfault no longer occurs. + + (DER - 2021/04/27, HDFFV-11239) + + - HSYS_GOTO_ERROR now emits the results of GetLastError() on Windows + + HSYS_GOTO_ERROR is an internal macro that is used to produce error + messages when system calls fail. These strings include errno and the + the associated strerror() value, which are not particularly useful + when a Win32 API call fails. + + On Windows, this macro has been updated to include the result of + GetLastError(). When a system call fails on Windows, usually only + one of errno and GetLastError() will be useful, however we emit both + for the user to parse. The Windows error message is not emitted as + it would be awkward to free the FormatMessage() buffer given the + existing HDF5 error framework. Users will have to look up the error + codes in MSDN. + + The format string on Windows has been changed from: + + "%s, errno = %d, error message = '%s'" + + to: + + "%s, errno = %d, error message = '%s', Win32 GetLastError() = %"PRIu32"" + + for those inclined to parse it for error values. + + (DER - 2021/03/21) + + - File locking now works on Windows + + Since version 1.10.0, the HDF5 library has used a file locking scheme + to help enforce one reader at a time accessing an HDF5 file, which can + be helpful when setting up readers and writers to use the single- + writer/multiple-readers (SWMR) access pattern. + + In the past, this was only functional on POSIX systems where flock() or + fcntl() were present. Windows used a no-op stub that always succeeded. + + HDF5 now uses LockFileEx() and UnlockFileEx() to lock the file using the + same scheme as POSIX systems. We lock the entire file when we set up the + locks (by passing DWORDMAX as both size parameters to LockFileEx()). + + (DER - 2021/03/19, HDFFV-10191) + + - H5Epush_ret() now requires a trailing semicolon + + H5Epush_ret() is a function-like macro that has been changed to + contain a `do {} while(0)` loop. Consequently, a trailing semicolon + is now required to end the `while` statement. Previously, a trailing + semi would work, but was not mandatory. This change was made to allow + clang-format to correctly format the source code. + + (SAM - 2021/03/03) + + - Improved performance of H5Sget_select_elem_pointlist + + Modified library to cache the point after the last block of points + retrieved by H5Sget_select_elem_pointlist, so a subsequent call to the + same function to retrieve the next block of points from the list can + proceed immediately without needing to iterate over the point list. + + (NAF - 2021/01/19) + + - Replaced H5E_ATOM with H5E_ID in H5Epubgen.h + + The term "atom" is archaic and not in line with current HDF5 library + terminology, which uses "ID" instead. "Atom" has mostly been purged + from the library internals and this change removes H5E_ATOM from + the H5Epubgen.h (exposed via H5Epublic.h) and replaces it with + H5E_ID. + + (DER - 2020/11/24, HDFFV-11190) + + - Add a new public function H5Ssel_iter_reset + + This function resets a dataspace selection iterator back to an + initial state so that it may be used for iteration once more. + This can be useful when needing to iterate over a selection + multiple times without having to repeatedly create/destroy + a selection iterator for that dataspace selection. + + (JTH - 2020/09/18) + + - Remove HDFS VFD stubs + + The original implementation of the HDFS VFD included non-functional + versions of the following public API calls when the HDFS VFD is + not built as a part of the HDF5 library: + + * H5FD_hdfs_init() + * H5Pget_fapl_hdfs() + * H5Pset_fapl_hdfs() + + They will remain present in HDF5 1.10 and HDF5 1.12 releases + for binary compatibility purposes but have been removed as of 1.14.0. + + Note that this has nothing to do with the real HDFS VFD API calls + that are fully functional when the HDFS VFD is configured and built. + + We simply changed: + + #ifdef LIBHDFS + + #else + + #endif + + to: + + #ifdef LIBHDFS + + #endif + + Which is how the other optional VFDs are handled. + + (DER - 2020/08/27) + + - Add Mirror VFD + + Use TCP/IP sockets to perform write-only (W/O) file I/O on a remote + machine. Must be used in conjunction with the Splitter VFD. + + (JOS - 2020/03/13, TBD) + + - Add Splitter VFD + + Maintain separate R/W and W/O channels for "concurrent" file writes + to two files using a single HDF5 file handle. + + (JOS - 2020/03/13, TBD) + + - Refactored public exposure of haddr_t type in favor of "object tokens" + + To better accommodate HDF5 VOL connectors where "object addresses in a file" + may not make much sense, the following changes were made to the library: + + * Introduced new H5O_token_t "object token" type, which represents a + unique and permanent identifier for referencing an HDF5 object within + a container; these "object tokens" are meant to replace object addresses. + Along with the new type, a new H5Oopen_by_token API call was introduced + to open an object by a token, similar to how object addresses were + previously used with H5Oopen_by_addr. + + * Introduced new H5Lget_info2, H5Lget_info_by_idx2, H5Literate2, H5Literate_by_name2, + H5Lvisit2 and H5Lvisit_by_name2 API calls, along with their associated H5L_info2_t + struct and H5L_iterate2_t callback function, which work with the newly-introduced + object tokens, instead of object addresses. The original functions have been + renamed to version 1 functions and are deprecated in favor of the new version 2 + functions. The H5L_info_t and H5L_iterate_t types have been renamed to version 1 + types and are now deprecated in favor of their version 2 counterparts. For each of + the functions and types, compatibility macros take place of the original symbols. + + * Introduced new H5Oget_info3, H5Oget_info_by_name3, H5Oget_info_by_idx3, + H5Ovisit3 and H5Ovisit_by_name3 API calls, along with their associated H5O_info2_t + struct and H5O_iterate2_t callback function, which work with the newly-introduced + object tokens, instead of object addresses. The version 2 functions are now + deprecated in favor of the version 3 functions. The H5O_info_t and H5O_iterate_t + types have been renamed to version 1 types and are now deprecated in favor of their + version 2 counterparts. For each, compatibility macros take place of the original + symbols. + + * Introduced new H5Oget_native_info, H5Oget_native_info_by_name and + H5Oget_native_info_by_idx API calls, along with their associated H5O_native_info_t + struct, which are used to retrieve the native HDF5 file format-specific information + about an object. This information (such as object header info and B-tree/heap info) + has been removed from the new H5O_info2_t struct so that the more generic + H5Oget_info(_by_name/_by_idx)3 routines will not try to retrieve it for non-native + VOL connectors. + + * Added new H5Otoken_cmp, H5Otoken_to_str and H5Otoken_from_str routines to compare + two object tokens, convert an object token into a nicely-readable string format and + to convert an object token string back into a real object token, respectively. + + (DER, QAK, JTH - 2020/01/16) + + - Add new public function H5Sselect_adjust. + + This function shifts a dataspace selection by a specified logical offset + within the dataspace extent. This can be useful for VOL developers to + implement chunked datasets. + + (NAF - 2019/11/18) + + - Add new public function H5Sselect_project_intersection. + + This function computes the intersection between two dataspace selections + and projects that intersection into a third selection. This can be useful + for VOL developers to implement chunked or virtual datasets. + + (NAF - 2019/11/13, ID-148) + + - Add new public function H5VLget_file_type. + + This function returns a datatype equivalent to the supplied datatype but + with the location set to be in the file. This datatype can then be used + with H5Tconvert to convert data between file and in-memory representation. + This function is intended for use only by VOL connector developers. + + (NAF - 2019/11/08, ID-127) + + + Parallel Library: + ----------------- + - + + + Fortran Library: + ---------------- + - H5Fget_name_f fixed to handle correctly trailing whitespaces and + newly allocated buffers. + + (MSB - 2021/08/30, github-826,972) + + - Add wrappers for H5Pset/get_file_locking() API calls + + h5pget_file_locking_f() + h5pset_file_locking_f() + + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. + + (DER - 2020/07/30, HDFFV-11092) + + C++ Library: + ------------ + - Add wrappers for H5Pset/get_file_locking() API calls + + FileAccPropList::setFileLocking() + FileAccPropList::getFileLocking() + + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. + + (DER - 2020/07/30, HDFFV-11092) + + + Java Library: + ------------- + - Replaced HDF5AtomException with HDF5IdException + + Since H5E_ATOM changed to H5E_ID in the C library, the Java exception + that wraps the error category was also renamed. Its functionality + remains unchanged aside from the name. + + (See also the HDFFV-11190 note in the C library section) + + (DER - 2020/11/24, HDFFV-11190) + + - Added new H5S functions. + + H5Sselect_copy, H5Sselect_shape_same, H5Sselect_adjust, + H5Sselect_intersect_block, H5Sselect_project_intersection, + H5Scombine_hyperslab, H5Smodify_select, H5Scombine_select + wrapper functions added. + + (ADB - 2020/10/27, HDFFV-10868) + + - Add wrappers for H5Pset/get_file_locking() API calls + + H5Pset_file_locking() + H5Pget_use_file_locking() + H5Pget_ignore_disabled_file_locking() + + Unlike the C++ and Fortran wrappers, there are separate getters for the + two file locking settings, each of which returns a boolean value. + + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. + + (DER - 2020/07/30, HDFFV-11092) + + - Added ability to test java library with VOLs. + + Created a new CMake script that combines the java and vol test scripts. + + (ADB - 2020/02/03, HDFFV-10996) + + - Tests fail for non-English locales. + + In the JUnit tests with a non-English locale, only the part before + the decimal comma is replaced by XXXX and this leads to a comparison + error. Changed the regex for the Time substitution. + + (ADB - 2020/01/09, HDFFV-10995) + + + Tools: + ------ + - Added a new (unix ONLY) parallel meta tool 'h5dwalk', which utilizes the + mpifileutils (https://hpc.github.io/mpifileutils) open source utility + library to enable parallel execution of other HDF5 tools. + This approach can greatly enhance the serial hdf5 tool performance over large + collections of files by utilizing MPI parallelism to distribute an application + load over many independent MPI ranks and files. + + An introduction to the mpifileutils library and initial 'User Guide' for + the new 'h5dwalk" tool can be found at: + https://github.com/HDFGroup/hdf5doc/tree/master/RFCs/HDF5/tools/parallel_tools + + (RAW - 2021/10/25) + + - Refactored the perform tools and removed depends on test library. + + Moved the perf and h5perf tools from tools/test/perform to + tools/src/h5perf so that they can be installed. This required + that the test library dependency be removed by copying the + needed functions from h5test.c. + The standalone scripts and other perform tools remain in the + tools/test/perform folder. + + (ADB - 2021/08/10) + + - Removed partial long exceptions + + Some of the tools accepted shortened versions of the long options + (ex: --datas instead of --dataset). These were implemented inconsistently, + are difficult to maintian, and occasionally block useful long option + names. These partial long options have been removed from all the tools. + + (DER - 2021/08/03) + + - h5repack added help text for user-defined filters. + + Added help text line that states the valid values of the filter flag + for user-defined filters; + filter_flag: 1 is OPTIONAL or 0 is MANDATORY + + (ADB - 2021/01/14, HDFFV-11099) + + - Added h5delete tool + + Deleting HDF5 storage when using the VOL can be tricky when the VOL + does not create files. The h5delete tool is a simple wrapper around + the H5Fdelete() API call that uses the VOL specified in the + HDF5_VOL_CONNECTOR environment variable to delete a "file". If + the call to H5Fdelete() fails, the tool will attempt to use + the POSIX remove(3) call to remove the file. + + Note that the HDF5 library does currently have support for + H5Fdelete() in the native VOL connector. + + (DER - 2020/12/16) + + - h5repack added options to control how external links are handled. + + Currently h5repack preserves external links and cannot copy and merge + data from the external files. Two options, merge and prune, were added to + control how to merge data from an external link into the resulting file. + --merge Follow external soft link recursively and merge data. + --prune Do not follow external soft links and remove link. + --merge --prune Follow external link, merge data and remove dangling link. + + (ADB - 2020/08/05, HDFFV-9984) + + - h5repack was fixed to repack the reference attributes properly. + The code line that checks if the update of reference inside a compound + datatype is misplaced outside the code block loop that carries out the + check. In consequence, the next attribute that is not the reference + type was repacked again as the reference type and caused the failure of + repacking. The fix is to move the corresponding code line to the correct + code block. + + (KY -2020/02/07, HDFFV-11014) + + - h5diff was updated to use the new reference APIs. + + h5diff uses the new reference APIs to compare references. + Attribute references can also be compared. + + (ADB - 2019/12/19, HDFFV-10980) + + - h5dump and h5ls were updated to use the new reference APIs. + + The tools library now use the new reference APIs to inspect a + file. Also the DDL spec was updated to reflect the format + changes produced with the new APIs. The export API and support + functions in the JNI were updated to match. + + (ADB - 2019/12/06, HDFFV-10876 and HDFFV-10877) + + + High-Level APIs: + ---------------- + - added set/get for unsigned long long attributes + + The attribute writing high-level API has been expanded to include + public set/get functions for ULL attributes, analogously to the + existing set/get for other types. + + (AF - 2021/09/08) + + C Packet Table API: + ------------------- + - + + Internal header file: + --------------------- + - + + Documentation: + -------------- + - + + +New platforms, languages and compilers tested +============================================= + - Linux 5.13.14-200.fc34 #1 SMP x86_64 GNU/Linux Fedora34 + - Linux 5.11.0-34-generic #36-Ubuntu SMP x86_64 GNU/Linux Ubuntu 20.04 + - Linux 5.3.18-22-default #1 SMP x86_64 GNU/Linux SUSE15sp2 + - Linux-4.14.0-115.21.2 #1 SMP ppc64le GNU/Linux + - Linux-4.12.14-150.75-default #1 SMP x86_64 GNU/Linux + - macOS Apple M1 11.6 Darwin 20.6.0 arm64 + - macOS Big Sur 11.3.1 Darwin 20.4.0 x86_64 + - clang versions 11.0.1, 12.0.5 + - Visual Studio 2019 w/ clang 12.0.0 with MSVC-like command-line + + +Bug Fixes since HDF5-1.12.0 release +=================================== + Library + ------- + - Fixed an H5Pget_filter_by_id1/2() assert w/ out of range filter IDs + + Both H5Pget_filter_by_id1 and 2 did not range check the filter ID, which + could trip as assert in debug versions of the library. The library now + returns a normal HDF5 error when the filter ID is out of range. + + (DER - 2021/11/23, HDFFV-11286) + + - Fixed an issue with collective metadata reads being permanently disabled + after a dataset chunk lookup operation. This would usually cause a + mismatched MPI_Bcast and MPI_ERR_TRUNCATE issue in the library for + simple cases of H5Dcreate() -> H5Dwrite() -> H5Dcreate(). + + (JTH - 2021/11/08, HDFFV-11090) + + - Fixed cross platform incompatibility of references within variable length + types + + Reference types within variable length types previously could not be + read on a platform with different endianness from where they were + written. Fixed so cross platform portability is restored. + + (NAF - 2021/09/30) + + - Detection of simple data transform function "x" + + In the case of the simple data transform function "x" the (parallel) + library recognizes this is the same as not applying this data transform + function. This improves the I/O performance. In the case of the parallel + library, it also avoids breaking to independent I/O, which makes it + possible to apply a filter when writing or reading data to or from + the HDF5 file. + + (JWSB - 2021/09/13) + + - Fixed an invalid read and memory leak when parsing corrupt file space + info messages + + When the corrupt file from CVE-2020-10810 was parsed by the library, + the code that imports the version 0 file space info object header + message to the version 1 struct could read past the buffer read from + the disk, causing an invalid memory read. Not catching this error would + cause downstream errors that eventually resulted in a previously + allocated buffer to be unfreed when the library shut down. In builds + where the free lists are in use, this could result in an infinite loop + and SIGABRT when the library shuts down. + + We now track the buffer size and raise an error on attempts to read + past the end of it. + + (DER - 2021/08/12, HDFFV-11053) + + + - Fixed CVE-2018-14460 + + The tool h5repack produced a segfault when the rank in dataspace + message was corrupted, causing invalid read while decoding the + dimension sizes. + + The problem was fixed by ensuring that decoding the dimension sizes + and max values will not go beyond the end of the buffer. + + (BMR - 2021/05/12, HDFFV-11223) + + - Fixed CVE-2018-11206 + + The tool h5dump produced a segfault when the size of a fill value + message was corrupted and caused a buffer overflow. + + The problem was fixed by verifying the fill value's size + against the buffer size before attempting to access the buffer. + + (BMR - 2021/03/15, HDFFV-10480) + + - Fixed CVE-2018-14033 (same issue as CVE-2020-10811) + + The tool h5dump produced a segfault when the storage size message + was corrupted and caused a buffer overflow. + + The problem was fixed by verifying the storage size against the + buffer size before attempting to access the buffer. + + (BMR - 2021/03/15, HDFFV-11159/HDFFV-11049) + + - Remove underscores on header file guards + + Header file guards used a variety of underscores at the beginning of the define. + + Removed all leading (some trailing) underscores from header file guards. + + (ADB - 2021/03/03, #361) + + - Fixed a segmentation fault + + A segmentation fault occurred with a Mathworks corrupted file. + + A detection of accessing a null pointer was added to prevent the problem. + + (BMR - 2021/02/19, HDFFV-11150) + + - Fixed issue with MPI communicator and info object not being + copied into new FAPL retrieved from H5F_get_access_plist + + Added logic to copy the MPI communicator and info object into + the output FAPL. MPI communicator is retrieved from the VFD, while + the MPI info object is retrieved from the file's original FAPL. + + (JTH - 2021/02/15, HDFFV-11109) + + - Fixed problems with vlens and refs inside compound using + H5VLget_file_type() + + Modified library to properly ref count H5VL_object_t structs and only + consider file vlen and reference types to be equal if their files are + the same. + + (NAF - 2021/01/22) + + - Fixed CVE-2018-17432 + + The tool h5repack produced a segfault on a corrupted file which had + invalid rank for scalar or NULL datatype. + + The problem was fixed by modifying the dataspace encode and decode + functions to detect and report invalid rank. h5repack now fails + with an error message for the corrupted file. + + (BMR - 2020/10/26, HDFFV-10590) + + - Creation of dataset with optional filter + + When the combination of type, space, etc doesn't work for filter + and the filter is optional, it was supposed to be skipped but it was + not skipped and the creation failed. + + Allowed the creation of the dataset in such a situation. + + (BMR - 2020/08/13, HDFFV-10933) + + - Explicitly declared dlopen to use RTLD_LOCAL + + dlopen documentation states that if neither RTLD_GLOBAL nor + RTLD_LOCAL are specified, then the default behavior is unspecified. + The default on linux is usually RTLD_LOCAL while macos will default + to RTLD_GLOBAL. + + (ADB - 2020/08/12, HDFFV-11127) + + - H5Sset_extent_none() sets the dataspace class to H5S_NO_CLASS which + causes asserts/errors when passed to other dataspace API calls. + + H5S_NO_CLASS is an internal class value that should not have been + exposed via a public API call. + + In debug builds of the library, this can cause assert() function to + trip. In non-debug builds, it will produce normal library errors. + + The new library behavior is for H5Sset_extent_none() to convert + the dataspace into one of type H5S_NULL, which is better handled + by the library and easier for developers to reason about. + + (DER - 2020/07/27, HDFFV-11027) + + - Fixed issues CVE-2018-13870 and CVE-2018-13869 + + When a buffer overflow occurred because a name length was corrupted + and became very large, h5dump crashed on memory access violation. + + A check for reading pass the end of the buffer was added to multiple + locations to prevent the crashes and h5dump now simply fails with an + error message when this error condition occurs. + + (BMR - 2020/07/22, HDFFV-11120 and HDFFV-11121) + + - Fixed the segmentation fault when reading attributes with multiple threads + + It was reported that the reading of attributes with variable length string + datatype will crash with segmentation fault particularly when the number of + threads is high (>16 threads). The problem was due to the file pointer that + was set in the variable length string datatype for the attribute. That file + pointer was already closed when the attribute was accessed. + + The problem was fixed by setting the file pointer to the current opened file pointer + when the attribute was accessed. Similar patch up was done before when reading + dataset with variable length string datatype. + + (VC - 2020/07/13, HDFFV-11080) + + - Fixed CVE-2020-10810 + + The tool h5clear produced a segfault during an error recovery in + the superblock decoding. An internal pointer was reset to prevent + further accessing when it is not assigned with a value. + + (BMR - 2020/06/29, HDFFV-11053) + + - Fixed CVE-2018-17435 + + The tool h52gif produced a segfault when the size of an attribute + message was corrupted and caused a buffer overflow. + + The problem was fixed by verifying the attribute message's size + against the buffer size before accessing the buffer. h52gif was + also fixed to display the failure instead of silently exiting + after the segfault was eliminated. + + (BMR - 2020/06/19, HDFFV-10591) + + + Java Library + ------------ + - JNI utility function does not handle new references. + + The JNI utility function for converting reference data to string did + not use the new APIs. In addition to fixing that function, added new + java tests for using the new APIs. + + (ADB - 2021/02/16, HDFFV-11212) + + - The H5FArray.java class, in which virtually the entire execution time + is spent using the HDFNativeData method that converts from an array + of bytes to an array of the destination Java type. + + 1. Convert the entire byte array into a 1-d array of the desired type, + rather than performing 1 conversion per row; + 2. Use the Java Arrays method copyOfRange to grab the section of the + array from (1) that is desired to be inserted into the destination array. + + (PGT,ADB - 2020/12/13, HDFFV-10865) + + - Added ability to test java library with VOLs. + + Created a new CMake script that combines the java and vol test scripts. + + (ADB - 2020/02/03, HDFFV-10996) + + - Tests fail for non-English locales. + + In the JUnit tests with a non-English locale, only the part before + the decimal comma is replaced by XXXX and this leads to a comparison + error. Changed the regex for the Time substitution. + + (ADB - 2020/01/09, HDFFV-10995) + + + Configuration + ------------- + - Corrected path searched by CMake find_package command + + The install path for cmake find_package files had been changed to use + "share/cmake" + for all platforms. However the trailing "hdf5" directory was not removed. + This "hdf5" additional directory has been removed. + + (ADB - 2021/09/27) + + - Corrected pkg-config compile script + + It was discovered that the position of the "$@" argument for the command + in the compile script may fail on some platforms and configurations. The + position of the "$@"command argument was moved before the pkg-config sub command. + + (ADB - 2021/08/30) + + - Fixed CMake C++ compiler flags + + A recent refactoring of the C++ configure files accidentally removed the + file that executed the enable_language command for C++ needed by the + HDFCXXCompilerFlags.cmake file. Also updated the intel warnings files, + including adding support for windows platforms. + + (ADB - 2021/08/10) + + - Better support for libaec (open-source Szip library) in CMake + + Implemented better support for libaec 1.0.5 (or later) library. This version + of libaec contains improvements for better integration with HDF5. Furthermore, + the variable USE_LIBAEC_STATIC has been introduced to allow to make use of + static version of libaec library. Use libaec_DIR or libaec_ROOT to set + the location in which libaec can be found. + + Be aware, the Szip library of libaec 1.0.4 depends on another library within + libaec library. This dependency is not specified in the current CMake + configuration which means that one can not use the static Szip library of + libaec 1.0.4 when building HDF5. This has been resolved in libaec 1.0.5. + + (JWSB - 2021/06/22) + + - Refactor CMake configure for Fortran + + The Fortran configure tests for KINDs reused a single output file that was + read to form the Integer and Real Kinds defines. However, if config was run + more then once, the CMake completed variable prevented the tests from executing + again and the last value saved in the file was used to create the define. + Creating separate files for each KIND solved the issue. + + In addition the test for H5_PAC_C_MAX_REAL_PRECISION was not pulling in + defines for proper operation and did not define H5_PAC_C_MAX_REAL_PRECISION + correctly for a zero value. This was fixed by supplying the required defines. + In addition it was moved from the Fortran specific HDF5UseFortran.camke file + to the C centric ConfigureChecks.cmake file. + + (ADB - 2021/06/03) + + - Move emscripten flag to compile flags + + The emscripten flag, -O0, was removed from target_link_libraries command + to the correct target_compile_options command. + + (ADB - 2021/04/26 HDFFV-11083) + + - Remove arbitrary warning flag groups from CMake builds + + The arbitrary groups were created to reduce the quantity of warnings being + reported that overwhelmed testing report systems. Considerable work has + been accomplished to reduce the warning count and these arbitrary groups + are no longer needed. + Also the default for all warnings, HDF5_ENABLE_ALL_WARNINGS, is now ON. + + Visual Studio warnings C4100, C4706, and C4127 have been moved to + developer warnings, HDF5_ENABLE_DEV_WARNINGS, and are disabled for normal builds. + + (ADB - 2021/03/22, HDFFV-11228) + + - Reclassify CMake messages, to allow new modes and --log-level option + + CMake message commands have a mode argument. By default, STATUS mode + was chosen for any non-error message. CMake version 3.15 added additional + modes, NOTICE, VERBOSE, DEBUG and TRACE. All message commands with a mode + of STATUS were reviewed and most were reclassified as VERBOSE. The new + mode was protected by a check for a CMake version of at least 3.15. If CMake + version 3.17 or above is used, the user can use the command line option + of "--log-level" to further restrict which message commands are displayed. + + (ADB - 2021/01/11, HDFFV-11144) + + - Fixes Autotools determination of the stat struct having an st_blocks field + + A missing parenthesis in an autoconf macro prevented building the test + code used to determine if the stat struct contains the st_blocks field. + Now that the test functions correctly, the H5_HAVE_STAT_ST_BLOCKS #define + found in H5pubconf.h will be defined correctly on both the Autotools and + CMake. This #define is only used in the tests and does not affect the + HDF5 C library. + + (DER - 2021/01/07, HDFFV-11201) + + - Add missing ENV variable line to hdfoptions.cmake file + + Using the build options to use system SZIP/ZLIB libraries need to also + specify the library root directory. Setting the {library}_ROOT ENV + variable was added to the hdfoptions.cmake file. + + (ADB - 2020/10/19 HDFFV-11108) + + + Tools + ----- + - Changed how h5dump and h5ls identify long double. + + Long double support is not consistent across platforms. Tools will always + identify long double as 128-bit [little/big]-endian float nn-bit precision. + New test file created for datasets with attributes for float, double and + long double. In addition any unknown integer or float datatype will now + also show the number of bits for precision. + These files are also used in the java tests. + + (ADB - 2021/03/24, HDFFV-11229,HDFFV-11113) + + - Fixed tools argument parsing. + + Tools parsing used the length of the option from the long array to match + the option from the command line. This incorrectly matched a shorter long + name option that happened to be a subset of another long option. + Changed to match whole names. + + (ADB - 2021/01/19, HDFFV-11106) + + - The tools library was updated by standardizing the error stack process. + + General sequence is: + h5tools_setprogname(PROGRAMNAME); + h5tools_setstatus(EXIT_SUCCESS); + h5tools_init(); + ... process the command-line (check for error-stack enable) ... + h5tools_error_report(); + ... (do work) ... + h5diff_exit(ret); + + (ADB - 2020/07/20, HDFFV-11066) + + - h5diff fixed a command line parsing error. + + h5diff would ignore the argument to -d (delta) if it is smaller than DBL_EPSILON. + The macro H5_DBL_ABS_EQUAL was removed and a direct value comparison was used. + + (ADB - 2020/07/20, HDFFV-10897) + + - h5diff added a command line option to ignore attributes. + + h5diff would ignore all objects with a supplied path if the exclude-path argument is used. + Adding the exclude-attribute argument will only exclude attributes, with the supplied path, + from comparison. + + (ADB - 2020/07/20, HDFFV-5935) + + - h5diff added another level to the verbose argument to print filenames. + + Added verbose level 3 that is level 2 plus the filenames. The levels are: + 0 : Identical to '-v' or '--verbose' + 1 : All level 0 information plus one-line attribute status summary + 2 : All level 1 information plus extended attribute status report + 3 : All level 2 information plus file names + + (ADB - 2020/07/20, HDFFV-1005) + + - h5repack was fixed to repack the reference attributes properly. + + The code line that checks if the update of reference inside a compound + datatype is misplaced outside the code block loop that carries out the + check. In consequence, the next attribute that is not the reference + type was repacked again as the reference type and caused the failure of + repacking. The fix is to move the corresponding code line to the correct + code block. + + (KY -2020/02/10, HDFFV-11014) + + - h5diff was updated to use the new reference APIs. + + h5diff uses the new reference APIs to compare references. + Attribute references can also be compared. + + (ADB - 2019/12/19, HDFFV-10980) + + - h5dump and h5ls were updated to use the new reference APIs. + + The tools library now use the new reference APIs to inspect a + file. Also the DDL spec was updated to reflect the format + changes produced with the new APIs. The export API and support + functions in the JNI were updated to match. + + (ADB - 2019/12/06, HDFFV-10876 and HDFFV-10877) + + + Performance + ------------- + - + + + Fortran API + ----------- + - Corrected INTERFACE INTENT(IN) to INTENT(OUT) for buf_size in h5fget_file_image_f. + + (MSB - 2020/02/18, HDFFV-11029) + + + High-Level Library + ------------------ + - Fixed HL_test_packet, test for packet table vlen of vlen. + + Incorrect length assignment. + + (ADB - 2021/10/14) + + + Fortran High-Level APIs + ----------------------- + - + + + Documentation + ------------- + - + + + F90 APIs + -------- + - + + + C++ APIs + -------- + - Added DataSet::operator= + + Some compilers complain if the copy constructor is given explicitly + but the assignment operator is implicitly set to default. + + (2021/05/19) + + + Testing + ------- + - Stopped java/test/junit.sh.in installing libs for testing under ${prefix} + + Lib files needed are now copied to a subdirectory in the java/test + directory, and on Macs the loader path for libhdf5.xxxs.so is changed + in the temporary copy of libhdf5_java.dylib. + + (LRK, 2020/07/02, HDFFV-11063) + + +Platforms Tested +=================== + + Linux 5.13.14-200.fc34 GNU gcc (GCC) 11.2.1 2021078 (Red Hat 11.2.1-1) + #1 SMP x86_64 GNU/Linux GNU Fortran (GCC) 11.2.1 2021078 (Red Hat 11.2.1-1) + Fedora34 clang version 12.0.1 (Fedora 12.0.1-1.fc34) + (cmake and autotools) + + Linux 5.11.0-34-generic GNU gcc (GCC) 9.3.0-17ubuntu1 + #36-Ubuntu SMP x86_64 GNU/Linux GNU Fortran (GCC) 9.3.0-17ubuntu1 + Ubuntu 20.04 Ubuntu clang version 10.0.0-4 + (cmake and autotools) + + Linux 5.8.0-63-generic GNU gcc (GCC) 10.3.0-1ubuntu1 + #71-Ubuntu SMP x86_64 GNU/Linux GNU Fortran (GCC) 10.3.0-1ubuntu1 + Ubuntu20.10 Ubuntu clang version 11.0.0-2 + (cmake and autotools) + + Linux 5.3.18-22-default GNU gcc (SUSE Linux) 7.5.0 + #1 SMP x86_64 GNU/Linux GNU Fortran (SUSE Linux) 7.5.0 + SUSE15sp2 clang version 7.0.1 (tags/RELEASE_701/final 349238) + (cmake and autotools) + + Linux-4.14.0-115.21.2 spectrum-mpi/rolling-release + #1 SMP ppc64le GNU/Linux clang 8.0.1, 11.0.1 + (lassen) GCC 7.3.1 + XL 16.1.1.2 + (cmake) + + Linux-4.12.14-150.75-default cray-mpich/7.7.10 + #1 SMP x86_64 GNU/Linux GCC 7.3.0, 8.2.0 + (cori) Intel (R) Version 19.0.3.199 + (cmake) + + Linux-4.12.14-197.86-default cray-mpich/7.7.6 + # 1SMP x86_64 GNU/Linux GCC 7.3.0, 9.3.0, 10.2.0 + (mutrino) Intel (R) Version 17.0.4, 18.0.5, 19.1.3 + (cmake) + + Linux 3.10.0-1160.36.2.el7.ppc64 gcc (GCC) 4.8.5 20150623 (Red Hat 4.8.5-39) + #1 SMP ppc64be GNU/Linux g++ (GCC) 4.8.5 20150623 (Red Hat 4.8.5-39) + Power8 (echidna) GNU Fortran (GCC) 4.8.5 20150623 (Red Hat 4.8.5-39) + + Linux 3.10.0-1160.24.1.el7 GNU C (gcc), Fortran (gfortran), C++ (g++) + #1 SMP x86_64 GNU/Linux compilers: + Centos7 Version 4.8.5 20150623 (Red Hat 4.8.5-4) + (jelly/kituo/moohan) Version 4.9.3, Version 5.3.0, Version 6.3.0, + Version 7.2.0, Version 8.3.0, Version 9.1.0 + Intel(R) C (icc), C++ (icpc), Fortran (icc) + compilers: + Version 17.0.0.098 Build 20160721 + GNU C (gcc) and C++ (g++) 4.8.5 compilers + with NAG Fortran Compiler Release 6.1(Tozai) + Intel(R) C (icc) and C++ (icpc) 17.0.0.098 compilers + with NAG Fortran Compiler Release 6.1(Tozai) + MPICH 3.1.4 compiled with GCC 4.9.3 + MPICH 3.3 compiled with GCC 7.2.0 + OpenMPI 2.1.6 compiled with icc 18.0.1 + OpenMPI 3.1.3 and 4.0.0 compiled with GCC 7.2.0 + PGI C, Fortran, C++ for 64-bit target on + x86_64; + Version 19.10-0 + + Linux-3.10.0-1127.0.0.1chaos openmpi-4.0.0 + #1 SMP x86_64 GNU/Linux clang 6.0.0, 11.0.1 + (quartz) GCC 7.3.0, 8.1.0 + Intel 16.0.4, 18.0.2, 19.0.4 + + macOS Apple M1 11.6 Apple clang version 12.0.5 (clang-1205.0.22.11) + Darwin 20.6.0 arm64 gfortran GNU Fortran (Homebrew GCC 11.2.0) 11.1.0 + (macmini-m1) Intel icc/icpc/ifort version 2021.3.0 202106092021.3.0 20210609 + + macOS Big Sur 11.3.1 Apple clang version 12.0.5 (clang-1205.0.22.9) + Darwin 20.4.0 x86_64 gfortran GNU Fortran (Homebrew GCC 10.2.0_3) 10.2.0 + (bigsur-1) Intel icc/icpc/ifort version 2021.2.0 20210228 + + macOS High Sierra 10.13.6 Apple LLVM version 10.0.0 (clang-1000.10.44.4) + 64-bit gfortran GNU Fortran (GCC) 6.3.0 + (bear) Intel icc/icpc/ifort version 19.0.4.233 20190416 + + macOS Sierra 10.12.6 Apple LLVM version 9.0.0 (clang-900.39.2) + 64-bit gfortran GNU Fortran (GCC) 7.4.0 + (kite) Intel icc/icpc/ifort version 17.0.2 + + Mac OS X El Capitan 10.11.6 Apple clang version 7.3.0 from Xcode 7.3 + 64-bit gfortran GNU Fortran (GCC) 5.2.0 + (osx1011test) Intel icc/icpc/ifort version 16.0.2 + + + Linux 2.6.32-573.22.1.el6 GNU C (gcc), Fortran (gfortran), C++ (g++) + #1 SMP x86_64 GNU/Linux compilers: + Centos6 Version 4.4.7 20120313 + (platypus) Version 4.9.3, 5.3.0, 6.2.0 + MPICH 3.1.4 compiled with GCC 4.9.3 + PGI C, Fortran, C++ for 64-bit target on + x86_64; + Version 19.10-0 + + Windows 10 x64 Visual Studio 2015 w/ Intel C/C++/Fortran 18 (cmake) + Visual Studio 2017 w/ Intel C/C++/Fortran 19 (cmake) + Visual Studio 2019 w/ clang 12.0.0 + with MSVC-like command-line (C/C++ only - cmake) + Visual Studio 2019 w/ Intel Fortran 19 (cmake) + Visual Studio 2019 w/ MSMPI 10.1 (C only - cmake) + + +Known Problems +============== + Setting a variable-length dataset fill value will leak the memory allocated + for the p field of the hvl_t struct. A fix is in progress for this. + HDFFV-10840 + + CMake files do not behave correctly with paths containing spaces. + Do not use spaces in paths because the required escaping for handling spaces + results in very complex and fragile build files. + ADB - 2019/05/07 + + At present, metadata cache images may not be generated by parallel + applications. Parallel applications can read files with metadata cache + images, but since this is a collective operation, a deadlock is possible + if one or more processes do not participate. + + CPP ptable test fails on both VS2017 and VS2019 with Intel compiler, JIRA + issue: HDFFV-10628. This test will pass with VS2015 with Intel compiler. + + The subsetting option in ph5diff currently will fail and should be avoided. + The subsetting option works correctly in serial h5diff. + + Known problems in previous releases can be found in the HISTORY*.txt files + in the HDF5 source. Please report any new problems found to + help@hdfgroup.org. + + +CMake vs. Autotools installations +================================= +While both build systems produce similar results, there are differences. +Each system produces the same set of folders on linux (only CMake works +on standard Windows); bin, include, lib and share. Autotools places the +COPYING and RELEASE.txt file in the root folder, CMake places them in +the share folder. + +The bin folder contains the tools and the build scripts. Additionally, CMake +creates dynamic versions of the tools with the suffix "-shared". Autotools +installs one set of tools depending on the "--enable-shared" configuration +option. + build scripts + ------------- + Autotools: h5c++, h5cc, h5fc + CMake: h5c++, h5cc, h5hlc++, h5hlcc + +The include folder holds the header files and the fortran mod files. CMake +places the fortran mod files into separate shared and static subfolders, +while Autotools places one set of mod files into the include folder. Because +CMake produces a tools library, the header files for tools will appear in +the include folder. + +The lib folder contains the library files, and CMake adds the pkgconfig +subfolder with the hdf5*.pc files used by the bin/build scripts created by +the CMake build. CMake separates the C interface code from the fortran code by +creating C-stub libraries for each Fortran library. In addition, only CMake +installs the tools library. The names of the szip libraries are different +between the build systems. + +The share folder will have the most differences because CMake builds include +a number of CMake specific files for support of CMake's find_package and support +for the HDF5 Examples CMake project. + +The issues with the gif tool are: + HDFFV-10592 CVE-2018-17433 + HDFFV-10593 CVE-2018-17436 + HDFFV-11048 CVE-2020-10809 +These CVE issues have not yet been addressed and can be avoided by not building +the gif tool. Disable building the High-Level tools with these options: + autotools: --disable-hltools + cmake: HDF5_BUILD_HL_TOOLS=OFF diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 045d594..160675f 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -36,7 +36,7 @@ CONTENTS - New Features - Support for new platforms and languages -- Bug Fixes since HDF5-1.12.0 +- Bug Fixes since HDF5-1.13.0 - Platforms Tested - Known Problems - CMake vs. Autotools installations @@ -73,462 +73,6 @@ New Features (ADB - 2021/11/23) - - Added new configure option to support building parallel tools. - See Tools below (autotools - CMake): - --enable-parallel-tools HDF5_BUILD_PARALLEL_TOOLS - - (RAW - 2021/10/25) - - - Added new configure options to enable dimension scales APIs (H5DS*) to - use new object references with the native VOL connector (aka native HDF5 - library). New references are always used for non-native terminal VOL - connectors (e.g., DAOS). - - Autotools --enable-dimension-scales-with-new-ref - CMake HDF5_DIMENSION_SCALES_NEW_REF=ON - - (EIP - 2021/10/25, HDFFV-11180) - - - Refactored the utils folder. - - Added subfolder test and moved the 'swmr_check_compat_vfd.c file' - from test into utils/test. Deleted the duplicate swmr_check_compat_vfd.c - file in hl/tools/h5watch folder. Also fixed vfd check options. - - (ADB - 2021/10/18) - - - Changed autotools and CMake configurations to derive both - compilation warnings-as-errors and warnings-only-warn configurations - from the same files, 'config/*/*error*'. Removed redundant files - 'config/*/*noerror*'. - - (DCY - 2021/09/29) - - - Added new option to control the build of High-Level tools - that default ON/enabled. - - Add configure options (autotools - CMake): - --enable-hltools HDF5_BUILD_HL_TOOLS - - Disabling this option prevents building the gif tool which - contains the following CVEs: - HDFFV-10592 CVE-2018-17433 - HDFFV-10593 CVE-2018-17436 - HDFFV-11048 CVE-2020-10809 - - (ADB - 2021/09/16, HDFFV-11266) - - - Adds C++ Autotools configuration file for Intel - - * Checks for icpc as the compiler - * Sets std=c++11 - * Copies most non-warning flags from intel-flags - - (DER - 2021/06/02) - - - Adds C++ Autotools configuration file for PGI - - * Checks for pgc++ as the compiler name (was: pgCC) - * Sets -std=c++11 - * Other options basically match new C options (below) - - (DER - 2021/06/02) - - - Updates PGI C options - - * -Minform set to warn (was: inform) to suppress spurious messages - * Sets -gopt -O2 as debug options - * Sets -O4 as 'high optimization' option - * Sets -O0 as 'no optimization' option - * Removes specific settings for PGI 9 and 10 - - (DER - 2021/06/02) - - - A C++11-compliant compiler is now required to build the C++ wrappers - - CMAKE_CXX_STANDARD is now set to 11 when building with CMake and - -std=c++11 is added when building with clang/gcc via the Autotools. - - (DER - 2021/05/27) - - - CMake will now run the shell script tests in test/ by default - - The test directory includes several shell script tests that previously - were not run by CMake. These are now run by default. TEST_SHELL_SCRIPTS - has been set to ON and SH_PROGRAM has been set to bash (some test - scripts use bash-isms). Platforms without bash (e.g., Windows) will - ignore the script tests. - - (DER - 2021/05/23) - - - Removed unused HDF5_ENABLE_HSIZET option from CMake - - This has been unused for some time and has no effect. - - (DER - 2021/05/23) - - - CMake no longer builds the C++ library by default - - HDF5_BUILD_CPP_LIB now defaults to OFF, which is in line with the - Autotools build defaults. - - (DER - 2021/04/20) - - - Removal of pre-VS2015 work-arounds - - HDF5 now requires Visual Studio 2015 or greater, so old work-around - code and definitions have been removed, including: - - * - * snprintf and vsnprintf - * llround, llroundf, lround, lroundf, round, roundf - * strtoll and strtoull - * va_copy - * struct timespec - - (DER - 2021/03/22) - - - Add CMake variable HDF5_LIB_INFIX - - This infix is added to all library names after 'hdf5'. - e.g. the infix '_openmpi' results in the library name 'libhdf5_openmpi.so' - This name is used in packages on debian based systems. - (see https://packages.debian.org/jessie/amd64/libhdf5-openmpi-8/filelist) - - (barcode - 2021/03/22) - - - On macOS, Universal Binaries can now be built, allowing native execution on - both Intel and Apple Silicon (ARM) based Macs. - - To do so, set CMAKE_OSX_ARCHITECTURES="x86_64;arm64" - - (SAM - 2021/02/07, github-311) - - - Added a configure-time option to control certain compiler warnings - diagnostics - - A new configure-time option was added that allows some compiler warnings - diagnostics to have the default operation. This is mainly intended for - library developers and currently only works for gcc 10 and above. The - diagnostics flags apply to C, C++ and Fortran compilers and will appear - in "H5 C Flags", H5 C++ Flags" and H5 Fortran Flags, respectively. They - will NOT be exported to h5cc, etc. - - The default is OFF, which will disable the warnings URL and color attributes - for the warnings output. ON will not add the flags and allow default behavior. - - Autotools: --enable-diags - - CMake: HDF5_ENABLE_BUILD_DIAGS - - (ADB - 2021/02/05, HDFFV-11213) - - - CMake option to build the HDF filter plugins project as an external project - - The HDF filter plugins project is a collection of registered compression - filters that can be dynamically loaded when needed to access data stored - in a hdf5 file. This CMake-only option allows the plugins to be built and - distributed with the hdf5 library and tools. Like the options for szip and - zlib, either a tgz file or a git repository can be specified for the source. - - The option was refactored to use the CMake FetchContent process. This allows - more control over the filter targets, but required external project command - options to be moved to a CMake include file, HDF5PluginCache.cmake. Also - enabled the filter examples to be used as tests for operation of the - filter plugins. - - (ADB - 2020/12/10, OESS-98) - - - FreeBSD Autotools configuration now defaults to 'cc' and 'c++' compilers - - On FreeBSD, the autotools defaulted to 'gcc' as the C compiler and did - not process C++ options. Since FreeBSD 10, the default compiler has - been clang (via 'cc'). - - The default compilers have been set to 'cc' for C and 'c++' for C++, - which will pick up clang and clang++ respectively on FreeBSD 10+. - Additionally, clang options are now set correctly for both C and C++ - and g++ options will now be set if that compiler is being used (an - omission from the former functionality). - - (DER - 2020/11/28, HDFFV-11193) - - - Fixed POSIX problems when building w/ gcc on Solaris - - When building on Solaris using gcc, the POSIX symbols were not - being set correctly, which could lead to issues like clock_gettime() - not being found. - - The standard is now set to gnu99 when building with gcc on Solaris, - which allows POSIX things to be #defined and linked correctly. This - differs slightly from the gcc norm, where we set the standard to c99 - and manually set POSIX #define symbols. - - (DER - 2020/11/25, HDFFV-11191) - - - Added a configure-time option to consider certain compiler warnings - as errors - - A new configure-time option was added that converts some compiler warnings - to errors. This is mainly intended for library developers and currently - only works for gcc and clang. The warnings that are considered errors - will appear in the generated libhdf5.settings file. These warnings apply - to C and C++ code and will appear in "H5 C Flags" and H5 C++ Flags", - respectively. They will NOT be exported to h5cc, etc. - - The default is OFF. Building with this option may fail when compiling - on operating systems and with compiler versions not commonly used by - the library developers. Compilation may also fail when headers not - under the control of the library developers (e.g., mpi.h, hdfs.h) raise - warnings. - - Autotools: --enable-warnings-as-errors - - CMake: HDF5_ENABLE_WARNINGS_AS_ERRORS - - (DER - 2020/11/23, HDFFV-11189) - - - Autotools and CMake target added to produce doxygen generated documentation - - The default is OFF or disabled. - Autoconf option is '--enable-doxygen' - autotools make target is 'doxygen' and will build all doxygen targets - CMake configure option is 'HDF5_BUILD_DOC'. - CMake target is 'doxygen' for all available doxygen targets - CMake target is 'hdf5lib_doc' for the src subdirectory - - (ADB - 2020/11/03) - - - CMake option to use MSVC naming conventions with MinGW - - HDF5_MSVC_NAMING_CONVENTION option enable to use MSVC naming conventions - when using a MinGW toolchain - - (xan - 2020/10/30) - - - CMake option to statically link gcc libs with MinGW - - HDF5_MINGW_STATIC_GCC_LIBS allows to statically link libg/libstdc++ - with the MinGW toolchain - - (xan - 2020/10/30) - - - CMake option to build the HDF filter plugins project as an external project - - The HDF filter plugins project is a collection of registered compression - filters that can be dynamically loaded when needed to access data stored - in a hdf5 file. This CMake-only option allows the plugins to be built and - distributed with the hdf5 library and tools. Like the options for szip and - zlib, either a tgz file or a git repository can be specified for the source. - - The necessary options are (see the INSTALL_CMake.txt file): - HDF5_ENABLE_PLUGIN_SUPPORT - PLUGIN_TGZ_NAME or PLUGIN_GIT_URL - There are more options necessary for various filters and the plugin project - documents should be referenced. - - (ADB - 2020/09/27, OESS-98) - - - Added CMake option to format source files - - HDF5_ENABLE_FORMATTERS option will enable creation of targets using the - pattern - HDF5_*_SRC_FORMAT - where * corresponds to the source folder - or tool folder. All sources can be formatted by executing the format target; - make format - - (ADB - 2020/08/24) - - - Add file locking configure and CMake options - - HDF5 1.10.0 introduced a file locking scheme, primarily to help - enforce SWMR setup. Formerly, the only user-level control of the scheme - was via the HDF5_USE_FILE_LOCKING environment variable. - - This change introduces configure-time options that control whether - or not file locking will be used and whether or not the library - ignores errors when locking has been disabled on the file system - (useful on some HPC Lustre installations). - - In both the Autotools and CMake, the settings have the effect of changing - the default property list settings (see the H5Pset/get_file_locking() - entry, below). - - The yes/no/best-effort file locking configure setting has also been - added to the libhdf5.settings file. - - Autotools: - - An --enable-file-locking=(yes|no|best-effort) option has been added. - - yes: Use file locking. - no: Do not use file locking. - best-effort: Use file locking and ignore "disabled" errors. - - CMake: - - Two self-explanatory options have been added: - - HDF5_USE_FILE_LOCKING - HDF5_IGNORE_DISABLED_FILE_LOCKS - - Setting both of these to ON is the equivalent to the Autotools' - best-effort setting. - - NOTE: - The precedence order of the various file locking control mechanisms is: - - 1) HDF5_USE_FILE_LOCKING environment variable (highest) - - 2) H5Pset_file_locking() - - 3) configure/CMake options (which set the property list defaults) - - 4) library defaults (currently best-effort) - - (DER - 2020/07/30, HDFFV-11092) - - - CMake option to link the generated Fortran MOD files into the include - directory. - - The Fortran generation of MOD files by a Fortran compile can produce - different binary files between SHARED and STATIC compiles with different - compilers and/or different platforms. Note that it has been found that - different versions of Fortran compilers will produce incompatible MOD - files. Currently, CMake will locate these MOD files in subfolders of - the include directory and add that path to the Fortran library target - in the CMake config file, which can be used by the CMake find library - process. For other build systems using the binary from a CMake install, - a new CMake configuration can be used to copy the pre-chosen version - of the Fortran MOD files into the install include directory. - - The default will depend on the configuration of - BUILD_STATIC_LIBS and BUILD_SHARED_LIBS: - YES YES Default to SHARED - YES NO Default to STATIC - NO YES Default to SHARED - NO NO Default to SHARED - The defaults can be overridden by setting the config option - HDF5_INSTALL_MOD_FORTRAN to one of NO, SHARED, or STATIC - - (ADB - 2020/07/09, HDFFV-11116) - - - CMake option to use AEC (open source SZip) library instead of SZip - - The open source AEC library is a replacement library for SZip. In - order to use it for hdf5 the libaec CMake source was changed to add - "-fPIC" and exclude test files. Autotools does not build the - compression libraries within hdf5 builds. New option USE_LIBAEC is - required to compensate for the different files produced by AEC build. - - (ADB - 2020/04/22, OESS-65) - - - CMake ConfigureChecks.cmake file now uses CHECK_STRUCT_HAS_MEMBER - - Some handcrafted tests in HDFTests.c has been removed and the CMake - CHECK_STRUCT_HAS_MEMBER module has been used. - - (ADB - 2020/03/24, TRILAB-24) - - - Both build systems use same set of warnings flags - - GNU C, C++ and gfortran warnings flags were moved to files in a config - sub-folder named gnu-warnings. Flags that only are available for a specific - version of the compiler are in files named with that version. - Clang C warnings flags were moved to files in a config sub-folder - named clang-warnings. - Intel C, Fortran warnings flags were moved to files in a config sub-folder - named intel-warnings. - - There are flags in named "error-xxx" files with warnings that may - be promoted to errors. Some source files may still need fixes. - - There are also pairs of files named "developer-xxx" and "no-developer-xxx" - that are chosen by the CMake option:HDF5_ENABLE_DEV_WARNINGS or the - configure option:--enable-developer-warnings. - - In addition, CMake no longer applies these warnings for examples. - - (ADB - 2020/03/24, TRILAB-192) - - - Added test script for file size compare - - If CMake minimum version is at least 3.14, the fileCompareTest.cmake - script will compare file sizes. - - (ADB - 2020/02/24, HDFFV-11036) - - - Update CMake minimum version to 3.12 - - Updated CMake minimum version to 3.12 and added version checks - for Windows features. - - (ADB - 2020/02/05, TRILABS-142) - - - Fixed CMake include properties for Fortran libraries - - Corrected the library properties for Fortran to use the - correct path for the Fortran module files. - - (ADB - 2020/02/04, HDFFV-11012) - - - Added common warnings files for gnu and intel - - Added warnings files to use one common set of flags - during configure for both autotools and CMake build - systems. The initial implementation only affects a - general set of flags for gnu and intel compilers. - - (ADB - 2020/01/17) - - - Added new options to CMake for control of testing - - Added CMake options (default ON); - HDF5_TEST_SERIAL AND/OR HDF5_TEST_PARALLEL - combined with: - HDF5_TEST_TOOLS - HDF5_TEST_EXAMPLES - HDF5_TEST_SWMR - HDF5_TEST_FORTRAN - HDF5_TEST_CPP - HDF5_TEST_JAVA - - (ADB - 2020/01/15, HDFFV-11001) - - - Added Clang sanitizers to CMake for analyzer support if compiler is clang. - - Added CMake code and files to execute the Clang sanitizers if - HDF5_ENABLE_SANITIZERS is enabled and the USE_SANITIZER option - is set to one of the following: - Address - Memory - MemoryWithOrigins - Undefined - Thread - Leak - 'Address;Undefined' - - (ADB - 2019/12/12, TRILAB-135) - - - Update CMake for VS2019 support - - CMake added support for VS2019 in version 3.15. Changes to the CMake - generator setting required changes to scripts. Also updated version - references in CMake files as necessary. - - (ADB - 2019/11/18, HDFFV-10962) - - - Update CMake options to match new autotools options - - Add configure options (autotools - CMake): - enable-asserts HDF5_ENABLE_ASSERTS - enable-symbols HDF5_ENABLE_SYMBOLS - enable-profiling HDF5_ENABLE_PROFILING - enable-optimization HDF5_ENABLE_OPTIMIZATION - In addition NDEBUG is no longer forced defined and relies on the CMake - process. - - (ADB - 2019/10/07, HDFFV-100901, HDFFV-10637, TRILAB-97) - Library: -------- @@ -539,347 +83,6 @@ New Features (NAF - 2022/01/11) - - Adds new file driver-level memory copy operation for - "ctl" callback and updates compact dataset I/O routines - to utilize it - - When accessing an HDF5 file with a file driver that uses - memory allocated in special ways (e.g., without standard - library's `malloc`), a crash could be observed when HDF5 - tries to perform `memcpy` operations on such a memory - region. - - These changes add a new H5FD_FEAT_MEMMANAGE VFD feature - flag, which, if specified as supported by a VFD, will - inform HDF5 that the VFD either uses special memory - management routines or wishes to perform memory management - in a specific way. Therefore, this flag instructs HDF5 to - ask the file driver to perform memory management for - certain operations. - - These changes also introduce a new "ctl" callback - operation identified by the H5FD_CTL__MEM_COPY op code. - This operation simply asks a VFD to perform a memory copy. - The arguments to this operation are passed to the "ctl" - callback's "input" parameter as a pointer to a struct - defined as: - - struct H5FD_ctl_memcpy_args_t { - void * dstbuf; /**< Destination buffer */ - hsize_t dst_off; /**< Offset within destination buffer */ - const void *srcbuf; /**< Source buffer */ - hsize_t src_off; /**< Offset within source buffer */ - size_t len; /**< Length of data to copy from source buffer */ - } H5FD_ctl_memcpy_args_t; - - Further, HDF5's compact dataset I/O routines were - identified as a problematic area that could cause a crash - for VFDs that make use of special memory management. Those - I/O routines were therefore updated to make use of this new - "ctl" callback operation in order to ask the underlying - file driver to correctly handle memory copies. - - (JTH - 2021/09/28) - - - Adds new "ctl" callback to VFD H5FD_class_t structure - with the following prototype: - - herr_t (*ctl)(H5FD_t *file, uint64_t op_code, - uint64_t flags, const void *input, - void **output); - - This newly-added "ctl" callback allows Virtual File - Drivers to intercept and handle arbitrary operations - identified by an operation code. Its parameters are - as follows: - - `file` [in] - A pointer to the file to be operated on - `op_code` [in] - The operation code identifying the - operation to be performed - `flags` [in] - Flags governing the behavior of the - operation performed (see H5FDpublic.h - for a list of valid flags) - `input` [in] - A pointer to arguments passed to the - VFD performing the operation - `output` [out] - A pointer for the receiving VFD to - use for output from the operation - - (JRM - 2021/08/16) - - - Change how the release part of version, in major.minor.release is checked - for compatibility - - The HDF5 library uses a function, H5check_version, to check that - the version defined in the header files, which is used to compile an - application is compatible with the version codified in the library, which - the application loads at runtime. This previously required an exact match - or the library would print a warning, dump the build settings and then - abort or continue. An environment variable controlled the logic. - - Now the function first checks that the library release version, in - major.minor.release, is not older than the version in the headers. - Secondly, if the release version is different, it checks if either - the library version or the header version is in the exception list, in - which case the release part of version, in major.minor.release, must - be exact. An environment variable still controls the logic. - - (ADB - 2021/07/27) - - - gcc warning suppression macros were moved out of H5public.h - - The HDF5 library uses a set of macros to suppress warnings on gcc. - These warnings were originally located in H5public.h so that the - multi VFD (which only uses public headers) could also make use of them - but internal macros should not be publicly exposed like this. - - These macros have now been moved to H5private.h. Pending future multi - VFD refactoring, the macros have been duplicated in H5FDmulti.c to - suppress the format string warnings there. - - (DER - 2021/06/03) - - - H5Gcreate1() now rejects size_hint parameters larger than UINT32_MAX - - The size_hint value is ultimately stored in a uint32_t struct field, - so specifying a value larger than this on a 64-bit machine can cause - undefined behavior including crashing the system. - - The documentation for this API call was also incorrect, stating that - passing a negative value would cause the library to use a default - value. Instead, passing a "negative" value actually passes a very large - value, which is probably not what the user intends and can cause - crashes on 64-bit systems. - - The Doxygen documentation has been updated and passing values larger - than UINT32_MAX for size_hint will now produce a normal HDF5 error. - - (DER - 2021/04/29, HDFFV-11241) - - - - H5Pset_fapl_log() no longer crashes when passed an invalid fapl ID - - When passed an invalid fapl ID, H5Pset_fapl_log() would usually - segfault when attempting to free an uninitialized pointer in the error - handling code. This behavior is more common in release builds or - when the memory sanitization checks were not selected as a build - option. - - The pointer is now correctly initialized and the API call now - produces a normal HDF5 error when fed an invalid fapl ID. - - (DER - 2021/04/28, HDFFV-11240) - - - Fixes a segfault when H5Pset_mdc_log_options() is called multiple times - - The call incorrectly attempts to free an internal copy of the previous - log location string, which causes a segfault. This only happens - when the call is invoked multiple times on the same property list. - On the first call to a given fapl, the log location is set to NULL so - the segfault does not occur. - - The string is now handled properly and the segfault no longer occurs. - - (DER - 2021/04/27, HDFFV-11239) - - - HSYS_GOTO_ERROR now emits the results of GetLastError() on Windows - - HSYS_GOTO_ERROR is an internal macro that is used to produce error - messages when system calls fail. These strings include errno and the - the associated strerror() value, which are not particularly useful - when a Win32 API call fails. - - On Windows, this macro has been updated to include the result of - GetLastError(). When a system call fails on Windows, usually only - one of errno and GetLastError() will be useful, however we emit both - for the user to parse. The Windows error message is not emitted as - it would be awkward to free the FormatMessage() buffer given the - existing HDF5 error framework. Users will have to look up the error - codes in MSDN. - - The format string on Windows has been changed from: - - "%s, errno = %d, error message = '%s'" - - to: - - "%s, errno = %d, error message = '%s', Win32 GetLastError() = %"PRIu32"" - - for those inclined to parse it for error values. - - (DER - 2021/03/21) - - - File locking now works on Windows - - Since version 1.10.0, the HDF5 library has used a file locking scheme - to help enforce one reader at a time accessing an HDF5 file, which can - be helpful when setting up readers and writers to use the single- - writer/multiple-readers (SWMR) access pattern. - - In the past, this was only functional on POSIX systems where flock() or - fcntl() were present. Windows used a no-op stub that always succeeded. - - HDF5 now uses LockFileEx() and UnlockFileEx() to lock the file using the - same scheme as POSIX systems. We lock the entire file when we set up the - locks (by passing DWORDMAX as both size parameters to LockFileEx()). - - (DER - 2021/03/19, HDFFV-10191) - - - H5Epush_ret() now requires a trailing semicolon - - H5Epush_ret() is a function-like macro that has been changed to - contain a `do {} while(0)` loop. Consequently, a trailing semicolon - is now required to end the `while` statement. Previously, a trailing - semi would work, but was not mandatory. This change was made to allow - clang-format to correctly format the source code. - - (SAM - 2021/03/03) - - - Improved performance of H5Sget_select_elem_pointlist - - Modified library to cache the point after the last block of points - retrieved by H5Sget_select_elem_pointlist, so a subsequent call to the - same function to retrieve the next block of points from the list can - proceed immediately without needing to iterate over the point list. - - (NAF - 2021/01/19) - - - Replaced H5E_ATOM with H5E_ID in H5Epubgen.h - - The term "atom" is archaic and not in line with current HDF5 library - terminology, which uses "ID" instead. "Atom" has mostly been purged - from the library internals and this change removes H5E_ATOM from - the H5Epubgen.h (exposed via H5Epublic.h) and replaces it with - H5E_ID. - - (DER - 2020/11/24, HDFFV-11190) - - - Add a new public function H5Ssel_iter_reset - - This function resets a dataspace selection iterator back to an - initial state so that it may be used for iteration once more. - This can be useful when needing to iterate over a selection - multiple times without having to repeatedly create/destroy - a selection iterator for that dataspace selection. - - (JTH - 2020/09/18) - - - Remove HDFS VFD stubs - - The original implementation of the HDFS VFD included non-functional - versions of the following public API calls when the HDFS VFD is - not built as a part of the HDF5 library: - - * H5FD_hdfs_init() - * H5Pget_fapl_hdfs() - * H5Pset_fapl_hdfs() - - They will remain present in HDF5 1.10 and HDF5 1.12 releases - for binary compatibility purposes but have been removed as of 1.14.0. - - Note that this has nothing to do with the real HDFS VFD API calls - that are fully functional when the HDFS VFD is configured and built. - - We simply changed: - - #ifdef LIBHDFS - - #else - - #endif - - to: - - #ifdef LIBHDFS - - #endif - - Which is how the other optional VFDs are handled. - - (DER - 2020/08/27) - - - Add Mirror VFD - - Use TCP/IP sockets to perform write-only (W/O) file I/O on a remote - machine. Must be used in conjunction with the Splitter VFD. - - (JOS - 2020/03/13, TBD) - - - Add Splitter VFD - - Maintain separate R/W and W/O channels for "concurrent" file writes - to two files using a single HDF5 file handle. - - (JOS - 2020/03/13, TBD) - - - Refactored public exposure of haddr_t type in favor of "object tokens" - - To better accommodate HDF5 VOL connectors where "object addresses in a file" - may not make much sense, the following changes were made to the library: - - * Introduced new H5O_token_t "object token" type, which represents a - unique and permanent identifier for referencing an HDF5 object within - a container; these "object tokens" are meant to replace object addresses. - Along with the new type, a new H5Oopen_by_token API call was introduced - to open an object by a token, similar to how object addresses were - previously used with H5Oopen_by_addr. - - * Introduced new H5Lget_info2, H5Lget_info_by_idx2, H5Literate2, H5Literate_by_name2, - H5Lvisit2 and H5Lvisit_by_name2 API calls, along with their associated H5L_info2_t - struct and H5L_iterate2_t callback function, which work with the newly-introduced - object tokens, instead of object addresses. The original functions have been - renamed to version 1 functions and are deprecated in favor of the new version 2 - functions. The H5L_info_t and H5L_iterate_t types have been renamed to version 1 - types and are now deprecated in favor of their version 2 counterparts. For each of - the functions and types, compatibility macros take place of the original symbols. - - * Introduced new H5Oget_info3, H5Oget_info_by_name3, H5Oget_info_by_idx3, - H5Ovisit3 and H5Ovisit_by_name3 API calls, along with their associated H5O_info2_t - struct and H5O_iterate2_t callback function, which work with the newly-introduced - object tokens, instead of object addresses. The version 2 functions are now - deprecated in favor of the version 3 functions. The H5O_info_t and H5O_iterate_t - types have been renamed to version 1 types and are now deprecated in favor of their - version 2 counterparts. For each, compatibility macros take place of the original - symbols. - - * Introduced new H5Oget_native_info, H5Oget_native_info_by_name and - H5Oget_native_info_by_idx API calls, along with their associated H5O_native_info_t - struct, which are used to retrieve the native HDF5 file format-specific information - about an object. This information (such as object header info and B-tree/heap info) - has been removed from the new H5O_info2_t struct so that the more generic - H5Oget_info(_by_name/_by_idx)3 routines will not try to retrieve it for non-native - VOL connectors. - - * Added new H5Otoken_cmp, H5Otoken_to_str and H5Otoken_from_str routines to compare - two object tokens, convert an object token into a nicely-readable string format and - to convert an object token string back into a real object token, respectively. - - (DER, QAK, JTH - 2020/01/16) - - - Add new public function H5Sselect_adjust. - - This function shifts a dataspace selection by a specified logical offset - within the dataspace extent. This can be useful for VOL developers to - implement chunked datasets. - - (NAF - 2019/11/18) - - - Add new public function H5Sselect_project_intersection. - - This function computes the intersection between two dataspace selections - and projects that intersection into a third selection. This can be useful - for VOL developers to implement chunked or virtual datasets. - - (NAF - 2019/11/13, ID-148) - - - Add new public function H5VLget_file_type. - - This function returns a datatype equivalent to the supplied datatype but - with the location set to be in the file. This datatype can then be used - with H5Tconvert to convert data between file and in-memory representation. - This function is intended for use only by VOL connector developers. - - (NAF - 2019/11/08, ID-127) - Parallel Library: ----------------- @@ -926,205 +129,49 @@ New Features Fortran Library: ---------------- - - H5Fget_name_f fixed to handle correctly trailing whitespaces and - newly allocated buffers. - - (MSB - 2021/08/30, github-826,972) - - - Add wrappers for H5Pset/get_file_locking() API calls - - h5pget_file_locking_f() - h5pset_file_locking_f() - - See the configure option discussion for HDFFV-11092 (above) for more - information on the file locking feature and how it's controlled. + - - (DER - 2020/07/30, HDFFV-11092) C++ Library: ------------ - - Add wrappers for H5Pset/get_file_locking() API calls - - FileAccPropList::setFileLocking() - FileAccPropList::getFileLocking() - - See the configure option discussion for HDFFV-11092 (above) for more - information on the file locking feature and how it's controlled. - - (DER - 2020/07/30, HDFFV-11092) + - Java Library: ------------- - - Replaced HDF5AtomException with HDF5IdException - - Since H5E_ATOM changed to H5E_ID in the C library, the Java exception - that wraps the error category was also renamed. Its functionality - remains unchanged aside from the name. - - (See also the HDFFV-11190 note in the C library section) - - (DER - 2020/11/24, HDFFV-11190) - - - Added new H5S functions. - - H5Sselect_copy, H5Sselect_shape_same, H5Sselect_adjust, - H5Sselect_intersect_block, H5Sselect_project_intersection, - H5Scombine_hyperslab, H5Smodify_select, H5Scombine_select - wrapper functions added. - - (ADB - 2020/10/27, HDFFV-10868) - - - Add wrappers for H5Pset/get_file_locking() API calls - - H5Pset_file_locking() - H5Pget_use_file_locking() - H5Pget_ignore_disabled_file_locking() - - Unlike the C++ and Fortran wrappers, there are separate getters for the - two file locking settings, each of which returns a boolean value. - - See the configure option discussion for HDFFV-11092 (above) for more - information on the file locking feature and how it's controlled. - - (DER - 2020/07/30, HDFFV-11092) - - - Added ability to test java library with VOLs. - - Created a new CMake script that combines the java and vol test scripts. - - (ADB - 2020/02/03, HDFFV-10996) - - - Tests fail for non-English locales. - - In the JUnit tests with a non-English locale, only the part before - the decimal comma is replaced by XXXX and this leads to a comparison - error. Changed the regex for the Time substitution. - - (ADB - 2020/01/09, HDFFV-10995) + - Tools: ------ - - Added a new (unix ONLY) parallel meta tool 'h5dwalk', which utilizes the - mpifileutils (https://hpc.github.io/mpifileutils) open source utility - library to enable parallel execution of other HDF5 tools. - This approach can greatly enhance the serial hdf5 tool performance over large - collections of files by utilizing MPI parallelism to distribute an application - load over many independent MPI ranks and files. - - An introduction to the mpifileutils library and initial 'User Guide' for - the new 'h5dwalk" tool can be found at: - https://github.com/HDFGroup/hdf5doc/tree/master/RFCs/HDF5/tools/parallel_tools - - (RAW - 2021/10/25) - - - Refactored the perform tools and removed depends on test library. - - Moved the perf and h5perf tools from tools/test/perform to - tools/src/h5perf so that they can be installed. This required - that the test library dependency be removed by copying the - needed functions from h5test.c. - The standalone scripts and other perform tools remain in the - tools/test/perform folder. - - (ADB - 2021/08/10) - - - Removed partial long exceptions - - Some of the tools accepted shortened versions of the long options - (ex: --datas instead of --dataset). These were implemented inconsistently, - are difficult to maintian, and occasionally block useful long option - names. These partial long options have been removed from all the tools. - - (DER - 2021/08/03) - - - h5repack added help text for user-defined filters. - - Added help text line that states the valid values of the filter flag - for user-defined filters; - filter_flag: 1 is OPTIONAL or 0 is MANDATORY - - (ADB - 2021/01/14, HDFFV-11099) - - - Added h5delete tool - - Deleting HDF5 storage when using the VOL can be tricky when the VOL - does not create files. The h5delete tool is a simple wrapper around - the H5Fdelete() API call that uses the VOL specified in the - HDF5_VOL_CONNECTOR environment variable to delete a "file". If - the call to H5Fdelete() fails, the tool will attempt to use - the POSIX remove(3) call to remove the file. - - Note that the HDF5 library does currently have support for - H5Fdelete() in the native VOL connector. - - (DER - 2020/12/16) - - - h5repack added options to control how external links are handled. - - Currently h5repack preserves external links and cannot copy and merge - data from the external files. Two options, merge and prune, were added to - control how to merge data from an external link into the resulting file. - --merge Follow external soft link recursively and merge data. - --prune Do not follow external soft links and remove link. - --merge --prune Follow external link, merge data and remove dangling link. - - (ADB - 2020/08/05, HDFFV-9984) - - - h5repack was fixed to repack the reference attributes properly. - The code line that checks if the update of reference inside a compound - datatype is misplaced outside the code block loop that carries out the - check. In consequence, the next attribute that is not the reference - type was repacked again as the reference type and caused the failure of - repacking. The fix is to move the corresponding code line to the correct - code block. - - (KY -2020/02/07, HDFFV-11014) - - - h5diff was updated to use the new reference APIs. - - h5diff uses the new reference APIs to compare references. - Attribute references can also be compared. - - (ADB - 2019/12/19, HDFFV-10980) - - - h5dump and h5ls were updated to use the new reference APIs. - - The tools library now use the new reference APIs to inspect a - file. Also the DDL spec was updated to reflect the format - changes produced with the new APIs. The export API and support - functions in the JNI were updated to match. - - (ADB - 2019/12/06, HDFFV-10876 and HDFFV-10877) + - High-Level APIs: ---------------- - - added set/get for unsigned long long attributes - - The attribute writing high-level API has been expanded to include - public set/get functions for ULL attributes, analogously to the - existing set/get for other types. + - - (AF - 2021/09/08) C Packet Table API: ------------------- - + Internal header file: --------------------- - + Documentation: -------------- - + Support for new platforms, languages and compilers ================================================== - + Bug Fixes since HDF5-1.12.0 release =================================== Library @@ -1163,14 +210,6 @@ Bug Fixes since HDF5-1.12.0 release (ADB - 2022/02/24, #1438) - - Fixed an H5Pget_filter_by_id1/2() assert w/ out of range filter IDs - - Both H5Pget_filter_by_id1 and 2 did not range check the filter ID, which - could trip as assert in debug versions of the library. The library now - returns a normal HDF5 error when the filter ID is out of range. - - (DER - 2021/11/23, HDFFV-11286) - - Unified handling of collective metadata reads to correctly fix old bugs Due to MPI-related issues occurring in HDF5 from mismanagement of the @@ -1204,440 +243,20 @@ Bug Fixes since HDF5-1.12.0 release (JTH - 2021/11/09) - - Fixed an issue with collective metadata reads being permanently disabled - after a dataset chunk lookup operation. This would usually cause a - mismatched MPI_Bcast and MPI_ERR_TRUNCATE issue in the library for - simple cases of H5Dcreate() -> H5Dwrite() -> H5Dcreate(). - - (JTH - 2021/11/08, HDFFV-11090) - - - Fixed cross platform incompatibility of references within variable length - types - - Reference types within variable length types previously could not be - read on a platform with different endianness from where they were - written. Fixed so cross platform portability is restored. - - (NAF - 2021/09/30) - - - Detection of simple data transform function "x" - - In the case of the simple data transform function "x" the (parallel) - library recognizes this is the same as not applying this data transform - function. This improves the I/O performance. In the case of the parallel - library, it also avoids breaking to independent I/O, which makes it - possible to apply a filter when writing or reading data to or from - the HDF5 file. - - (JWSB - 2021/09/13) - - - Fixed an invalid read and memory leak when parsing corrupt file space - info messages - - When the corrupt file from CVE-2020-10810 was parsed by the library, - the code that imports the version 0 file space info object header - message to the version 1 struct could read past the buffer read from - the disk, causing an invalid memory read. Not catching this error would - cause downstream errors that eventually resulted in a previously - allocated buffer to be unfreed when the library shut down. In builds - where the free lists are in use, this could result in an infinite loop - and SIGABRT when the library shuts down. - - We now track the buffer size and raise an error on attempts to read - past the end of it. - - (DER - 2021/08/12, HDFFV-11053) - - - - Fixed CVE-2018-14460 - - The tool h5repack produced a segfault when the rank in dataspace - message was corrupted, causing invalid read while decoding the - dimension sizes. - - The problem was fixed by ensuring that decoding the dimension sizes - and max values will not go beyond the end of the buffer. - - (BMR - 2021/05/12, HDFFV-11223) - - - Fixed CVE-2018-11206 - - The tool h5dump produced a segfault when the size of a fill value - message was corrupted and caused a buffer overflow. - - The problem was fixed by verifying the fill value's size - against the buffer size before attempting to access the buffer. - - (BMR - 2021/03/15, HDFFV-10480) - - - Fixed CVE-2018-14033 (same issue as CVE-2020-10811) - - The tool h5dump produced a segfault when the storage size message - was corrupted and caused a buffer overflow. - - The problem was fixed by verifying the storage size against the - buffer size before attempting to access the buffer. - - (BMR - 2021/03/15, HDFFV-11159/HDFFV-11049) - - - Remove underscores on header file guards - - Header file guards used a variety of underscores at the beginning of the define. - - Removed all leading (some trailing) underscores from header file guards. - - (ADB - 2021/03/03, #361) - - - Fixed a segmentation fault - - A segmentation fault occurred with a Mathworks corrupted file. - - A detection of accessing a null pointer was added to prevent the problem. - - (BMR - 2021/02/19, HDFFV-11150) - - - Fixed issue with MPI communicator and info object not being - copied into new FAPL retrieved from H5F_get_access_plist - - Added logic to copy the MPI communicator and info object into - the output FAPL. MPI communicator is retrieved from the VFD, while - the MPI info object is retrieved from the file's original FAPL. - - (JTH - 2021/02/15, HDFFV-11109) - - - Fixed problems with vlens and refs inside compound using - H5VLget_file_type() - - Modified library to properly ref count H5VL_object_t structs and only - consider file vlen and reference types to be equal if their files are - the same. - - (NAF - 2021/01/22) - - - Fixed CVE-2018-17432 - - The tool h5repack produced a segfault on a corrupted file which had - invalid rank for scalar or NULL datatype. - - The problem was fixed by modifying the dataspace encode and decode - functions to detect and report invalid rank. h5repack now fails - with an error message for the corrupted file. - - (BMR - 2020/10/26, HDFFV-10590) - - - Creation of dataset with optional filter - - When the combination of type, space, etc doesn't work for filter - and the filter is optional, it was supposed to be skipped but it was - not skipped and the creation failed. - - Allowed the creation of the dataset in such a situation. - - (BMR - 2020/08/13, HDFFV-10933) - - - Explicitly declared dlopen to use RTLD_LOCAL - - dlopen documentation states that if neither RTLD_GLOBAL nor - RTLD_LOCAL are specified, then the default behavior is unspecified. - The default on linux is usually RTLD_LOCAL while macos will default - to RTLD_GLOBAL. - - (ADB - 2020/08/12, HDFFV-11127) - - - H5Sset_extent_none() sets the dataspace class to H5S_NO_CLASS which - causes asserts/errors when passed to other dataspace API calls. - - H5S_NO_CLASS is an internal class value that should not have been - exposed via a public API call. - - In debug builds of the library, this can cause assert() function to - trip. In non-debug builds, it will produce normal library errors. - - The new library behavior is for H5Sset_extent_none() to convert - the dataspace into one of type H5S_NULL, which is better handled - by the library and easier for developers to reason about. - - (DER - 2020/07/27, HDFFV-11027) - - - Fixed issues CVE-2018-13870 and CVE-2018-13869 - - When a buffer overflow occurred because a name length was corrupted - and became very large, h5dump crashed on memory access violation. - - A check for reading pass the end of the buffer was added to multiple - locations to prevent the crashes and h5dump now simply fails with an - error message when this error condition occurs. - - (BMR - 2020/07/22, HDFFV-11120 and HDFFV-11121) - - - Fixed the segmentation fault when reading attributes with multiple threads - - It was reported that the reading of attributes with variable length string - datatype will crash with segmentation fault particularly when the number of - threads is high (>16 threads). The problem was due to the file pointer that - was set in the variable length string datatype for the attribute. That file - pointer was already closed when the attribute was accessed. - - The problem was fixed by setting the file pointer to the current opened file pointer - when the attribute was accessed. Similar patch up was done before when reading - dataset with variable length string datatype. - - (VC - 2020/07/13, HDFFV-11080) - - - Fixed CVE-2020-10810 - - The tool h5clear produced a segfault during an error recovery in - the superblock decoding. An internal pointer was reset to prevent - further accessing when it is not assigned with a value. - - (BMR - 2020/06/29, HDFFV-11053) - - - Fixed CVE-2018-17435 - - The tool h52gif produced a segfault when the size of an attribute - message was corrupted and caused a buffer overflow. - - The problem was fixed by verifying the attribute message's size - against the buffer size before accessing the buffer. h52gif was - also fixed to display the failure instead of silently exiting - after the segfault was eliminated. - - (BMR - 2020/06/19, HDFFV-10591) - Java Library ------------ - - JNI utility function does not handle new references. - - The JNI utility function for converting reference data to string did - not use the new APIs. In addition to fixing that function, added new - java tests for using the new APIs. - - (ADB - 2021/02/16, HDFFV-11212) - - - The H5FArray.java class, in which virtually the entire execution time - is spent using the HDFNativeData method that converts from an array - of bytes to an array of the destination Java type. - - 1. Convert the entire byte array into a 1-d array of the desired type, - rather than performing 1 conversion per row; - 2. Use the Java Arrays method copyOfRange to grab the section of the - array from (1) that is desired to be inserted into the destination array. - - (PGT,ADB - 2020/12/13, HDFFV-10865) - - - Added ability to test java library with VOLs. - - Created a new CMake script that combines the java and vol test scripts. - - (ADB - 2020/02/03, HDFFV-10996) - - - Tests fail for non-English locales. - - In the JUnit tests with a non-English locale, only the part before - the decimal comma is replaced by XXXX and this leads to a comparison - error. Changed the regex for the Time substitution. - - (ADB - 2020/01/09, HDFFV-10995) + - Configuration ------------- - - Corrected path searched by CMake find_package command - - The install path for cmake find_package files had been changed to use - "share/cmake" - for all platforms. However the trailing "hdf5" directory was not removed. - This "hdf5" additional directory has been removed. - - (ADB - 2021/09/27) - - - Corrected pkg-config compile script - - It was discovered that the position of the "$@" argument for the command - in the compile script may fail on some platforms and configurations. The - position of the "$@"command argument was moved before the pkg-config sub command. - - (ADB - 2021/08/30) - - - Fixed CMake C++ compiler flags - - A recent refactoring of the C++ configure files accidentally removed the - file that executed the enable_language command for C++ needed by the - HDFCXXCompilerFlags.cmake file. Also updated the intel warnings files, - including adding support for windows platforms. - - (ADB - 2021/08/10) - - - Better support for libaec (open-source Szip library) in CMake - - Implemented better support for libaec 1.0.5 (or later) library. This version - of libaec contains improvements for better integration with HDF5. Furthermore, - the variable USE_LIBAEC_STATIC has been introduced to allow to make use of - static version of libaec library. Use libaec_DIR or libaec_ROOT to set - the location in which libaec can be found. - - Be aware, the Szip library of libaec 1.0.4 depends on another library within - libaec library. This dependency is not specified in the current CMake - configuration which means that one can not use the static Szip library of - libaec 1.0.4 when building HDF5. This has been resolved in libaec 1.0.5. - - (JWSB - 2021/06/22) - - - Refactor CMake configure for Fortran - - The Fortran configure tests for KINDs reused a single output file that was - read to form the Integer and Real Kinds defines. However, if config was run - more then once, the CMake completed variable prevented the tests from executing - again and the last value saved in the file was used to create the define. - Creating separate files for each KIND solved the issue. - - In addition the test for H5_PAC_C_MAX_REAL_PRECISION was not pulling in - defines for proper operation and did not define H5_PAC_C_MAX_REAL_PRECISION - correctly for a zero value. This was fixed by supplying the required defines. - In addition it was moved from the Fortran specific HDF5UseFortran.camke file - to the C centric ConfigureChecks.cmake file. - - (ADB - 2021/06/03) - - - Move emscripten flag to compile flags - - The emscripten flag, -O0, was removed from target_link_libraries command - to the correct target_compile_options command. - - (ADB - 2021/04/26 HDFFV-11083) - - - Remove arbitrary warning flag groups from CMake builds - - The arbitrary groups were created to reduce the quantity of warnings being - reported that overwhelmed testing report systems. Considerable work has - been accomplished to reduce the warning count and these arbitrary groups - are no longer needed. - Also the default for all warnings, HDF5_ENABLE_ALL_WARNINGS, is now ON. - - Visual Studio warnings C4100, C4706, and C4127 have been moved to - developer warnings, HDF5_ENABLE_DEV_WARNINGS, and are disabled for normal builds. - - (ADB - 2021/03/22, HDFFV-11228) - - - Reclassify CMake messages, to allow new modes and --log-level option - - CMake message commands have a mode argument. By default, STATUS mode - was chosen for any non-error message. CMake version 3.15 added additional - modes, NOTICE, VERBOSE, DEBUG and TRACE. All message commands with a mode - of STATUS were reviewed and most were reclassified as VERBOSE. The new - mode was protected by a check for a CMake version of at least 3.15. If CMake - version 3.17 or above is used, the user can use the command line option - of "--log-level" to further restrict which message commands are displayed. - - (ADB - 2021/01/11, HDFFV-11144) - - - Fixes Autotools determination of the stat struct having an st_blocks field - - A missing parenthesis in an autoconf macro prevented building the test - code used to determine if the stat struct contains the st_blocks field. - Now that the test functions correctly, the H5_HAVE_STAT_ST_BLOCKS #define - found in H5pubconf.h will be defined correctly on both the Autotools and - CMake. This #define is only used in the tests and does not affect the - HDF5 C library. - - (DER - 2021/01/07, HDFFV-11201) - - - Add missing ENV variable line to hdfoptions.cmake file - - Using the build options to use system SZIP/ZLIB libraries need to also - specify the library root directory. Setting the {library}_ROOT ENV - variable was added to the hdfoptions.cmake file. - - (ADB - 2020/10/19 HDFFV-11108) + - Tools ----- - - Changed how h5dump and h5ls identify long double. - - Long double support is not consistent across platforms. Tools will always - identify long double as 128-bit [little/big]-endian float nn-bit precision. - New test file created for datasets with attributes for float, double and - long double. In addition any unknown integer or float datatype will now - also show the number of bits for precision. - These files are also used in the java tests. - - (ADB - 2021/03/24, HDFFV-11229,HDFFV-11113) - - - Fixed tools argument parsing. - - Tools parsing used the length of the option from the long array to match - the option from the command line. This incorrectly matched a shorter long - name option that happened to be a subset of another long option. - Changed to match whole names. - - (ADB - 2021/01/19, HDFFV-11106) - - - The tools library was updated by standardizing the error stack process. - - General sequence is: - h5tools_setprogname(PROGRAMNAME); - h5tools_setstatus(EXIT_SUCCESS); - h5tools_init(); - ... process the command-line (check for error-stack enable) ... - h5tools_error_report(); - ... (do work) ... - h5diff_exit(ret); - - (ADB - 2020/07/20, HDFFV-11066) - - - h5diff fixed a command line parsing error. - - h5diff would ignore the argument to -d (delta) if it is smaller than DBL_EPSILON. - The macro H5_DBL_ABS_EQUAL was removed and a direct value comparison was used. - - (ADB - 2020/07/20, HDFFV-10897) - - - h5diff added a command line option to ignore attributes. - - h5diff would ignore all objects with a supplied path if the exclude-path argument is used. - Adding the exclude-attribute argument will only exclude attributes, with the supplied path, - from comparison. - - (ADB - 2020/07/20, HDFFV-5935) - - - h5diff added another level to the verbose argument to print filenames. - - Added verbose level 3 that is level 2 plus the filenames. The levels are: - 0 : Identical to '-v' or '--verbose' - 1 : All level 0 information plus one-line attribute status summary - 2 : All level 1 information plus extended attribute status report - 3 : All level 2 information plus file names - - (ADB - 2020/07/20, HDFFV-1005) - - - h5repack was fixed to repack the reference attributes properly. - - The code line that checks if the update of reference inside a compound - datatype is misplaced outside the code block loop that carries out the - check. In consequence, the next attribute that is not the reference - type was repacked again as the reference type and caused the failure of - repacking. The fix is to move the corresponding code line to the correct - code block. - - (KY -2020/02/10, HDFFV-11014) - - - h5diff was updated to use the new reference APIs. - - h5diff uses the new reference APIs to compare references. - Attribute references can also be compared. - - (ADB - 2019/12/19, HDFFV-10980) - - - h5dump and h5ls were updated to use the new reference APIs. - - The tools library now use the new reference APIs to inspect a - file. Also the DDL spec was updated to reflect the format - changes produced with the new APIs. The export API and support - functions in the JNI were updated to match. - - (ADB - 2019/12/06, HDFFV-10876 and HDFFV-10877) + - Performance @@ -1647,18 +266,12 @@ Bug Fixes since HDF5-1.12.0 release Fortran API ----------- - - Corrected INTERFACE INTENT(IN) to INTENT(OUT) for buf_size in h5fget_file_image_f. - - (MSB - 2020/02/18, HDFFV-11029) + - High-Level Library ------------------ - - Fixed HL_test_packet, test for packet table vlen of vlen. - - Incorrect length assignment. - - (ADB - 2021/10/14) + - Fortran High-Level APIs @@ -1678,23 +291,7 @@ Bug Fixes since HDF5-1.12.0 release C++ APIs -------- - - Added DataSet::operator= - - Some compilers complain if the copy constructor is given explicitly - but the assignment operator is implicitly set to default. - - (2021/05/19) - - - Testing - ------- - - Stopped java/test/junit.sh.in installing libs for testing under ${prefix} - - Lib files needed are now copied to a subdirectory in the java/test - directory, and on Macs the loader path for libhdf5.xxxs.so is changed - in the temporary copy of libhdf5_java.dylib. - - (LRK, 2020/07/02, HDFFV-11063) + - Platforms Tested -- cgit v0.12 From ba032bb28bfff3b8d330b026cbd01e6e44be7578 Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Thu, 3 Mar 2022 09:46:18 -0600 Subject: Disable memory alloc sanity checks by default for Autotools debug builds (#1468) --- configure.ac | 17 +++++++++-------- release_docs/RELEASE.txt | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/configure.ac b/configure.ac index 0ad2228..e426632 100644 --- a/configure.ac +++ b/configure.ac @@ -2710,9 +2710,11 @@ AC_ARG_ENABLE([memory-alloc-sanity-check], [Enable this option to turn on internal memory allocation sanity checking. This could cause more memory use and somewhat slower allocation. - This option is orthogonal to the - --enable-using-memchecker option. - [default=yes if debug build, otherwise no] + This option may also cause issues with HDF5 + filter plugins, so should not be enabled if + filters are to be used. This option is orthogonal + to the --enable-using-memchecker option. + [default=no] ])], [MEMORYALLOCSANITYCHECK=$enableval]) @@ -2722,11 +2724,10 @@ AC_SUBST([MEMORYALLOCSANITYCHECK]) ## Set default if test "X-$MEMORYALLOCSANITYCHECK" = X- ; then - if test "X-$BUILD_MODE" = "X-debug" ; then - MEMORYALLOCSANITYCHECK=yes - else - MEMORYALLOCSANITYCHECK=no - fi +# Should consider enabling this option by default for +# 'developer' builds if that build mode is added in +# the future + MEMORYALLOCSANITYCHECK=no fi case "X-$MEMORYALLOCSANITYCHECK" in diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 160675f..8804d52 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -47,6 +47,25 @@ New Features Configuration: ------------- + - HDF5 memory allocation sanity checking is now off by default for + Autotools debug builds + + HDF5 can be configured to perform sanity checking on internal memory + allocations by adding heap canaries to these allocations. However, + enabling this option can cause issues with external filter plugins + when working with (reallocating/freeing/allocating and passing back) + buffers. + + Previously, this option was off by default for all CMake build types, + but only off by default for non-debug Autotools builds. Since debug + is the default build mode for HDF5 when built from source with + Autotools, this can result in surprising segfaults that don't occur + when an application is built against a release version of HDF5. + Therefore, this option is now off by default for all build types + across both CMake and Autotools. + + (JTH - 2022/03/01) + - CPack will now generate RPM/DEB packages. Enabled the RPM and DEB CPack generators on linux. In addition to -- cgit v0.12 From 44d992684080f52e31c46c9c88b2ed16b1572e5f Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Mon, 7 Mar 2022 08:31:58 -0600 Subject: Update HISTORY-1_13.txt and clean RELEASE.txt (#1471) * Committing clang-format changes * Spelling of preceed was corrected to proceed, but should have been corrected to precede. * Correct spelling correction of 'preceed' incorrectly to 'proceed'. It should be 'precede'. * Update version to 1.13.2-1 after 1.13.1 release. Add Makefile.in to MANIFEST for addition of utils/tools and h5dwalk. * Update VERS_RELEASE_EXCEPTIONS with new incompatible release version. * Add HDF5 1.13.1 RELEASE.txt to HISTORY-1_13.txt contents to HISTORY-1_13.txt. Clean entries from RELEASE.txt. Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- release_docs/HISTORY-1_13.txt | 391 ++++++++++++++++++++++++++++++++++++++++++ release_docs/RELEASE.txt | 144 +--------------- 2 files changed, 396 insertions(+), 139 deletions(-) diff --git a/release_docs/HISTORY-1_13.txt b/release_docs/HISTORY-1_13.txt index 06be75b..91320a1 100644 --- a/release_docs/HISTORY-1_13.txt +++ b/release_docs/HISTORY-1_13.txt @@ -5,6 +5,7 @@ This file contains development history of the HDF5 1.13 releases from the develop branch 01. Release Information for hdf5-1.13.0 +02. Release Information for hdf5-1.13.1 [Search on the string '%%%%' for section breaks of each release.] @@ -1756,3 +1757,393 @@ These CVE issues have not yet been addressed and can be avoided by not building the gif tool. Disable building the High-Level tools with these options: autotools: --disable-hltools cmake: HDF5_BUILD_HL_TOOLS=OFF + + +%%%%1.13.1%%%% + +HDF5 version 1.13.1 released on 2022-03-02 +================================================================================ + + +INTRODUCTION +============ + +This document describes the differences between this release and the previous +HDF5 release. It contains information on the platforms tested and known +problems in this release. For more details check the HISTORY*.txt files in the +HDF5 source. + +Note that documentation in the links below will be updated at the time of each +final release. + +Links to HDF5 documentation can be found on The HDF5 web page: + + https://portal.hdfgroup.org/display/HDF5/HDF5 + +The official HDF5 releases can be obtained from: + + https://www.hdfgroup.org/downloads/hdf5/ + +Changes from Release to Release and New Features in the HDF5-1.13.x release series +can be found at: + + https://portal.hdfgroup.org/display/HDF5/HDF5+Application+Developer%27s+Guide + +If you have any questions or comments, please send them to the HDF Help Desk: + + help@hdfgroup.org + + +CONTENTS +======== + +- New Features +- Support for new platforms and languages +- Bug Fixes since HDF5-1.13.0 +- Platforms Tested +- Known Problems +- CMake vs. Autotools installations + + +New Features +============ + + Configuration: + ------------- + - CPack will now generate RPM/DEB packages. + + Enabled the RPM and DEB CPack generators on linux. In addition to + generating STGZ and TGZ packages, CPack will try to package the + library for RPM and DEB packages. This is the initial attempt and + may change as issues are resolved. + + (ADB - 2022/01/27) + + - Added new option to the h5cc scripts produced by CMake. + + Add -showconfig option to h5cc scripts to cat the + libhdf5.settings file to the standard output. + + (ADB - 2022/01/25) + + - CMake will now run the PowerShell script tests in test/ by default + on Windows. + + The test directory includes several shell script tests that previously + were not run by CMake on Windows. These are now run by default. + If TEST_SHELL_SCRIPTS is ON and PWSH is found, the PowerShell scripts + will execute. Similar to the bash scripts on unix platforms. + + (ADB - 2021/11/23) + + + Library: + -------- + - Add a new public function, H5ESget_requests() + + This function allows the user to retrieve request pointers from an event + set. It is intended for use primarily by VOL plugin developers. + + (NAF - 2022/01/11) + + + Parallel Library: + ----------------- + - Several improvements to parallel compression feature, including: + + * Improved support for collective I/O (for both writes and reads) + + * Significant reduction of memory usage for the feature as a whole + + * Reduction of copying of application data buffers passed to H5Dwrite + + * Addition of support for incremental file space allocation for filtered + datasets created in parallel. Incremental file space allocation is the + default for these types of datasets (early file space allocation is + also still supported), while early file space allocation is still the + default (and only supported at allocation time) for unfiltered datasets + created in parallel. Incremental file space allocation should help with + parallel HDF5 applications that wish to use fill values on filtered + datasets, but would typically avoid doing so since dataset creation in + parallel would often take an excessive amount of time. Since these + datasets previously used early file space allocation, HDF5 would + allocate space for and write fill values to every chunk in the dataset + at creation time, leading to noticeable overhead. Instead, with + incremental file space allocation, allocation of file space for chunks + and writing of fill values to those chunks will be delayed until each + individual chunk is initially written to. + + * Addition of support for HDF5's "don't filter partial edge chunks" flag + (https://portal.hdfgroup.org/display/HDF5/H5P_SET_CHUNK_OPTS) + + * Addition of proper support for HDF5 fill values with the feature + + * Addition of 'H5_HAVE_PARALLEL_FILTERED_WRITES' macro to H5pubconf.h + so HDF5 applications can determine at compile-time whether the feature + is available + + * Addition of simple examples (ph5_filtered_writes.c and + ph5_filtered_writes_no_sel.c) under examples directory to demonstrate + usage of the feature + + * Improved coverage of regression testing for the feature + + (JTH - 2022/2/23) + + +Support for new platforms, languages and compilers +================================================== + - None + + +Bug Fixes since HDF5-1.13.0 release +=================================== + Library + ------- + - Fixed a metadata cache bug when resizing a pinned/protected cache entry + + When resizing a pinned/protected cache entry, the metadata + cache code previously would wait until after resizing the + entry to attempt to log the newly-dirtied entry. This + caused H5C_resize_entry to mark the entry as dirty and made + H5AC_resize_entry think that it didn't need to add the + newly-dirtied entry to the dirty entries skiplist. + + Thus, a subsequent H5AC__log_moved_entry would think it + needed to allocate a new entry for insertion into the dirty + entry skip list, since the entry didGn't exist on that list. + This caused an assertion failure, as the code to allocate a + new entry assumes that the entry is not dirty. + + (JRM - 2022/02/28) + + - Issue #1436 identified a problem with the H5_VERS_RELEASE check in the + H5check_version function. + + Investigating the original fix, #812, we discovered some inconsistencies + with a new block added to check H5_VERS_RELEASE for incompatibilities. + This new block was not using the new warning text dealing with the + H5_VERS_RELEASE check and would cause the warning to be duplicated. + + By removing the H5_VERS_RELEASE argument in the first check for + H5_VERS_MAJOR and H5_VERS_MINOR, the second check would only check + the H5_VERS_RELEASE for incompatible release versions. This adheres + to the statement that except for the develop branch, all release versions + in a major.minor maintenance branch should be compatible. The prerequisite + is that an application will not use any APIs not present in all release versions. + + (ADB - 2022/02/24, #1438) + + - Unified handling of collective metadata reads to correctly fix old bugs + + Due to MPI-related issues occurring in HDF5 from mismanagement of the + status of collective metadata reads, they were forced to be disabled + during chunked dataset raw data I/O in the HDF5 1.10.5 release. This + wouldn't generally have affected application performance because HDF5 + already disables collective metadata reads during chunk lookup, since + it is generally unlikely that the same chunks will be read by all MPI + ranks in the I/O operation. However, this was only a partial solution + that wasn't granular enough. + + This change now unifies the handling of the file-global flag and the + API context-level flag for collective metadata reads in order to + simplify querying of the true status of collective metadata reads. Thus, + collective metadata reads are once again enabled for chunked dataset + raw data I/O, but manually controlled at places where some processing + occurs on MPI rank 0 only and would cause issues when collective + metadata reads are enabled. + + (JTH - 2021/11/16, HDFFV-10501/HDFFV-10562) + + - Fixed several potential MPI deadlocks in library failure conditions + + In the parallel library, there were several places where MPI rank 0 + could end up skipping past collective MPI operations when some failure + occurs in rank 0-specific processing. This would lead to deadlocks + where rank 0 completes an operation while other ranks wait in the + collective operation. These places have been rewritten to have rank 0 + push an error and try to cleanup after the failure, then continue to + participate in the collective operation to the best of its ability. + + (JTH - 2021/11/09) + + +Platforms Tested +=================== + + Linux 5.13.14-200.fc34 GNU gcc (GCC) 11.2.1 2021078 (Red Hat 11.2.1-1) + #1 SMP x86_64 GNU/Linux GNU Fortran (GCC) 11.2.1 2021078 (Red Hat 11.2.1-1) + Fedora34 clang version 12.0.1 (Fedora 12.0.1-1.fc34) + (cmake and autotools) + + Linux 5.11.0-34-generic GNU gcc (GCC) 9.3.0-17ubuntu1 + #36-Ubuntu SMP x86_64 GNU/Linux GNU Fortran (GCC) 9.3.0-17ubuntu1 + Ubuntu 20.04 Ubuntu clang version 10.0.0-4 + (cmake and autotools) + + Linux 5.8.0-63-generic GNU gcc (GCC) 10.3.0-1ubuntu1 + #71-Ubuntu SMP x86_64 GNU/Linux GNU Fortran (GCC) 10.3.0-1ubuntu1 + Ubuntu20.10 Ubuntu clang version 11.0.0-2 + (cmake and autotools) + + Linux 5.3.18-22-default GNU gcc (SUSE Linux) 7.5.0 + #1 SMP x86_64 GNU/Linux GNU Fortran (SUSE Linux) 7.5.0 + SUSE15sp2 clang version 7.0.1 (tags/RELEASE_701/final 349238) + (cmake and autotools) + + Linux-4.14.0-115.21.2 spectrum-mpi/rolling-release + #1 SMP ppc64le GNU/Linux clang 8.0.1, 11.0.1 + (lassen) GCC 7.3.1 + XL 16.1.1.2 + (cmake) + + Linux-3.10.0-1160.49.1 openmpi-intel/4.1 + #1 SMP x86_64 GNU/Linux Intel(R) Version 18.0.5, 19.1.2 + (chama) (cmake) + + Linux-4.12.14-150.75-default cray-mpich/7.7.10 + #1 SMP x86_64 GNU/Linux GCC 7.3.0, 8.2.0 + (cori) Intel (R) Version 19.0.3.199 + (cmake) + + Linux-4.12.14-197.86-default cray-mpich/7.7.6 + # 1SMP x86_64 GNU/Linux GCC 7.3.0, 9.3.0, 10.2.0 + (mutrino) Intel (R) Version 17.0.4, 18.0.5, 19.1.3 + (cmake) + + Linux 3.10.0-1160.36.2.el7.ppc64 gcc (GCC) 4.8.5 20150623 (Red Hat 4.8.5-39) + #1 SMP ppc64be GNU/Linux g++ (GCC) 4.8.5 20150623 (Red Hat 4.8.5-39) + Power8 (echidna) GNU Fortran (GCC) 4.8.5 20150623 (Red Hat 4.8.5-39) + + Linux 3.10.0-1160.24.1.el7 GNU C (gcc), Fortran (gfortran), C++ (g++) + #1 SMP x86_64 GNU/Linux compilers: + Centos7 Version 4.8.5 20150623 (Red Hat 4.8.5-4) + (jelly/kituo/moohan) Version 4.9.3, Version 5.3.0, Version 6.3.0, + Version 7.2.0, Version 8.3.0, Version 9.1.0 + Intel(R) C (icc), C++ (icpc), Fortran (icc) + compilers: + Version 17.0.0.098 Build 20160721 + GNU C (gcc) and C++ (g++) 4.8.5 compilers + with NAG Fortran Compiler Release 6.1(Tozai) + Intel(R) C (icc) and C++ (icpc) 17.0.0.098 compilers + with NAG Fortran Compiler Release 6.1(Tozai) + MPICH 3.1.4 compiled with GCC 4.9.3 + MPICH 3.3 compiled with GCC 7.2.0 + OpenMPI 2.1.6 compiled with icc 18.0.1 + OpenMPI 3.1.3 and 4.0.0 compiled with GCC 7.2.0 + PGI C, Fortran, C++ for 64-bit target on + x86_64; + Version 19.10-0 + + Linux-3.10.0-1127.0.0.1chaos openmpi-4.0.0 + #1 SMP x86_64 GNU/Linux clang 6.0.0, 11.0.1 + (quartz) GCC 7.3.0, 8.1.0 + Intel 16.0.4, 18.0.2, 19.0.4 + + macOS Apple M1 11.6 Apple clang version 12.0.5 (clang-1205.0.22.11) + Darwin 20.6.0 arm64 gfortran GNU Fortran (Homebrew GCC 11.2.0) 11.1.0 + (macmini-m1) Intel icc/icpc/ifort version 2021.3.0 202106092021.3.0 20210609 + + macOS Big Sur 11.3.1 Apple clang version 12.0.5 (clang-1205.0.22.9) + Darwin 20.4.0 x86_64 gfortran GNU Fortran (Homebrew GCC 10.2.0_3) 10.2.0 + (bigsur-1) Intel icc/icpc/ifort version 2021.2.0 20210228 + + macOS High Sierra 10.13.6 Apple LLVM version 10.0.0 (clang-1000.10.44.4) + 64-bit gfortran GNU Fortran (GCC) 6.3.0 + (bear) Intel icc/icpc/ifort version 19.0.4.233 20190416 + + macOS Sierra 10.12.6 Apple LLVM version 9.0.0 (clang-900.39.2) + 64-bit gfortran GNU Fortran (GCC) 7.4.0 + (kite) Intel icc/icpc/ifort version 17.0.2 + + Mac OS X El Capitan 10.11.6 Apple clang version 7.3.0 from Xcode 7.3 + 64-bit gfortran GNU Fortran (GCC) 5.2.0 + (osx1011test) Intel icc/icpc/ifort version 16.0.2 + + + Linux 2.6.32-573.22.1.el6 GNU C (gcc), Fortran (gfortran), C++ (g++) + #1 SMP x86_64 GNU/Linux compilers: + Centos6 Version 4.4.7 20120313 + (platypus) Version 4.9.3, 5.3.0, 6.2.0 + MPICH 3.1.4 compiled with GCC 4.9.3 + PGI C, Fortran, C++ for 64-bit target on + x86_64; + Version 19.10-0 + + Windows 10 x64 Visual Studio 2015 w/ Intel C/C++/Fortran 18 (cmake) + Visual Studio 2017 w/ Intel C/C++/Fortran 19 (cmake) + Visual Studio 2019 w/ clang 12.0.0 + with MSVC-like command-line (C/C++ only - cmake) + Visual Studio 2019 w/ Intel Fortran 19 (cmake) + Visual Studio 2019 w/ MSMPI 10.1 (C only - cmake) + + +Known Problems +============== + Setting a variable-length dataset fill value will leak the memory allocated + for the p field of the hvl_t struct. A fix is in progress for this. + HDFFV-10840 + + CMake files do not behave correctly with paths containing spaces. + Do not use spaces in paths because the required escaping for handling spaces + results in very complex and fragile build files. + ADB - 2019/05/07 + + At present, metadata cache images may not be generated by parallel + applications. Parallel applications can read files with metadata cache + images, but since this is a collective operation, a deadlock is possible + if one or more processes do not participate. + + CPP ptable test fails on both VS2017 and VS2019 with Intel compiler, JIRA + issue: HDFFV-10628. This test will pass with VS2015 with Intel compiler. + + The subsetting option in ph5diff currently will fail and should be avoided. + The subsetting option works correctly in serial h5diff. + + Known problems in previous releases can be found in the HISTORY*.txt files + in the HDF5 source. Please report any new problems found to + help@hdfgroup.org. + + +CMake vs. Autotools installations +================================= +While both build systems produce similar results, there are differences. +Each system produces the same set of folders on linux (only CMake works +on standard Windows); bin, include, lib and share. Autotools places the +COPYING and RELEASE.txt file in the root folder, CMake places them in +the share folder. + +The bin folder contains the tools and the build scripts. Additionally, CMake +creates dynamic versions of the tools with the suffix "-shared". Autotools +installs one set of tools depending on the "--enable-shared" configuration +option. + build scripts + ------------- + Autotools: h5c++, h5cc, h5fc + CMake: h5c++, h5cc, h5hlc++, h5hlcc + +The include folder holds the header files and the fortran mod files. CMake +places the fortran mod files into separate shared and static subfolders, +while Autotools places one set of mod files into the include folder. Because +CMake produces a tools library, the header files for tools will appear in +the include folder. + +The lib folder contains the library files, and CMake adds the pkgconfig +subfolder with the hdf5*.pc files used by the bin/build scripts created by +the CMake build. CMake separates the C interface code from the fortran code by +creating C-stub libraries for each Fortran library. In addition, only CMake +installs the tools library. The names of the szip libraries are different +between the build systems. + +The share folder will have the most differences because CMake builds include +a number of CMake specific files for support of CMake's find_package and support +for the HDF5 Examples CMake project. + +The issues with the gif tool are: + HDFFV-10592 CVE-2018-17433 + HDFFV-10593 CVE-2018-17436 + HDFFV-11048 CVE-2020-10809 +These CVE issues have not yet been addressed and can be avoided by not building +the gif tool. Disable building the High-Level tools with these options: + autotools: --disable-hltools + cmake: HDF5_BUILD_HL_TOOLS=OFF diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 8804d52..5e488be 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -36,7 +36,7 @@ CONTENTS - New Features - Support for new platforms and languages -- Bug Fixes since HDF5-1.13.0 +- Bug Fixes since HDF5-1.13.1 - Platforms Tested - Known Problems - CMake vs. Autotools installations @@ -66,85 +66,16 @@ New Features (JTH - 2022/03/01) - - CPack will now generate RPM/DEB packages. - - Enabled the RPM and DEB CPack generators on linux. In addition to - generating STGZ and TGZ packages, CPack will try to package the - library for RPM and DEB packages. This is the initial attempt and - may change as issues are resolved. - - (ADB - 2022/01/27) - - - Added new option to the h5cc scripts produced by CMake. - - Add -showconfig option to h5cc scripts that cat the - libhdf5-settings to the standard output. - - (ADB - 2022/01/25) - - - CMake will now run the PowerShell script tests in test/ by default - on Windows. - - The test directory includes several shell script tests that previously - were not run by CMake on Windows. These are now run by default. - If TEST_SHELL_SCRIPTS is ON and PWSH is found, the PowerShell scripts - will execute. Similar to the bash scripts on unix platforms. - - (ADB - 2021/11/23) - Library: -------- - - Add a new public function, H5ESget_requests() - - This function allows the user to retrieve request pointers from an event - set. It is intended for use primarily by VOL plug in developers. - - (NAF - 2022/01/11) + - Parallel Library: ----------------- - - Several improvements to parallel compression feature, including: - - * Improved support for collective I/O (for both writes and reads) - - * Significant reduction of memory usage for the feature as a whole - - * Reduction of copying of application data buffers passed to H5Dwrite - - * Addition of support for incremental file space allocation for filtered - datasets created in parallel. Incremental file space allocation is the - default for these types of datasets (early file space allocation is - also still supported), while early file space allocation is still the - default (and only supported allocation time) for unfiltered datasets - created in parallel. Incremental file space allocation should help with - parallel HDF5 applications that wish to use fill values on filtered - datasets, but would typically avoid doing so since dataset creation in - parallel would often take an excessive amount of time. Since these - datasets previously used early file space allocation, HDF5 would - allocate space for and write fill values to every chunk in the dataset - at creation time, leading to noticeable overhead. Instead, with - incremental file space allocation, allocation of file space for chunks - and writing of fill values to those chunks will be delayed until each - individual chunk is initially written to. - - * Addition of support for HDF5's "don't filter partial edge chunks" flag - (https://portal.hdfgroup.org/display/HDF5/H5P_SET_CHUNK_OPTS) - - * Addition of proper support for HDF5 fill values with the feature - - * Addition of 'H5_HAVE_PARALLEL_FILTERED_WRITES' macro to H5pubconf.h - so HDF5 applications can determine at compile-time whether the feature - is available - - * Addition of simple examples (ph5_filtered_writes.c and - ph5_filtered_writes_no_sel.c) under examples directory to demonstrate - usage of the feature - - * Improved coverage of regression testing for the feature + - - (JTH - 2022/2/23) Fortran Library: ---------------- @@ -191,76 +122,11 @@ Support for new platforms, languages and compilers - -Bug Fixes since HDF5-1.12.0 release +Bug Fixes since HDF5-1.13.1 release =================================== Library ------- - - Fixed a metadata cache bug when resizing a pinned/protected cache entry - - When resizing a pinned/protected cache entry, the metadata - cache code previously would wait until after resizing the - entry to attempt to log the newly-dirtied entry. This would - cause H5C_resize_entry to mark the entry as dirty and make - H5AC_resize_entry think that it doesn't need to add the - newly-dirtied entry to the dirty entries skiplist. - - Thus, a subsequent H5AC__log_moved_entry would think it - needs to allocate a new entry for insertion into the dirty - entry skip list, since the entry doesn't exist on that list. - This causes an assertion failure, as the code to allocate a - new entry assumes that the entry is not dirty. - - (JRM - 2022/02/28) - - - Issue #1436 identified a problem with the H5_VERS_RELEASE check in the - H5check_version function. - - Investigating the original fix, #812, we discovered some inconsistencies - with a new block added to check H5_VERS_RELEASE for incompatibilities. - This new block was not using the new warning text dealing with the - H5_VERS_RELEASE check and would cause the warning to be duplicated. - - By removing the H5_VERS_RELEASE argument in the first check for - H5_VERS_MAJOR and H5_VERS_MINOR, the second check would only check - the H5_VERS_RELEASE for incompatible release versions. This adheres - to the statement that except for the develop branch, all release versions - in a major.minor maintenance branch should be compatible. The prerequisite - is that an application will not use any APIs not present in all release versions. - - (ADB - 2022/02/24, #1438) - - - Unified handling of collective metadata reads to correctly fix old bugs - - Due to MPI-related issues occurring in HDF5 from mismanagement of the - status of collective metadata reads, they were forced to be disabled - during chunked dataset raw data I/O in the HDF5 1.10.5 release. This - wouldn't generally have affected application performance because HDF5 - already disables collective metadata reads during chunk lookup, since - it is generally unlikely that the same chunks will be read by all MPI - ranks in the I/O operation. However, this was only a partial solution - that wasn't granular enough. - - This change now unifies the handling of the file-global flag and the - API context-level flag for collective metadata reads in order to - simplify querying of the true status of collective metadata reads. Thus, - collective metadata reads are once again enabled for chunked dataset - raw data I/O, but manually controlled at places where some processing - occurs on MPI rank 0 only and would cause issues when collective - metadata reads are enabled. - - (JTH - 2021/11/16, HDFFV-10501/HDFFV-10562) - - - Fixed several potential MPI deadlocks in library failure conditions - - In the parallel library, there were several places where MPI rank 0 - could end up skipping past collective MPI operations when some failure - occurs in rank 0-specific processing. This would lead to deadlocks - where rank 0 completes an operation while other ranks wait in the - collective operation. These places have been rewritten to have rank 0 - push an error and try to cleanup after the failure, then continue to - participate in the collective operation to the best of its ability. - - (JTH - 2021/11/09) + - Java Library -- cgit v0.12 From 16f482de8f6f9a7447bf1d73702377b994b58aff Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Mon, 7 Mar 2022 08:32:45 -0600 Subject: Remove addition of non-standalone use of H5_ATTR_FORMAT (#1470) --- tools/src/h5perf/pio_perf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/src/h5perf/pio_perf.c b/tools/src/h5perf/pio_perf.c index 7e12bed..c233684 100644 --- a/tools/src/h5perf/pio_perf.c +++ b/tools/src/h5perf/pio_perf.c @@ -199,7 +199,7 @@ static int destroy_comm_world(void); static void output_results(const struct options *options, const char *name, minmax *table, int table_size, off_t data_size); static void output_times(const struct options *options, const char *name, minmax *table, int table_size); -static void output_report(const char *fmt, ...) H5_ATTR_FORMAT(printf, 1, 2); +static void output_report(const char *fmt, ...); static void print_indent(register int indent); static void usage(const char *prog); static void report_parameters(struct options *opts); -- cgit v0.12 From d5ddf61803f30e87bdfa629339fabf6fe1eb4a3c Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Wed, 9 Mar 2022 12:07:54 -0800 Subject: Minor tweaks from VFD SWMR normalization (#1474) --- MANIFEST | 2 +- src/H5C.c | 2 + src/H5Cepoch.c | 31 +++---- src/H5Cpkg.h | 226 +++++++++++++++++++++++----------------------- src/H5ES.c | 2 +- src/H5FD.c | 5 +- src/H5Ocopy.c | 1 - src/H5Tcommit.c | 2 +- src/H5VLcallback.c | 2 +- src/H5VLnative.h | 6 +- test/swmr_common.c | 5 +- test/swmr_sparse_reader.c | 2 +- 12 files changed, 143 insertions(+), 143 deletions(-) diff --git a/MANIFEST b/MANIFEST index bc967be..4d11d7f 100644 --- a/MANIFEST +++ b/MANIFEST @@ -3775,8 +3775,8 @@ ./tools/test/perform/CMakeTests.cmake ./utils/CMakeLists.txt -./utils/test/CMakeLists.txt ./utils/mirror_vfd/CMakeLists.txt +./utils/test/CMakeLists.txt # CMake-specific User Scripts ./config/cmake/CTestScript.cmake diff --git a/src/H5C.c b/src/H5C.c index 3868204..fa46ff2 100644 --- a/src/H5C.c +++ b/src/H5C.c @@ -1414,6 +1414,7 @@ H5C_insert_entry(H5F_t *f, const H5C_class_t *type, haddr_t addr, void *thing, u entry_ptr->serialization_count = 0; #endif /* NDEBUG */ + /* initialize tag list fields */ entry_ptr->tl_next = NULL; entry_ptr->tl_prev = NULL; entry_ptr->tag_info = NULL; @@ -7416,6 +7417,7 @@ H5C__load_entry(H5F_t *f, entry->serialization_count = 0; #endif /* NDEBUG */ + /* initialize tag list fields */ entry->tl_next = NULL; entry->tl_prev = NULL; entry->tag_info = NULL; diff --git a/src/H5Cepoch.c b/src/H5Cepoch.c index 3434fed..8655881 100644 --- a/src/H5Cepoch.c +++ b/src/H5Cepoch.c @@ -78,22 +78,21 @@ static herr_t H5C__epoch_marker_fsf_size(const void H5_ATTR_UNUSED *thing, /* Local Variables */ /*******************/ -const H5AC_class_t H5AC_EPOCH_MARKER[1] = {{ - /* id = */ H5AC_EPOCH_MARKER_ID, - /* name = */ "epoch marker", - /* mem_type = */ H5FD_MEM_DEFAULT, /* value doesn't matter */ - /* flags = */ H5AC__CLASS_NO_FLAGS_SET, - /* get_initial_load_size = */ H5C__epoch_marker_get_initial_load_size, - /* get_final_load_size = */ H5C__epoch_marker_get_final_load_size, - /* verify_chksum = */ H5C__epoch_marker_verify_chksum, - /* deserialize = */ H5C__epoch_marker_deserialize, - /* image_len = */ H5C__epoch_marker_image_len, - /* pre_serialize = */ H5C__epoch_marker_pre_serialize, - /* serialize = */ H5C__epoch_marker_serialize, - /* notify = */ H5C__epoch_marker_notify, - /* free_icr = */ H5C__epoch_marker_free_icr, - /* fsf_size = */ H5C__epoch_marker_fsf_size, -}}; +const H5AC_class_t H5AC_EPOCH_MARKER[1] = { + {/* id = */ H5AC_EPOCH_MARKER_ID, + /* name = */ "epoch marker", + /* mem_type = */ H5FD_MEM_DEFAULT, /* value doesn't matter */ + /* flags = */ H5AC__CLASS_NO_FLAGS_SET, + /* get_initial_load_size = */ H5C__epoch_marker_get_initial_load_size, + /* get_final_load_size = */ H5C__epoch_marker_get_final_load_size, + /* verify_chksum = */ H5C__epoch_marker_verify_chksum, + /* deserialize = */ H5C__epoch_marker_deserialize, + /* image_len = */ H5C__epoch_marker_image_len, + /* pre_serialize = */ H5C__epoch_marker_pre_serialize, + /* serialize = */ H5C__epoch_marker_serialize, + /* notify = */ H5C__epoch_marker_notify, + /* free_icr = */ H5C__epoch_marker_free_icr, + /* fsf_size = */ H5C__epoch_marker_fsf_size}}; /*************************************************************************** * Class functions for H5C__EPOCH_MAKER_TYPE: diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h index 30b86b9..61c3afc 100644 --- a/src/H5Cpkg.h +++ b/src/H5Cpkg.h @@ -1011,7 +1011,7 @@ if ( ( (cache_ptr) == NULL ) || \ ( H5C__HASH_FCN((entry_ptr)->addr) >= H5C__HASH_TABLE_LEN ) || \ ( (cache_ptr)->index_size != \ ((cache_ptr)->clean_index_size + \ - (cache_ptr)->dirty_index_size) ) || \ + (cache_ptr)->dirty_index_size) ) || \ ( (cache_ptr)->index_size < ((cache_ptr)->clean_index_size) ) || \ ( (cache_ptr)->index_size < ((cache_ptr)->dirty_index_size) ) || \ ( (entry_ptr)->ring <= H5C_RING_UNDEFINED ) || \ @@ -1034,7 +1034,7 @@ if ( ( (cache_ptr) == NULL ) || \ ( (cache_ptr)->magic != H5C__H5C_T_MAGIC ) || \ ( (cache_ptr)->index_size != \ ((cache_ptr)->clean_index_size + \ - (cache_ptr)->dirty_index_size) ) || \ + (cache_ptr)->dirty_index_size) ) || \ ( (cache_ptr)->index_size < ((cache_ptr)->clean_index_size) ) || \ ( (cache_ptr)->index_size < ((cache_ptr)->dirty_index_size) ) || \ ( (cache_ptr)->index_ring_len[(entry_ptr)->ring] == 0 ) || \ @@ -1071,7 +1071,7 @@ if ( ( (cache_ptr) == NULL ) || \ ( (entry_ptr)->ht_prev != NULL ) ) || \ ( (cache_ptr)->index_size != \ ((cache_ptr)->clean_index_size + \ - (cache_ptr)->dirty_index_size) ) || \ + (cache_ptr)->dirty_index_size) ) || \ ( (cache_ptr)->index_size < ((cache_ptr)->clean_index_size) ) || \ ( (cache_ptr)->index_size < ((cache_ptr)->dirty_index_size) ) || \ ( (entry_ptr)->ring <= H5C_RING_UNDEFINED ) || \ @@ -1102,7 +1102,7 @@ if ( ( (cache_ptr) == NULL ) || \ ( (entry_ptr)->ht_prev != NULL ) || \ ( (cache_ptr)->index_size != \ ((cache_ptr)->clean_index_size + \ - (cache_ptr)->dirty_index_size) ) || \ + (cache_ptr)->dirty_index_size) ) || \ ( (cache_ptr)->index_size < ((cache_ptr)->clean_index_size) ) || \ ( (cache_ptr)->index_size < ((cache_ptr)->dirty_index_size) ) || \ ( (cache_ptr)->index_ring_len[(entry_ptr)->ring] > \ @@ -1161,7 +1161,7 @@ if ( ( (cache_ptr) == NULL ) || \ } #define H5C__PRE_HT_ENTRY_SIZE_CHANGE_SC(cache_ptr, old_size, new_size, \ - entry_ptr, was_clean) \ + entry_ptr, was_clean) \ if ( ( (cache_ptr) == NULL ) || \ ( (cache_ptr)->index_len <= 0 ) || \ ( (cache_ptr)->index_size <= 0 ) || \ @@ -1175,9 +1175,9 @@ if ( ( (cache_ptr) == NULL ) || \ ( (cache_ptr)->index_size < ((cache_ptr)->clean_index_size) ) || \ ( (cache_ptr)->index_size < ((cache_ptr)->dirty_index_size) ) || \ ( ( !( was_clean ) || \ - ( (cache_ptr)->clean_index_size < (old_size) ) ) && \ - ( ( (was_clean) ) || \ - ( (cache_ptr)->dirty_index_size < (old_size) ) ) ) || \ + ( (cache_ptr)->clean_index_size < (old_size) ) ) && \ + ( ( (was_clean) ) || \ + ( (cache_ptr)->dirty_index_size < (old_size) ) ) ) || \ ( (entry_ptr) == NULL ) || \ ( (entry_ptr)->ring <= H5C_RING_UNDEFINED ) || \ ( (entry_ptr)->ring >= H5C_RING_NTYPES ) || \ @@ -1196,20 +1196,20 @@ if ( ( (cache_ptr) == NULL ) || \ } #define H5C__POST_HT_ENTRY_SIZE_CHANGE_SC(cache_ptr, old_size, new_size, \ - entry_ptr) \ + entry_ptr) \ if ( ( (cache_ptr) == NULL ) || \ ( (cache_ptr)->index_len <= 0 ) || \ ( (cache_ptr)->index_size <= 0 ) || \ ( (new_size) > (cache_ptr)->index_size ) || \ ( (cache_ptr)->index_size != \ - ((cache_ptr)->clean_index_size + \ + ((cache_ptr)->clean_index_size + \ (cache_ptr)->dirty_index_size) ) || \ ( (cache_ptr)->index_size < ((cache_ptr)->clean_index_size) ) || \ ( (cache_ptr)->index_size < ((cache_ptr)->dirty_index_size) ) || \ ( ( !((entry_ptr)->is_dirty ) || \ - ( (cache_ptr)->dirty_index_size < (new_size) ) ) && \ - ( ( ((entry_ptr)->is_dirty) ) || \ - ( (cache_ptr)->clean_index_size < (new_size) ) ) ) || \ + ( (cache_ptr)->dirty_index_size < (new_size) ) ) && \ + ( ( ((entry_ptr)->is_dirty) ) || \ + ( (cache_ptr)->clean_index_size < (new_size) ) ) ) || \ ( ( (cache_ptr)->index_len == 1 ) && \ ( (cache_ptr)->index_size != (new_size) ) ) || \ ( (cache_ptr)->index_ring_len[(entry_ptr)->ring] > \ @@ -1465,10 +1465,10 @@ if ( ( (cache_ptr)->index_size != \ H5C__PRE_HT_UPDATE_FOR_ENTRY_CLEAN_SC(cache_ptr, entry_ptr); \ (cache_ptr)->dirty_index_size -= (entry_ptr)->size; \ ((cache_ptr)->dirty_index_ring_size[entry_ptr->ring]) \ - -= (entry_ptr)->size; \ + -= (entry_ptr)->size; \ (cache_ptr)->clean_index_size += (entry_ptr)->size; \ ((cache_ptr)->clean_index_ring_size[entry_ptr->ring]) \ - += (entry_ptr)->size; \ + += (entry_ptr)->size; \ H5C__POST_HT_UPDATE_FOR_ENTRY_CLEAN_SC(cache_ptr, entry_ptr); \ } @@ -1477,18 +1477,18 @@ if ( ( (cache_ptr)->index_size != \ H5C__PRE_HT_UPDATE_FOR_ENTRY_DIRTY_SC(cache_ptr, entry_ptr); \ (cache_ptr)->clean_index_size -= (entry_ptr)->size; \ ((cache_ptr)->clean_index_ring_size[entry_ptr->ring]) \ - -= (entry_ptr)->size; \ + -= (entry_ptr)->size; \ (cache_ptr)->dirty_index_size += (entry_ptr)->size; \ ((cache_ptr)->dirty_index_ring_size[entry_ptr->ring]) \ - += (entry_ptr)->size; \ + += (entry_ptr)->size; \ H5C__POST_HT_UPDATE_FOR_ENTRY_DIRTY_SC(cache_ptr, entry_ptr); \ } #define H5C__UPDATE_INDEX_FOR_SIZE_CHANGE(cache_ptr, old_size, new_size, \ - entry_ptr, was_clean) \ + entry_ptr, was_clean) \ { \ H5C__PRE_HT_ENTRY_SIZE_CHANGE_SC(cache_ptr, old_size, new_size, \ - entry_ptr, was_clean) \ + entry_ptr, was_clean) \ (cache_ptr)->index_size -= (old_size); \ (cache_ptr)->index_size += (new_size); \ ((cache_ptr)->index_ring_size[entry_ptr->ring]) -= (old_size); \ @@ -1497,14 +1497,14 @@ if ( ( (cache_ptr)->index_size != \ (cache_ptr)->clean_index_size -= (old_size); \ ((cache_ptr)->clean_index_ring_size[entry_ptr->ring])-= (old_size); \ } else { \ - (cache_ptr)->dirty_index_size -= (old_size); \ + (cache_ptr)->dirty_index_size -= (old_size); \ ((cache_ptr)->dirty_index_ring_size[entry_ptr->ring])-= (old_size); \ } \ if((entry_ptr)->is_dirty) { \ (cache_ptr)->dirty_index_size += (new_size); \ ((cache_ptr)->dirty_index_ring_size[entry_ptr->ring])+= (new_size); \ } else { \ - (cache_ptr)->clean_index_size += (new_size); \ + (cache_ptr)->clean_index_size += (new_size); \ ((cache_ptr)->clean_index_ring_size[entry_ptr->ring])+= (new_size); \ } \ H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr)->il_len, \ @@ -1791,7 +1791,7 @@ if ( ( (cache_ptr)->index_size != \ } else { /* slist disabled */ \ \ HDassert( (cache_ptr)->slist_len == 0 ); \ - HDassert( (cache_ptr)->slist_size == 0 ); \ + HDassert( (cache_ptr)->slist_size == 0 ); \ } \ } /* H5C__REMOVE_ENTRY_FROM_SLIST */ @@ -2033,16 +2033,16 @@ if ( ( (cache_ptr)->index_size != \ /* modified LRU specific code */ \ \ /* remove the entry from the LRU list, and re-insert it at the head.\ - */ \ + */ \ \ H5C__DLL_REMOVE((entry_ptr), (cache_ptr)->LRU_head_ptr, \ (cache_ptr)->LRU_tail_ptr, \ - (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_len, \ (cache_ptr)->LRU_list_size, (fail_val)) \ \ H5C__DLL_PREPEND((entry_ptr), (cache_ptr)->LRU_head_ptr, \ (cache_ptr)->LRU_tail_ptr, \ - (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_len, \ (cache_ptr)->LRU_list_size, (fail_val)) \ \ /* Use the dirty flag to infer whether the entry is on the clean or \ @@ -2096,16 +2096,16 @@ if ( ( (cache_ptr)->index_size != \ /* modified LRU specific code */ \ \ /* remove the entry from the LRU list, and re-insert it at the head \ - */ \ + */ \ \ H5C__DLL_REMOVE((entry_ptr), (cache_ptr)->LRU_head_ptr, \ (cache_ptr)->LRU_tail_ptr, \ - (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_len, \ (cache_ptr)->LRU_list_size, (fail_val)) \ \ H5C__DLL_PREPEND((entry_ptr), (cache_ptr)->LRU_head_ptr, \ (cache_ptr)->LRU_tail_ptr, \ - (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_len, \ (cache_ptr)->LRU_list_size, (fail_val)) \ \ /* End modified LRU specific code. */ \ @@ -2288,28 +2288,28 @@ if ( ( (cache_ptr)->index_size != \ /* modified LRU specific code */ \ \ /* remove the entry from the LRU list, and re-insert it at the \ - * head. \ - */ \ + * head. \ + */ \ \ H5C__DLL_REMOVE((entry_ptr), (cache_ptr)->LRU_head_ptr, \ (cache_ptr)->LRU_tail_ptr, \ - (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_len, \ (cache_ptr)->LRU_list_size, (fail_val)) \ \ H5C__DLL_PREPEND((entry_ptr), (cache_ptr)->LRU_head_ptr, \ (cache_ptr)->LRU_tail_ptr, \ - (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_len, \ (cache_ptr)->LRU_list_size, (fail_val)) \ \ /* since the entry is being flushed or cleared, one would think \ - * that it must be dirty -- but that need not be the case. Use the \ - * dirty flag to infer whether the entry is on the clean or dirty \ - * LRU list, and remove it. Then insert it at the head of the \ - * clean LRU list. \ + * that it must be dirty -- but that need not be the case. Use the \ + * dirty flag to infer whether the entry is on the clean or dirty \ + * LRU list, and remove it. Then insert it at the head of the \ + * clean LRU list. \ * \ * The function presumes that a dirty entry will be either cleared \ - * or flushed shortly, so it is OK if we put a dirty entry on the \ - * clean LRU list. \ + * or flushed shortly, so it is OK if we put a dirty entry on the \ + * clean LRU list. \ */ \ \ if ( (entry_ptr)->is_dirty ) { \ @@ -2350,17 +2350,17 @@ if ( ( (cache_ptr)->index_size != \ /* modified LRU specific code */ \ \ /* remove the entry from the LRU list, and re-insert it at the \ - * head. \ - */ \ + * head. \ + */ \ \ H5C__DLL_REMOVE((entry_ptr), (cache_ptr)->LRU_head_ptr, \ (cache_ptr)->LRU_tail_ptr, \ - (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_len, \ (cache_ptr)->LRU_list_size, (fail_val)) \ \ H5C__DLL_PREPEND((entry_ptr), (cache_ptr)->LRU_head_ptr, \ (cache_ptr)->LRU_tail_ptr, \ - (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_len, \ (cache_ptr)->LRU_list_size, (fail_val)) \ \ /* End modified LRU specific code. */ \ @@ -2424,7 +2424,7 @@ if ( ( (cache_ptr)->index_size != \ \ H5C__DLL_APPEND((entry_ptr), (cache_ptr)->LRU_head_ptr, \ (cache_ptr)->LRU_tail_ptr, \ - (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_len, \ (cache_ptr)->LRU_list_size, (fail_val)) \ \ /* insert the entry at the tail of the clean or dirty LRU list as \ @@ -2465,7 +2465,7 @@ if ( ( (cache_ptr)->index_size != \ (cache_ptr)->pel_tail_ptr, \ (cache_ptr)->pel_len, \ (cache_ptr)->pel_size, (fail_val)) \ - \ + \ } else { \ \ /* modified LRU specific code */ \ @@ -2474,7 +2474,7 @@ if ( ( (cache_ptr)->index_size != \ \ H5C__DLL_APPEND((entry_ptr), (cache_ptr)->LRU_head_ptr, \ (cache_ptr)->LRU_tail_ptr, \ - (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_len, \ (cache_ptr)->LRU_list_size, (fail_val)) \ \ /* End modified LRU specific code. */ \ @@ -2558,7 +2558,7 @@ if ( ( (cache_ptr)->index_size != \ \ H5C__DLL_PREPEND((entry_ptr), (cache_ptr)->LRU_head_ptr, \ (cache_ptr)->LRU_tail_ptr, \ - (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_len, \ (cache_ptr)->LRU_list_size, (fail_val)) \ \ /* insert the entry at the head of the clean or dirty LRU list as \ @@ -2599,7 +2599,7 @@ if ( ( (cache_ptr)->index_size != \ (cache_ptr)->pel_tail_ptr, \ (cache_ptr)->pel_len, \ (cache_ptr)->pel_size, (fail_val)) \ - \ + \ } else { \ \ /* modified LRU specific code */ \ @@ -2608,7 +2608,7 @@ if ( ( (cache_ptr)->index_size != \ \ H5C__DLL_PREPEND((entry_ptr), (cache_ptr)->LRU_head_ptr, \ (cache_ptr)->LRU_tail_ptr, \ - (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_len, \ (cache_ptr)->LRU_list_size, (fail_val)) \ \ /* End modified LRU specific code. */ \ @@ -2677,12 +2677,12 @@ if ( ( (cache_ptr)->index_size != \ HDassert( !((entry_ptr)->is_read_only) ); \ HDassert( ((entry_ptr)->ro_ref_count) == 0 ); \ HDassert( (entry_ptr)->size > 0 ); \ - \ + \ if ( (entry_ptr)->is_pinned ) { \ \ H5C__DLL_REMOVE((entry_ptr), (cache_ptr)->pel_head_ptr, \ - (cache_ptr)->pel_tail_ptr, \ - (cache_ptr)->pel_len, \ + (cache_ptr)->pel_tail_ptr, \ + (cache_ptr)->pel_len, \ (cache_ptr)->pel_size, (fail_val)) \ \ } else { \ @@ -2693,7 +2693,7 @@ if ( ( (cache_ptr)->index_size != \ \ H5C__DLL_REMOVE((entry_ptr), (cache_ptr)->LRU_head_ptr, \ (cache_ptr)->LRU_tail_ptr, \ - (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_len, \ (cache_ptr)->LRU_list_size, (fail_val)) \ \ /* Similarly, remove the entry from the clean or dirty LRU list \ @@ -2739,12 +2739,12 @@ if ( ( (cache_ptr)->index_size != \ HDassert( !((entry_ptr)->is_read_only) ); \ HDassert( ((entry_ptr)->ro_ref_count) == 0 ); \ HDassert( (entry_ptr)->size > 0 ); \ - \ + \ if ( (entry_ptr)->is_pinned ) { \ \ H5C__DLL_REMOVE((entry_ptr), (cache_ptr)->pel_head_ptr, \ - (cache_ptr)->pel_tail_ptr, \ - (cache_ptr)->pel_len, \ + (cache_ptr)->pel_tail_ptr, \ + (cache_ptr)->pel_len, \ (cache_ptr)->pel_size, (fail_val)) \ \ } else { \ @@ -2755,7 +2755,7 @@ if ( ( (cache_ptr)->index_size != \ \ H5C__DLL_REMOVE((entry_ptr), (cache_ptr)->LRU_head_ptr, \ (cache_ptr)->LRU_tail_ptr, \ - (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_len, \ (cache_ptr)->LRU_list_size, (fail_val)) \ \ /* End modified LRU specific code. */ \ @@ -2804,21 +2804,21 @@ if ( ( (cache_ptr)->index_size != \ HDassert( ((entry_ptr)->ro_ref_count) == 0 ); \ HDassert( (entry_ptr)->size > 0 ); \ \ - if ( ! ( (entry_ptr)->is_pinned ) && ! ( (entry_ptr->is_protected ) ) ) { \ - \ + if ( ! ( (entry_ptr)->is_pinned ) && ! ( (entry_ptr->is_protected ) ) ) {\ + \ /* modified LRU specific code */ \ \ /* remove the entry from the LRU list, and re-insert it at the head. \ - */ \ + */ \ \ H5C__DLL_REMOVE((entry_ptr), (cache_ptr)->LRU_head_ptr, \ (cache_ptr)->LRU_tail_ptr, \ - (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_len, \ (cache_ptr)->LRU_list_size, (fail_val)) \ \ H5C__DLL_PREPEND((entry_ptr), (cache_ptr)->LRU_head_ptr, \ (cache_ptr)->LRU_tail_ptr, \ - (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_len, \ (cache_ptr)->LRU_list_size, (fail_val)) \ \ /* remove the entry from either the clean or dirty LUR list as \ @@ -2827,7 +2827,7 @@ if ( ( (cache_ptr)->index_size != \ if ( was_dirty ) { \ \ H5C__AUX_DLL_REMOVE((entry_ptr), \ - (cache_ptr)->dLRU_head_ptr, \ + (cache_ptr)->dLRU_head_ptr, \ (cache_ptr)->dLRU_tail_ptr, \ (cache_ptr)->dLRU_list_len, \ (cache_ptr)->dLRU_list_size, \ @@ -2836,34 +2836,34 @@ if ( ( (cache_ptr)->index_size != \ } else { \ \ H5C__AUX_DLL_REMOVE((entry_ptr), \ - (cache_ptr)->cLRU_head_ptr, \ + (cache_ptr)->cLRU_head_ptr, \ (cache_ptr)->cLRU_tail_ptr, \ (cache_ptr)->cLRU_list_len, \ (cache_ptr)->cLRU_list_size, \ - (fail_val)) \ + (fail_val)) \ } \ \ /* insert the entry at the head of either the clean or dirty \ - * LRU list as appropriate. \ + * LRU list as appropriate. \ */ \ \ if ( (entry_ptr)->is_dirty ) { \ \ H5C__AUX_DLL_PREPEND((entry_ptr), \ - (cache_ptr)->dLRU_head_ptr, \ + (cache_ptr)->dLRU_head_ptr, \ (cache_ptr)->dLRU_tail_ptr, \ (cache_ptr)->dLRU_list_len, \ (cache_ptr)->dLRU_list_size, \ - (fail_val)) \ + (fail_val)) \ \ } else { \ \ H5C__AUX_DLL_PREPEND((entry_ptr), \ - (cache_ptr)->cLRU_head_ptr, \ + (cache_ptr)->cLRU_head_ptr, \ (cache_ptr)->cLRU_tail_ptr, \ (cache_ptr)->cLRU_list_len, \ (cache_ptr)->cLRU_list_size, \ - (fail_val)) \ + (fail_val)) \ } \ \ /* End modified LRU specific code. */ \ @@ -2872,7 +2872,7 @@ if ( ( (cache_ptr)->index_size != \ #else /* H5C_MAINTAIN_CLEAN_AND_DIRTY_LRU_LISTS */ -#define H5C__UPDATE_RP_FOR_MOVE(cache_ptr, entry_ptr, was_dirty, fail_val) \ +#define H5C__UPDATE_RP_FOR_MOVE(cache_ptr, entry_ptr, was_dirty, fail_val) \ { \ HDassert( (cache_ptr) ); \ HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ @@ -2881,21 +2881,21 @@ if ( ( (cache_ptr)->index_size != \ HDassert( ((entry_ptr)->ro_ref_count) == 0 ); \ HDassert( (entry_ptr)->size > 0 ); \ \ - if ( ! ( (entry_ptr)->is_pinned ) && ! ( (entry_ptr->is_protected ) ) ) { \ - \ + if ( ! ( (entry_ptr)->is_pinned ) && ! ( (entry_ptr->is_protected ) ) ) {\ + \ /* modified LRU specific code */ \ \ /* remove the entry from the LRU list, and re-insert it at the head. \ - */ \ + */ \ \ H5C__DLL_REMOVE((entry_ptr), (cache_ptr)->LRU_head_ptr, \ (cache_ptr)->LRU_tail_ptr, \ - (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_len, \ (cache_ptr)->LRU_list_size, (fail_val)) \ \ H5C__DLL_PREPEND((entry_ptr), (cache_ptr)->LRU_head_ptr, \ (cache_ptr)->LRU_tail_ptr, \ - (cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_len, \ (cache_ptr)->LRU_list_size, (fail_val)) \ \ /* End modified LRU specific code. */ \ @@ -2952,49 +2952,49 @@ if ( ( (cache_ptr)->index_size != \ \ if ( (entry_ptr)->coll_access ) { \ \ - H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr)->coll_list_len, \ - (cache_ptr)->coll_list_size, \ - (entry_ptr)->size, \ - (new_size)); \ - \ + H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr)->coll_list_len, \ + (cache_ptr)->coll_list_size, \ + (entry_ptr)->size, \ + (new_size)); \ + \ } \ \ if ( (entry_ptr)->is_pinned ) { \ \ - H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr)->pel_len, \ - (cache_ptr)->pel_size, \ - (entry_ptr)->size, \ - (new_size)); \ - \ + H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr)->pel_len, \ + (cache_ptr)->pel_size, \ + (entry_ptr)->size, \ + (new_size)); \ + \ } else { \ \ /* modified LRU specific code */ \ \ - /* Update the size of the LRU list */ \ + /* Update the size of the LRU list */ \ \ - H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr)->LRU_list_len, \ - (cache_ptr)->LRU_list_size, \ - (entry_ptr)->size, \ - (new_size)); \ + H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_size, \ + (entry_ptr)->size, \ + (new_size)); \ \ /* Similarly, update the size of the clean or dirty LRU list as \ - * appropriate. At present, the entry must be clean, but that \ - * could change. \ + * appropriate. At present, the entry must be clean, but that \ + * could change. \ */ \ \ if ( (entry_ptr)->is_dirty ) { \ \ - H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr)->dLRU_list_len, \ - (cache_ptr)->dLRU_list_size, \ - (entry_ptr)->size, \ - (new_size)); \ + H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr)->dLRU_list_len, \ + (cache_ptr)->dLRU_list_size, \ + (entry_ptr)->size, \ + (new_size)); \ \ } else { \ \ - H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr)->cLRU_list_len, \ - (cache_ptr)->cLRU_list_size, \ - (entry_ptr)->size, \ - (new_size)); \ + H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr)->cLRU_list_len, \ + (cache_ptr)->cLRU_list_size, \ + (entry_ptr)->size, \ + (new_size)); \ } \ \ /* End modified LRU specific code. */ \ @@ -3017,21 +3017,21 @@ if ( ( (cache_ptr)->index_size != \ \ if ( (entry_ptr)->is_pinned ) { \ \ - H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr)->pel_len, \ - (cache_ptr)->pel_size, \ - (entry_ptr)->size, \ - (new_size)); \ + H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr)->pel_len, \ + (cache_ptr)->pel_size, \ + (entry_ptr)->size, \ + (new_size)); \ \ } else { \ \ /* modified LRU specific code */ \ \ - /* Update the size of the LRU list */ \ + /* Update the size of the LRU list */ \ \ - H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr)->LRU_list_len, \ - (cache_ptr)->LRU_list_size, \ - (entry_ptr)->size, \ - (new_size)); \ + H5C__DLL_UPDATE_FOR_SIZE_CHANGE((cache_ptr)->LRU_list_len, \ + (cache_ptr)->LRU_list_size, \ + (entry_ptr)->size, \ + (new_size)); \ \ /* End modified LRU specific code. */ \ } \ @@ -3318,7 +3318,7 @@ if ( ( (hd_ptr) == NULL ) || \ ( (Size) < (entry_ptr)->size ) || \ ( ( (Size) == (entry_ptr)->size ) && ( ! ( (len) == 1 ) ) ) || \ ( ( (entry_ptr)->coll_prev == NULL ) && ( (hd_ptr) != (entry_ptr) ) ) || \ - ( ( (entry_ptr)->coll_next == NULL ) && ( (tail_ptr) != (entry_ptr) ) ) || \ + ( ( (entry_ptr)->coll_next == NULL ) && ( (tail_ptr) != (entry_ptr) ) ) ||\ ( ( (len) == 1 ) && \ ( ! ( ( (hd_ptr) == (entry_ptr) ) && ( (tail_ptr) == (entry_ptr) ) && \ ( (entry_ptr)->coll_next == NULL ) && \ @@ -3350,10 +3350,10 @@ if ( ( ( ( (head_ptr) == NULL ) || ( (tail_ptr) == NULL ) ) && \ ) \ ) { \ HDassert(0 && "COLL DLL sanity check failed"); \ - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, (fv), "COLL DLL sanity check failed") \ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, (fv), "COLL DLL sanity check failed")\ } -#define H5C__COLL_DLL_PRE_INSERT_SC(entry_ptr, hd_ptr, tail_ptr, len, Size, fv) \ +#define H5C__COLL_DLL_PRE_INSERT_SC(entry_ptr, hd_ptr, tail_ptr, len, Size, fv)\ if ( ( (entry_ptr) == NULL ) || \ ( (entry_ptr)->coll_next != NULL ) || \ ( (entry_ptr)->coll_prev != NULL ) || \ @@ -5074,7 +5074,7 @@ H5_DLL herr_t H5C__generate_cache_image(H5F_t *f, H5C_t *cache_ptr); H5_DLL herr_t H5C__load_cache_image(H5F_t *f); H5_DLL herr_t H5C__mark_flush_dep_serialized(H5C_cache_entry_t * entry_ptr); H5_DLL herr_t H5C__mark_flush_dep_unserialized(H5C_cache_entry_t * entry_ptr); -H5_DLL herr_t H5C__make_space_in_cache(H5F_t * f, size_t space_needed, +H5_DLL herr_t H5C__make_space_in_cache(H5F_t * f, size_t space_needed, hbool_t write_permitted); H5_DLL herr_t H5C__flush_marked_entries(H5F_t * f); H5_DLL herr_t H5C__serialize_cache(H5F_t *f); diff --git a/src/H5ES.c b/src/H5ES.c index 9abaa54..ad42000 100644 --- a/src/H5ES.c +++ b/src/H5ES.c @@ -269,7 +269,7 @@ H5ESget_requests(hid_t es_id, H5_iter_order_t order, hid_t *connector_ids, void herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE5("e", "iIo*i**xx", es_id, order, connector_ids, requests, count); + H5TRACE6("e", "iIo*i**xzx", es_id, order, connector_ids, requests, array_len, count); /* Check arguments */ if (NULL == (es = H5I_object_verify(es_id, H5I_EVENTSET))) diff --git a/src/H5FD.c b/src/H5FD.c index 397da34..20f69cb 100644 --- a/src/H5FD.c +++ b/src/H5FD.c @@ -928,9 +928,10 @@ H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2) { int ret_value = -1; /* Return value */ - FUNC_ENTER_NOAPI_NOERR /* return value is arbitrary */ + FUNC_ENTER_NOAPI_NOERR; /* return value is arbitrary */ - if ((!f1 || !f1->cls) && (!f2 || !f2->cls)) HGOTO_DONE(0) + if ((!f1 || !f1->cls) && (!f2 || !f2->cls)) + HGOTO_DONE(0) if (!f1 || !f1->cls) HGOTO_DONE(-1) if (!f2 || !f2->cls) diff --git a/src/H5Ocopy.c b/src/H5Ocopy.c index 52b1b5c..0b0bb55 100644 --- a/src/H5Ocopy.c +++ b/src/H5Ocopy.c @@ -33,7 +33,6 @@ #include "H5Aprivate.h" /* Attributes */ #include "H5CXprivate.h" /* API Contexts */ #include "H5Eprivate.h" /* Error handling */ -#include "H5ESprivate.h" /* Event Sets */ #include "H5FLprivate.h" /* Free lists */ #include "H5Iprivate.h" /* IDs */ #include "H5HGprivate.h" /* Global Heaps */ diff --git a/src/H5Tcommit.c b/src/H5Tcommit.c index e99494a..a3a1aa0 100644 --- a/src/H5Tcommit.c +++ b/src/H5Tcommit.c @@ -1127,7 +1127,7 @@ H5T_open(const H5G_loc_t *loc) done: if (ret_value == NULL) { if (dt) { - if (shared_fo == NULL) { /* Need to free shared of */ + if (shared_fo == NULL) { /* Need to free shared file object */ if (dt->shared->owned_vol_obj && H5VL_free_object(dt->shared->owned_vol_obj) < 0) HDONE_ERROR(H5E_DATATYPE, H5E_CANTCLOSEOBJ, NULL, "unable to close owned VOL object") dt->shared = H5FL_FREE(H5T_shared_t, dt->shared); diff --git a/src/H5VLcallback.c b/src/H5VLcallback.c index 4cf4d53..0c5c73d 100644 --- a/src/H5VLcallback.c +++ b/src/H5VLcallback.c @@ -30,7 +30,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5ESprivate.h" /* Event Sets */ -#include "H5Fprivate.h" /* File access */ +#include "H5Fprivate.h" /* File access */ #include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory management */ #include "H5Pprivate.h" /* Property lists */ diff --git a/src/H5VLnative.h b/src/H5VLnative.h index fe8ede2..5e43c4e 100644 --- a/src/H5VLnative.h +++ b/src/H5VLnative.h @@ -74,7 +74,7 @@ typedef union H5VL_native_attr_optional_args_t { #define H5VL_NATIVE_DATASET_CHUNK_WRITE 7 /* H5Dchunk_write */ #define H5VL_NATIVE_DATASET_GET_VLEN_BUF_SIZE 8 /* H5Dvlen_get_buf_size */ #define H5VL_NATIVE_DATASET_GET_OFFSET 9 /* H5Dget_offset */ -#define H5VL_NATIVE_DATASET_CHUNK_ITER 10 /* H5Dget_offset */ +#define H5VL_NATIVE_DATASET_CHUNK_ITER 10 /* H5Dchunk_iter */ /* NOTE: If values over 1023 are added, the H5VL_RESERVED_NATIVE_OPTIONAL macro * must be updated. */ @@ -208,8 +208,8 @@ typedef union H5VL_native_dataset_optional_args_t { #ifdef H5_HAVE_PARALLEL #define H5VL_NATIVE_FILE_GET_MPI_ATOMICITY 26 /* H5Fget_mpi_atomicity */ #define H5VL_NATIVE_FILE_SET_MPI_ATOMICITY 27 /* H5Fset_mpi_atomicity */ -#endif /* H5_HAVE_PARALLEL */ -#define H5VL_NATIVE_FILE_POST_OPEN 28 /* Adjust file after open, with wrapping context */ +#endif +#define H5VL_NATIVE_FILE_POST_OPEN 28 /* Adjust file after open, with wrapping context */ /* NOTE: If values over 1023 are added, the H5VL_RESERVED_NATIVE_OPTIONAL macro * must be updated. */ diff --git a/test/swmr_common.c b/test/swmr_common.c index 2201427..b359bc6 100644 --- a/test/swmr_common.c +++ b/test/swmr_common.c @@ -202,13 +202,12 @@ generate_symbols(void) unsigned u, v; /* Local index variables */ for (u = 0; u < NLEVELS; u++) { - symbol_info[u] = (symbol_info_t *)HDmalloc(symbol_count[u] * sizeof(symbol_info_t)); + symbol_info[u] = HDmalloc(symbol_count[u] * sizeof(symbol_info_t)); for (v = 0; v < symbol_count[u]; v++) { char name_buf[64]; generate_name(name_buf, u, v); - symbol_info[u][v].name = (char *)HDmalloc(HDstrlen(name_buf) + 1); - HDstrcpy(symbol_info[u][v].name, name_buf); + symbol_info[u][v].name = HDstrdup(name_buf); symbol_info[u][v].dsid = -1; symbol_info[u][v].nrecords = 0; } /* end for */ diff --git a/test/swmr_sparse_reader.c b/test/swmr_sparse_reader.c index 15dcb67..8f1c781 100644 --- a/test/swmr_sparse_reader.c +++ b/test/swmr_sparse_reader.c @@ -117,7 +117,7 @@ check_dataset(hid_t fid, unsigned verbose, const symbol_info_t *symbol, symbol_t (uintmax_t)start[0], (uintmax_t)start[1]); /* Read record from dataset */ - record->rec_id = (uint64_t)ULLONG_MAX; + record->rec_id = UINT64_MAX; if (H5Dread(dsid, symbol_tid, rec_sid, file_sid, H5P_DEFAULT, record) < 0) return -1; -- cgit v0.12 From b9470be379354af209daf8418b6c571dde99e776 Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Thu, 10 Mar 2022 10:22:16 -0800 Subject: Plugin test cleanup (#1479) * Autotools plugin test cleanup * Combines filter, VFD, and VOL plugin tests * Adds VFD plugin tests to the Autotools * Implements a uniform shell script naming scheme in test/ * codespell fix * Changes after code review --- MANIFEST | 31 ++- configure.ac | 25 +- test/CMakeTests.cmake | 42 ++-- test/Makefile.am | 67 ++--- test/ShellTests.cmake | 58 ++--- test/filter_plugin.c | 10 - test/test_abort_fail.sh.in | 66 +++++ test/test_check_version.sh.in | 261 ++++++++++++++++++++ test/test_error.sh.in | 126 ++++++++++ test/test_external_env.sh.in | 42 ++++ test/test_filter_plugin.sh.in | 113 --------- test/test_flush_refresh.sh.in | 238 ++++++++++++++++++ test/test_libinfo.sh.in | 120 +++++++++ test/test_links_env.sh.in | 42 ++++ test/test_plugin.sh.in | 140 +++++++++++ test/test_swmr.pwsh.in | 519 ++++++++++++++++++++++++++++++++++++++ test/test_swmr.sh.in | 563 ++++++++++++++++++++++++++++++++++++++++++ test/test_use_cases.sh.in | 219 ++++++++++++++++ test/test_usecases.sh.in | 219 ---------------- test/test_vds_env.sh.in | 44 ++++ test/test_vds_swmr.pwsh.in | 190 ++++++++++++++ test/test_vds_swmr.sh.in | 233 +++++++++++++++++ test/test_vol_plugin.sh.in | 84 ------- test/testabort_fail.sh.in | 66 ----- test/testcheck_version.sh.in | 261 -------------------- test/testerror.sh.in | 126 ---------- test/testexternal_env.sh.in | 42 ---- test/testflushrefresh.sh.in | 238 ------------------ test/testlibinfo.sh.in | 120 --------- test/testlinks_env.sh.in | 42 ---- test/testswmr.pwsh.in | 519 -------------------------------------- test/testswmr.sh.in | 563 ------------------------------------------ test/testvds_env.sh.in | 44 ---- test/testvdsswmr.pwsh.in | 190 -------------- test/testvdsswmr.sh.in | 233 ----------------- 35 files changed, 2910 insertions(+), 2986 deletions(-) create mode 100644 test/test_abort_fail.sh.in create mode 100644 test/test_check_version.sh.in create mode 100644 test/test_error.sh.in create mode 100644 test/test_external_env.sh.in delete mode 100644 test/test_filter_plugin.sh.in create mode 100644 test/test_flush_refresh.sh.in create mode 100644 test/test_libinfo.sh.in create mode 100644 test/test_links_env.sh.in create mode 100644 test/test_plugin.sh.in create mode 100644 test/test_swmr.pwsh.in create mode 100644 test/test_swmr.sh.in create mode 100644 test/test_use_cases.sh.in delete mode 100644 test/test_usecases.sh.in create mode 100644 test/test_vds_env.sh.in create mode 100644 test/test_vds_swmr.pwsh.in create mode 100644 test/test_vds_swmr.sh.in delete mode 100644 test/test_vol_plugin.sh.in delete mode 100644 test/testabort_fail.sh.in delete mode 100644 test/testcheck_version.sh.in delete mode 100644 test/testerror.sh.in delete mode 100644 test/testexternal_env.sh.in delete mode 100644 test/testflushrefresh.sh.in delete mode 100644 test/testlibinfo.sh.in delete mode 100644 test/testlinks_env.sh.in delete mode 100644 test/testswmr.pwsh.in delete mode 100644 test/testswmr.sh.in delete mode 100644 test/testvds_env.sh.in delete mode 100644 test/testvdsswmr.pwsh.in delete mode 100644 test/testvdsswmr.sh.in diff --git a/MANIFEST b/MANIFEST index 4d11d7f..cd6d086 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1341,28 +1341,27 @@ ./test/tchecksum.c ./test/tconfig.c ./test/tcoords.c -./test/test_filter_plugin.sh.in +./test/test_abort_fail.sh.in +./test/test_check_version.sh.in +./test/test_error.sh.in +./test/test_external_env.sh.in ./test/test_filters_be.h5 ./test/test_filters_le.h5 -./test/test_usecases.sh.in -./test/test_vol_plugin.sh.in -./test/testabort_fail.sh.in -./test/testcheck_version.sh.in -./test/testerror.sh.in -./test/testexternal_env.sh.in -./test/testflushrefresh.sh.in +./test/test_flush_refresh.sh.in +./test/test_libinfo.sh.in +./test/test_links_env.sh.in +./test/test_mirror.sh.in +./test/test_plugin.sh.in +./test/test_swmr.pwsh.in +./test/test_swmr.sh.in +./test/test_use_cases.sh.in +./test/test_vds_env.sh.in +./test/test_vds_swmr.pwsh.in +./test/test_vds_swmr.sh.in ./test/testframe.c ./test/testhdf5.c ./test/testhdf5.h -./test/testlibinfo.sh.in -./test/testlinks_env.sh.in ./test/testmeta.c -./test/test_mirror.sh.in -./test/testswmr.pwsh.in -./test/testswmr.sh.in -./test/testvds_env.sh.in -./test/testvdsswmr.pwsh.in -./test/testvdsswmr.sh.in ./test/tfile.c ./test/tgenprop.c ./test/th5o.c diff --git a/configure.ac b/configure.ac index e426632..4d7be31 100644 --- a/configure.ac +++ b/configure.ac @@ -4165,20 +4165,19 @@ AC_CONFIG_FILES([src/libhdf5.settings src/Makefile test/Makefile test/H5srcdir_str.h - test/testabort_fail.sh - test/testcheck_version.sh - test/testerror.sh - test/testexternal_env.sh - test/testflushrefresh.sh - test/testlibinfo.sh - test/testlinks_env.sh - test/testswmr.sh - test/testvds_env.sh - test/testvdsswmr.sh - test/test_filter_plugin.sh + test/test_abort_fail.sh + test/test_check_version.sh + test/test_error.sh + test/test_external_env.sh + test/test_flush_refresh.sh + test/test_libinfo.sh + test/test_links_env.sh test/test_mirror.sh - test/test_usecases.sh - test/test_vol_plugin.sh + test/test_plugin.sh + test/test_swmr.sh + test/test_use_cases.sh + test/test_vds_env.sh + test/test_vds_swmr.sh testpar/Makefile testpar/testpflush.sh utils/Makefile diff --git a/test/CMakeTests.cmake b/test/CMakeTests.cmake index 4e828f8..6abef3b 100644 --- a/test/CMakeTests.cmake +++ b/test/CMakeTests.cmake @@ -674,18 +674,18 @@ set_tests_properties (H5TEST-tcheck_version-release PROPERTIES # flushrefresh ############################################################################## # autotools script tests -# error_test and err_compat are built at the same time as the other tests, but executed by testerror.sh. -# NOT CONVERTED accum_swmr_reader is used by accum.c. -# NOT CONVERTED atomic_writer and atomic_reader are standalone programs. -# links_env is used by testlinks_env.sh -# filenotclosed and del_many_dense_attrs are used by testabort_fail.sh -# NOT CONVERTED flushrefresh is used by testflushrefresh.sh. -# NOT CONVERTED use_append_chunk, use_append_mchunks and use_disable_mdc_flushes are used by test_usecases.sh -# NOT CONVERTED swmr_* files (besides swmr.c) are used by testswmr.sh. -# NOT CONVERTED vds_swmr_* files are used by testvdsswmr.sh -# NOT CONVERTED 'make check' doesn't run them directly, so they are not included in TEST_PROG. -# NOT CONVERTED Also build testmeta, which is used for timings test. It builds quickly, -# NOT CONVERTED and this lets automake keep all its test programs in one place. +# error_test and err_compat are built at the same time as the other tests, but executed by test_error.sh +# NOT CONVERTED accum_swmr_reader is used by accum.c +# NOT CONVERTED atomic_writer and atomic_reader are stand-alone programs +# links_env is used by test_links_env.sh +# filenotclosed and del_many_dense_attrs are used by test_abort_fail.sh +# NOT CONVERTED flushrefresh is used by test_flush_refresh.sh +# NOT CONVERTED use_append_chunk, use_append_mchunks and use_disable_mdc_flushes are used by test_use_cases.sh +# NOT CONVERTED swmr_* files (besides swmr.c) are used by test_swmr.sh +# NOT CONVERTED vds_swmr_* files are used by test_vds_swmr.sh +# 'make check' doesn't run them directly, so they are not included in TEST_PROG. +# Also build testmeta, which is used for timings test. It builds quickly +# and this lets automake keep all its test programs in one place. ############################################################################## #-- Adding test for filenotclosed @@ -853,16 +853,6 @@ if (BUILD_SHARED_LIBS) ENVIRONMENT "HDF5_PLUGIN_PATH=${CMAKE_BINARY_DIR}/filter_plugin_dir1${CMAKE_SEP}${CMAKE_BINARY_DIR}/filter_plugin_dir2;srcdir=${HDF5_TEST_BINARY_DIR}" WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR} ) - -############################################################################## -# HDFFV-9655 relative plugin test disabled -# -# add_test (NAME H5PLUGIN-pluginRelative COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $) -# set_tests_properties (H5PLUGIN-pluginRelative PROPERTIES -# ENVIRONMENT "HDF5_PLUGIN_PATH=@/${BIN_REL_PATH}testdir1${CMAKE_SEP}@/${BIN_REL_PATH}testdir2;srcdir=${HDF5_TEST_BINARY_DIR}" -# WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR} -# ) -############################################################################## endif () option (TEST_SHELL_SCRIPTS "Enable shell script tests" ON) @@ -875,10 +865,10 @@ if (ENABLE_EXTENDED_TESTS) ############################################################################## ### S W M R T E S T S ############################################################################## -# testflushrefresh.sh: flushrefresh -# test_usecases.sh: use_append_chunk, use_append_mchunks, use_disable_mdc_flushes -# testswmr.sh: swmr* -# testvdsswmr.sh: vds_swmr* +# test_flush_refresh.sh: flushrefresh +# test_use_cases.sh: use_append_chunk, use_append_mchunks, use_disable_mdc_flushes +# test_swmr.sh: swmr* +# test_vds_swmr.sh: vds_swmr* #-- Adding test for flushrefresh file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/flushrefresh_test") diff --git a/test/Makefile.am b/test/Makefile.am index ff09003..1d31525 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -22,21 +22,23 @@ include $(top_srcdir)/config/commence.am AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_builddir)/src # Test scripts-- -# testerror.sh: err_compat, error_test -# testlibinfo.sh: -# testcheck_version.sh: tcheck_version -# testlinks_env.sh: links_env -# testexternal_env.sh: external_env -# testflushrefresh.sh: flushrefresh -# testvds_env.sh: vds_env -# testswmr.sh: swmr* -# testvdsswmr.sh: vds_swmr* -# testabort_fail.sh: filenotclosed.c and del_many_dense_attrs.c -# test_filter_plugin.sh: filter_plugin.c +# test_abort_fail.sh: filenotclosed.c and del_many_dense_attrs.c +# test_check_version.sh: tcheck_version +# test_error.sh: err_compat, error_test +# test_external_env.sh: external_env +# test_flush_refresh.sh: flushrefresh +# test_libinfo.sh: +# test_links_env.sh: links_env # test_mirror.sh: mirror_vfd ../utils/mirror_vfd/* -# test_usecases.sh: use_append_chunk, use_append_mchunks, use_disable_mdc_flushes -TEST_SCRIPT = testerror.sh testlibinfo.sh testcheck_version.sh testlinks_env.sh testexternal_env.sh \ - testswmr.sh testvds_env.sh testvdsswmr.sh testflushrefresh.sh test_usecases.sh testabort_fail.sh +# test_plugin.sh: filter_plugin.c vfd_plugin.c vol_plugin.c +# test_swmr.sh: swmr* +# test_vds_env.sh: vds_env +# test_vds_swmr.sh: vds_swmr* +# test_use_cases.sh: use_append_chunk, use_append_mchunks, use_disable_mdc_flushes +TEST_SCRIPT = test_abort_fail.sh test_check_version.sh test_error.sh \ + test_flush_refresh.sh test_external_env.sh test_libinfo.sh \ + test_links_env.sh test_swmr.sh test_vds_env.sh test_vds_swmr.sh \ + test_use_cases.sh SCRIPT_DEPEND = error_test$(EXEEXT) err_compat$(EXEEXT) links_env$(EXEEXT) \ external_env$(EXEEXT) filenotclosed$(EXEEXT) del_many_dense_attrs$(EXEEXT) \ flushrefresh$(EXEEXT) use_append_chunk$(EXEEXT) use_append_mchunks$(EXEEXT) use_disable_mdc_flushes$(EXEEXT) \ @@ -45,10 +47,9 @@ SCRIPT_DEPEND = error_test$(EXEEXT) err_compat$(EXEEXT) links_env$(EXEEXT) \ swmr_sparse_reader$(EXEEXT) swmr_sparse_writer$(EXEEXT) swmr_start_write$(EXEEXT) \ vds_env$(EXEEXT) vds_swmr_gen$(EXEEXT) vds_swmr_reader$(EXEEXT) vds_swmr_writer$(EXEEXT) if HAVE_SHARED_CONDITIONAL - TEST_SCRIPT += test_filter_plugin.sh test_vol_plugin.sh - SCRIPT_DEPEND += filter_plugin$(EXEEXT) vol_plugin$(EXEEXT) + TEST_SCRIPT += test_plugin.sh + SCRIPT_DEPEND += filter_plugin$(EXEEXT) vfd_plugin $(EXEEXT) vol_plugin$(EXEEXT) endif - if MIRROR_VFD_CONDITIONAL TEST_SCRIPT += test_mirror.sh endif @@ -71,22 +72,25 @@ TEST_PROG= testhdf5 \ dangle dtransform reserved cross_read freespace mf vds file_image \ unregister cache_logging cork swmr thread_id vol timer event_set -# List programs to be built when testing here. -# error_test and err_compat are built at the same time as the other tests, but executed by testerror.sh. -# tcheck_version is used by testcheck_version.sh. -# accum_swmr_reader is used by accum.c. -# atomic_writer and atomic_reader are standalone programs. -# links_env is used by testlinks_env.sh -# external_env is used by testexternal_env.sh -# filenotclosed and del_many_dense_attrs are used by testabort_fail.sh -# flushrefresh is used by testflushrefresh.sh. -# use_append_chunk, use_append_mchunks and use_disable_mdc_flushes are used by test_usecases.sh -# swmr_* files (besides swmr.c) are used by testswmr.sh. -# vds_swmr_* files are used by testvdsswmr.sh -# vds_env is used by testvds_env.sh +# List programs to be built when testing here +# +# error_test and err_compat are built at the same time as the other tests, but executed by test_error.sh +# tcheck_version is used by test_check_version.sh +# accum_swmr_reader is used by accum.c +# atomic_writer and atomic_reader are stand-alone programs +# links_env is used by test_links_env.sh +# external_env is used by test_external_env.sh +# filenotclosed and del_many_dense_attrs are used by test_abort_fail.sh +# flushrefresh is used by test_flush_refresh.sh +# use_append_chunk, use_append_mchunks and use_disable_mdc_flushes are used by test_use_cases.sh +# swmr_* files (besides swmr.c) are used by test_swmr.sh +# vds_swmr_* files are used by test_vds_swmr.sh +# vds_env is used by test_vds_env.sh # mirror_vfd is used by test_mirror.sh +# # 'make check' doesn't run them directly, so they are not included in TEST_PROG. -# Also build testmeta, which is used for timings test. It builds quickly, +# +# Also build testmeta, which is used for the timing test. It builds quickly # and this lets automake keep all its test programs in one place. check_PROGRAMS=$(TEST_PROG) error_test err_compat tcheck_version \ testmeta accum_swmr_reader atomic_writer atomic_reader external_env \ @@ -98,7 +102,6 @@ check_PROGRAMS=$(TEST_PROG) error_test err_compat tcheck_version \ if HAVE_SHARED_CONDITIONAL check_PROGRAMS+= filter_plugin vfd_plugin vol_plugin endif - if MIRROR_VFD_CONDITIONAL check_PROGRAMS+= mirror_vfd endif diff --git a/test/ShellTests.cmake b/test/ShellTests.cmake index e7de735..140da6f 100644 --- a/test/ShellTests.cmake +++ b/test/ShellTests.cmake @@ -17,21 +17,21 @@ find_program (PWSH NAMES pwsh powershell) if (PWSH) - file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/usecases_test") + file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/use_cases_test") file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/swmr_test") file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/vds_swmr_test") set (srcdir ${HDF5_TEST_SOURCE_DIR}) set (H5_UTILS_TEST_BUILDDIR ${CMAKE_TEST_OUTPUT_DIRECTORY}) set (H5_TEST_BUILDDIR ${HDF5_TEST_BINARY_DIR}/H5TEST) - configure_file(${HDF5_TEST_SOURCE_DIR}/testswmr.pwsh.in ${HDF5_TEST_BINARY_DIR}/H5TEST/testswmr.ps1 @ONLY) + configure_file(${HDF5_TEST_SOURCE_DIR}/test_swmr.pwsh.in ${HDF5_TEST_BINARY_DIR}/H5TEST/test_swmr.ps1 @ONLY) # test commented out as currently the programs are not allowing another access to the data file #add_test (H5SHELL-testswmr ${PWSH} ${HDF5_TEST_BINARY_DIR}/H5TEST/testswmr.ps1) #set_tests_properties (H5SHELL-testswmr PROPERTIES # ENVIRONMENT "PATH=$ENV{PATH}:${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" # WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST #) - configure_file(${HDF5_TEST_SOURCE_DIR}/testvdsswmr.pwsh.in ${HDF5_TEST_BINARY_DIR}/H5TEST/testvdsswmr.ps1 @ONLY) + configure_file(${HDF5_TEST_SOURCE_DIR}/test_vds_swmr.pwsh.in ${HDF5_TEST_BINARY_DIR}/H5TEST/test_vds_swmr.ps1 @ONLY) # test commented out as currently the programs are not allowing another access to the data file #add_test (H5SHELL-testvdsswmr ${PWSH} ${HDF5_TEST_BINARY_DIR}/H5TEST/testvdsswmr.ps1) #set_tests_properties (H5SHELL-testvdsswmr PROPERTIES @@ -48,11 +48,11 @@ elseif (UNIX) # configure scripts to test dir ############################################################################## if (H5_PERL_FOUND) - configure_file(${HDF5_TEST_SOURCE_DIR}/testflushrefresh.sh.in ${HDF5_TEST_BINARY_DIR}/H5TEST/testflushrefresh.sh @ONLY) + configure_file(${HDF5_TEST_SOURCE_DIR}/test_flush_refresh.sh.in ${HDF5_TEST_BINARY_DIR}/H5TEST/test_flush_refresh.sh @ONLY) endif () - configure_file(${HDF5_TEST_SOURCE_DIR}/test_usecases.sh.in ${HDF5_TEST_BINARY_DIR}/H5TEST/test_usecases.sh @ONLY) - configure_file(${HDF5_TEST_SOURCE_DIR}/testswmr.sh.in ${HDF5_TEST_BINARY_DIR}/H5TEST/testswmr.sh @ONLY) - configure_file(${HDF5_TEST_SOURCE_DIR}/testvdsswmr.sh.in ${HDF5_TEST_BINARY_DIR}/H5TEST/testvdsswmr.sh @ONLY) + configure_file(${HDF5_TEST_SOURCE_DIR}/test_use_cases.sh.in ${HDF5_TEST_BINARY_DIR}/H5TEST/test_use_cases.sh @ONLY) + configure_file(${HDF5_TEST_SOURCE_DIR}/test_swmr.sh.in ${HDF5_TEST_BINARY_DIR}/H5TEST/test_swmr.sh @ONLY) + configure_file(${HDF5_TEST_SOURCE_DIR}/test_vds_swmr.sh.in ${HDF5_TEST_BINARY_DIR}/H5TEST/test_vds_swmr.sh @ONLY) ############################################################################## # copy test programs to test dir @@ -78,44 +78,44 @@ elseif (UNIX) # flushrefresh ############################################################################## # autotools script tests - # error_test and err_compat are built at the same time as the other tests, but executed by testerror.sh. - # NOT CONVERTED accum_swmr_reader is used by accum.c. - # NOT CONVERTED atomic_writer and atomic_reader are standalone programs. - # links_env is used by testlinks_env.sh - # filenotclosed and del_many_dense_attrs are used by testabort_fail.sh - # NOT CONVERTED flushrefresh is used by testflushrefresh.sh. - # NOT CONVERTED use_append_chunk, use_append_mchunks and use_disable_mdc_flushes are used by test_usecases.sh - # NOT CONVERTED swmr_* files (besides swmr.c) are used by testswmr.sh. - # NOT CONVERTED vds_swmr_* files are used by testvdsswmr.sh + # error_test and err_compat are built at the same time as the other tests, but executed by test_error.sh + # NOT CONVERTED accum_swmr_reader is used by accum.c + # NOT CONVERTED atomic_writer and atomic_reader are stand-alone programs + # links_env is used by test_links_env.sh + # filenotclosed and del_many_dense_attrs are used by test_abort_fail.sh + # NOT CONVERTED flushrefresh is used by test_flush_refresh.sh. + # NOT CONVERTED use_append_chunk, use_append_mchunks and use_disable_mdc_flushes are used by test_use_cases.sh + # NOT CONVERTED swmr_* files (besides swmr.c) are used by test_swmr.sh. + # NOT CONVERTED vds_swmr_* files are used by test_vds_swmr.sh # NOT CONVERTED 'make check' doesn't run them directly, so they are not included in TEST_PROG. - # NOT CONVERTED Also build testmeta, which is used for timings test. It builds quickly, + # NOT CONVERTED Also build testmeta, which is used for timing test. It builds quickly # NOT CONVERTED and this lets automake keep all its test programs in one place. ############################################################################## ############################################################################## ### S W M R T E S T S ############################################################################## - # testflushrefresh.sh: flushrefresh - # test_usecases.sh: use_append_chunk, use_append_mchunks, use_disable_mdc_flushes - # testswmr.sh: swmr* - # testvdsswmr.sh: vds_swmr* - add_test (H5SHELL-testflushrefresh ${SH_PROGRAM} ${HDF5_TEST_BINARY_DIR}/H5TEST/testflushrefresh.sh) - set_tests_properties (H5SHELL-testflushrefresh PROPERTIES + # test_flush_refresh.sh: flushrefresh + # test_use_cases.sh: use_append_chunk, use_append_mchunks, use_disable_mdc_flushes + # test_swmr.sh: swmr* + # test_vds_swmr.sh: vds_swmr* + add_test (H5SHELL-test_flush_refresh ${SH_PROGRAM} ${HDF5_TEST_BINARY_DIR}/H5TEST/test_flush_refresh.sh) + set_tests_properties (H5SHELL-test_flush_refresh PROPERTIES ENVIRONMENT "LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}:${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST ) - add_test (H5SHELL-test_usecases ${SH_PROGRAM} ${HDF5_TEST_BINARY_DIR}/H5TEST/test_usecases.sh) - set_tests_properties (H5SHELL-test_usecases PROPERTIES + add_test (H5SHELL-test_use_cases ${SH_PROGRAM} ${HDF5_TEST_BINARY_DIR}/H5TEST/test_use_cases.sh) + set_tests_properties (H5SHELL-test_use_cases PROPERTIES ENVIRONMENT "LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}:${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST ) - add_test (H5SHELL-testswmr ${SH_PROGRAM} ${HDF5_TEST_BINARY_DIR}/H5TEST/testswmr.sh) - set_tests_properties (H5SHELL-testswmr PROPERTIES + add_test (H5SHELL-test_swmr ${SH_PROGRAM} ${HDF5_TEST_BINARY_DIR}/H5TEST/test_swmr.sh) + set_tests_properties (H5SHELL-test_swmr PROPERTIES ENVIRONMENT "LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}:${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST ) - add_test (H5SHELL-testvdsswmr ${SH_PROGRAM} ${HDF5_TEST_BINARY_DIR}/H5TEST/testvdsswmr.sh) - set_tests_properties (H5SHELL-testvdsswmr PROPERTIES + add_test (H5SHELL-test_vds_swmr ${SH_PROGRAM} ${HDF5_TEST_BINARY_DIR}/H5TEST/test_vds_swmr.sh) + set_tests_properties (H5SHELL-test_vds_swmr PROPERTIES ENVIRONMENT "LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}:${CMAKE_RUNTIME_OUTPUT_DIRECTORY}" WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST ) diff --git a/test/filter_plugin.c b/test/filter_plugin.c index c373b3b..dd61a43 100644 --- a/test/filter_plugin.c +++ b/test/filter_plugin.c @@ -968,16 +968,6 @@ test_path_api_calls(void) */ n_starting_paths = 42; - /* Check that initialization is correct */ - TESTING(" initialize"); - - if (H5PLsize(&n_paths) < 0) - TEST_ERROR; - if (n_paths != 2) - TEST_ERROR; - - PASSED(); - /****************/ /* H5PLremove() */ /****************/ diff --git a/test/test_abort_fail.sh.in b/test/test_abort_fail.sh.in new file mode 100644 index 0000000..a866f9e --- /dev/null +++ b/test/test_abort_fail.sh.in @@ -0,0 +1,66 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Test to verify that the assertion/abort failure is fixed when the application +# does not close the file. (See HDFFV-10160) +# +# Test to verify that the infinite loop closing library/abort failure is fixed +# when the application creates and removes dense attributes (See HDFFV-10659) + +srcdir=@srcdir@ + +nerrors=0 + +############################################################################## +############################################################################## +### T H E T E S T S ### +############################################################################## +############################################################################## +# +# +echo "Testing file not closed assertion/abort failure" +TEST_NAME=filenotclosed # The test name +TEST_BIN=`pwd`/$TEST_NAME # The path of the test binary +# +# Run the test +$RUNSERIAL $TEST_BIN >/dev/null 2>&1 +exitcode=$? +if [ $exitcode -eq 0 ]; then + echo "Test PASSED" +else + echo "Test FAILED" + nerrors="`expr $nerrors + 1`" +fi +# +# +echo "Testing infinite loop closing library/abort failure" +TEST_NAME=del_many_dense_attrs # The test name +TEST_BIN=`pwd`/$TEST_NAME # The path of the test binary +# Run the test +$RUNSERIAL $TEST_BIN >/dev/null 2>&1 +exitcode=$? +if [ $exitcode -eq 0 ]; then + echo "Test PASSED" +else + echo "Test FAILED" + nerrors="`expr $nerrors + 1`" +fi +# +# +if test $nerrors -eq 0 ; then + echo "All tests for abort failure passed." + exit 0 +else + echo "Tests for abort failure failed with $nerrors errors." + exit 1 +fi diff --git a/test/test_check_version.sh.in b/test/test_check_version.sh.in new file mode 100644 index 0000000..273702e --- /dev/null +++ b/test/test_check_version.sh.in @@ -0,0 +1,261 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. + +# +# Tests for the H5check_version function. +# +# Programmer: Albert Cheng +# Sep 28, 2009 + +srcdir=@srcdir@ + +# Variables filled in by the configure process. +# Determine the configure options of the hdf5 library and executables. +Shared_Lib=@enable_shared@ +Static_Lib=@enable_static@ +Static_exec=@STATIC_EXEC@ +h5haveexitcode=yes # default is yes + +CMP='cmp -s' +DIFF='diff -c' +RM='rm -f' + +# Function definitions +# +# Show the purpose of this test script and a note about the abort messages. +PURPOSE() { + echo "Tests for the H5check_version function." + echo "Note that abort messages may appear due to the expected termination" + echo "of the program when it is tested with mis-matched version numbers." +} + +# Print a line-line message left justified in a field of 70 characters. +# +LINEMSG() { + SPACES=" " + echo "$* $SPACES" | cut -c1-70 | tr -d '\012' +} + + +# Print a "SKIP" message +SKIP() { + LINEMSG $* + echo " -SKIP-" +} + + +# Print warning message of version mismatch. +WarnMesg(){ + echo "Warning! ***HDF5 library version mismatched error***" + echo "The HDF5 header files used to compile this application do not match" + echo "the version used by the HDF5 library to which this application is linked." + echo "Data corruption or segmentation faults may occur if the application continues." + echo "This can happen when an application was compiled by one version of HDF5 but" + echo "linked with a different version of static or shared HDF5 library." + echo "You should recompile the application or check your shared library related" + echo "settings such as 'LD_LIBRARY_PATH'." + echo "You can, at your own risk, disable this warning by setting the environment" + echo "variable 'HDF5_DISABLE_VERSION_CHECK' to a value of '1'." + echo "Setting it to 2 or higher will suppress the warning messages totally." + echo "Headers are $xxh5versmajor.$xxh5versminor.$xxh5versrelease, library is $h5versmajor.$h5versminor.$h5versrelease" + test -n "$H5_HAVE_EMBEDDED_LIBINFO" && cat $h5libsettings + echo "Bye..." +} + + +# Print warning message2 of version mismatch. +WarnMesg2(){ + echo "Warning! ***HDF5 library version mismatched error***" + echo "The HDF5 header files used to compile this application do not match" + echo "the version used by the HDF5 library to which this application is linked." + echo "Data corruption or segmentation faults may occur if the application continues." + echo "This can happen when an application was compiled by one version of HDF5 but" + echo "linked with a different version of static or shared HDF5 library." + echo "You should recompile the application or check your shared library related" + echo "settings such as 'LD_LIBRARY_PATH'." + echo "'HDF5_DISABLE_VERSION_CHECK' environment variable is set to 1, application will" + echo "continue at your own risk." + echo "Headers are $xxh5versmajor.$xxh5versminor.$xxh5versrelease, library is $h5versmajor.$h5versminor.$h5versrelease" + test -n "$H5_HAVE_EMBEDDED_LIBINFO" && cat $h5libsettings +} + + +# Run a test and print PASS or *FAIL*. If a test fails then increment +# the `nerrors' global variable and (if $verbose is set) display the +# difference between the actual output and the expected output. The +# expected output generated according to the parameter values and compared +# against actual output. +# The expected and actual output files are removed unless $HDF5_NOCLEANUP +# has a non-zero value. +# $1: the set value of $HDF5_DISABLE_VERSION_CHECK. (unset means not to set +# it at all. +# $2: Change the version number(s) to cause a mismatch. (none means no +# mismatch). +# +# Expected results: +# Value of $HDF5_DISABLE_VERSION_CHECK +# unset "" -1 0 1 2 3 +# Matched OK OK OK OK OK OK OK +# Mismatched W/A W/A W/A W/A W2/OK OK W2/OK +# Result codes: +# OK: No warning, exit 0. +# W/A: Warning, abort and exit non-0. +# W2/OK: Different Warning, exit 0. +# +# Implemented only exit code matching. Still need to match output. +TESTING() { + DEBUGPRINT command is $0 $* + TEST_NAME=tcheck_version # The test name + TEST_BIN=`pwd`/$TEST_NAME # The path of the test binary + + expect=${TEST_NAME}_expect.out + actual=${TEST_NAME}_actual.out + actual_err=${TEST_NAME}_actual.err + arguments= + + h5DisableVersion="$1" + wrongversionnumbers="$2" + xxh5versmajor=$h5versmajor + xxh5versminor=$h5versminor + xxh5versrelease=$h5versrelease + + if [ "$h5DisableVersion" = unset ]; then + envcmd="" # noop + else + envcmd="env HDF5_DISABLE_VERSION_CHECK=$h5DisableVersion" + fi + + if [ "$wrongversionnumbers" = none ]; then + # OK: No warning, exit 0 + cp /dev/null $expect + expect_code=0 + else + arguments=-t"$wrongversionnumbers" + # calculate mismatched version numbers by listing. + case $wrongversionnumbers in + "M") xxh5versmajor=`expr $h5versmajor + 1` + ;; + "m") xxh5versminor=`expr $h5versminor + 1` + ;; + "r") xxh5versrelease=`expr $h5versrelease + 1` + ;; + esac + case "$h5DisableVersion" in + 1) + # W2/OK: Different Warning, exit 0. + WarnMesg2 > $expect + expect_code=0 + ;; + [2-9]|[1-9][0-9]*) + # OK: No warning, exit 0 + cp /dev/null $expect + expect_code=0 + ;; + *) # W/A: Warning, abort and exit non-0. + WarnMesg > $expect + expect_code=6 # Signal Abort exit code (128+6) + ;; + esac + fi + + # Run test. + LINEMSG $envcmd $TEST_NAME $arguments + ( + $envcmd $RUNSERIAL $TEST_BIN $arguments + ) >$actual 2>$actual_err + ret_code=$? + cat $actual_err >> $actual + + if [ $h5haveexitcode = 'yes' -a \( $expect_code -ne $ret_code \) ]; then + echo "*FAILED*" + echo " Expected exit code ($expect_code) differs from actual code ($ret_code)" + nerrors="`expr $nerrors + 1`" + elif $CMP $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result differs from actual result" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + fi + + # Clean up output file. + # Also clean the core file generated by H5check_version's abort. + if test -z "$HDF5_NOCLEANUP"; then + $RM $expect $actual $actual_err + $RM core + fi +} + + +# Echo parameters for debugging if verbose mode is on. +DEBUGPRINT() { + if [ -n "$debugmode" ]; then + echo $* + fi +} + + +# MAIN Body +nerrors=0 +verbose=yes # default on +debugmode= # default off +H5_HAVE_EMBEDDED_LIBINFO=`grep '#define H5_HAVE_EMBEDDED_LIBINFO ' ../src/H5pubconf.h` +h5libsettings=../src/libhdf5.settings + +PURPOSE + +# Figure out library version numbers from the header file. +h5versmajor=`grep '#define H5_VERS_MAJOR' $srcdir/../src/H5public.h | awk '{print $3}'` +h5versminor=`grep '#define H5_VERS_MINOR' $srcdir/../src/H5public.h | awk '{print $3}'` +h5versrelease=`grep '#define H5_VERS_RELEASE' $srcdir/../src/H5public.h | awk '{print $3}'` +DEBUGPRINT $h5versmajor.$h5versminor.$h5versrelease +case "$h5versmajor$h5versminor$h5versrelease" in + [0-9]*) # good. noop. + ;; + *) + echo "Illegal library version numbers($h5versmajor.$h5versminor.$h5versrelease)" + echo "Test aborted" + exit 1 + ;; +esac + +# RUNSERIAL is used. Check if it can return exit code from executalbe correctly. +if [ -n "$RUNSERIAL_NOEXITCODE" ]; then + echo "***Warning*** Serial Exit Code is not passed back to shell correctly." + echo "***Warning*** Exit code checking is skipped." + h5haveexitcode=no +fi + +# Three Categories of tests: +# Normal: where the version numbers all matched (wrong_version == none). +# Mismatched version numbers (could be Major or minor version +# or release numbers or a combination of all three.) +# Test all the above with different values of the environment variable, +# HDF5_DISABLE_VERSION_CHECK, as unset, "", -1, 0, 1, 2, 3 + +for val_disable_version_check in unset "" -1 0 1 2 3; do + for wrong_version in none M m; do + TESTING "$val_disable_version_check" "$wrong_version" + done +done + + +# Check and report results. +if [ $nerrors -gt 0 ]; then + echo "***$nerrors errors encountered***" + exit 1 +else + echo "No error encountered" + exit 0 +fi diff --git a/test/test_error.sh.in b/test/test_error.sh.in new file mode 100644 index 0000000..4fb2a81 --- /dev/null +++ b/test/test_error.sh.in @@ -0,0 +1,126 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Tests for test_error and err_compat + +srcdir=@srcdir@ + +# Determine backward compatibility options eneabled +DEPRECATED_SYMBOLS="@DEPRECATED_SYMBOLS@" + +CMP='cmp -s' +DIFF='diff -c' + +# Skip plugin module to test missing filter +# Also reset the VOL connector to only use the native connector, because of the +# error stack checking. QAK - 2019/03/09 +ENVCMD="env HDF5_PLUGIN_PRELOAD=:: HDF5_VOL_CONNECTOR=native" + +nerrors=0 +verbose=yes + +test -d ./testfiles || mkdir ./testfiles + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +TESTING() { + SPACES=" " + echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# Run a test and print PASS or *FAIL*. If a test fails then increment +# the `nerrors' global variable and (if $verbose is set) display the +# difference between the actual output and the expected output. The +# expected output is given as the first argument to this function and +# the actual output file is calculated by replacing the `.ddl' with +# `.out'. The actual output is not removed if $HDF5_NOCLEANUP has a +# non-zero value. +# +TEST() { + TEST_ERR=$1 # The test name + TEST_ERR_BIN=`pwd`/$TEST_ERR # The path of the test binary + + expect1="$srcdir/testfiles/$1_1" + expect2="$srcdir/testfiles/$1_2" + actual="./`basename $1`.out" + actual_err="./`basename $1`.err" + actual_ext="./`basename $1`.ext" + shift + + # Run test. + TESTING $TEST_ERR + ( + # Skip the plugin for testing missing filter. + $ENVCMD $RUNSERIAL $TEST_ERR_BIN + ) >$actual 2>$actual_err + + # Check for core dump + if [ $? != 0 ]; then + nerrors=`expr $nerrors + 1` + fi + + # Extract file name, line number, version and thread IDs because they may be different + sed -e 's/thread [0-9]*/thread (IDs)/' -e 's/: .*\.c /: (file name) /' \ + -e 's/line [0-9]*/line (number)/' \ + -e 's/v[1-9]*\.[0-9]*\./version (number)\./' \ + -e 's/[1-9]*\.[0-9]*\.[0-9]*[^)]*/version (number)/' \ + -e 's/H5Eget_auto[1-2]*/H5Eget_auto(1 or 2)/' \ + -e 's/H5Eset_auto[1-2]*/H5Eset_auto(1 or 2)/' \ + $actual_err > $actual_ext + cat $actual_ext >> $actual + + if $CMP $expect1 $actual; then + echo " PASSED" + elif $CMP $expect2 $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result differs from actual result" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect1 $actual |sed 's/^/ /' + fi + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $actual $actual_err $actual_ext + fi +} + +# Print a "SKIP" message +SKIP() { + TESTING $@ + echo " -SKIP-" +} + +############################################################################## +############################################################################## +### T H E T E S T S ### +############################################################################## +############################################################################## + +# test for err_compat +if test $DEPRECATED_SYMBOLS != "yes"; then +SKIP err_compat +else +TEST err_compat +fi + +# test for error_test. Skip the plugin for testing missing filter. +TEST error_test + +if test $nerrors -eq 0 ; then + echo "All Error API tests passed." +fi + +exit $nerrors diff --git a/test/test_external_env.sh.in b/test/test_external_env.sh.in new file mode 100644 index 0000000..94fbb88 --- /dev/null +++ b/test/test_external_env.sh.in @@ -0,0 +1,42 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Test for external file with environment variable: HDF5_EXTFILE_PREFIX + +srcdir=@srcdir@ + +nerrors=0 + +############################################################################## +############################################################################## +### T H E T E S T S ### +############################################################################## +############################################################################## + +# test for external file with HDF5_EXTFILE_PREFIX +echo "Testing external file with HDF5_EXTFILE_PREFIX" +TEST_NAME=external_env # The test name +TEST_BIN=`pwd`/$TEST_NAME # The path of the test binary +ENVCMD="env HDF5_EXTFILE_PREFIX=\${ORIGIN}" # The environment variable & value +# +# Run the test +# echo "$ENVCMD $RUNSERIAL $TEST_BIN" +$ENVCMD $RUNSERIAL $TEST_BIN +exitcode=$? +if [ $exitcode -eq 0 ]; then + echo "Test prefix for HDF5_EXTFILE_PREFIX PASSED" +else + nerrors="`expr $nerrors + 1`" + echo "***Error encountered for HDF5_EXTFILE_PREFIX test***" +fi +exit $nerrors diff --git a/test/test_filter_plugin.sh.in b/test/test_filter_plugin.sh.in deleted file mode 100644 index 78cdb3a..0000000 --- a/test/test_filter_plugin.sh.in +++ /dev/null @@ -1,113 +0,0 @@ -#! /bin/sh -# -# Copyright by The HDF Group. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. -# -srcdir=@srcdir@ -TOP_BUILDDIR=@top_builddir@ - -# Determine if backward compatibility options enabled -DEPRECATED_SYMBOLS="@DEPRECATED_SYMBOLS@" - -EXIT_SUCCESS=0 -EXIT_FAILURE=1 - -nerrors=0 -verbose=yes -exit_code=$EXIT_SUCCESS - -TEST_NAME=filter_plugin -TEST_BIN=`pwd`/$TEST_NAME -FROM_DIR=`pwd`/.libs -case $(uname) in - CYGWIN* ) - PLUGINS_FOR_DIR1="$FROM_DIR/cygfilter_plugin1* $FROM_DIR/cygfilter_plugin3*" - PLUGINS_FOR_DIR2="$FROM_DIR/cygfilter_plugin2* $FROM_DIR/cygfilter_plugin4*" - ;; - *) - PLUGINS_FOR_DIR1="$FROM_DIR/libfilter_plugin1* $FROM_DIR/libfilter_plugin3*" - PLUGINS_FOR_DIR2="$FROM_DIR/libfilter_plugin2* $FROM_DIR/libfilter_plugin4*" - ;; -esac -PLUGIN_DIR1=filter_plugin_dir1 -PLUGIN_DIR2=filter_plugin_dir2 -CP="cp -p" # Use -p to preserve mode,ownership,timestamps -RM="rm -rf" - -# Print a line-line message left justified in a field of 70 characters -# beginning with the word "Testing". -# -TESTING() { - SPACES=" " - echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' -} - -# Main Body -# Create test directories if necessary. -test -d $PLUGIN_DIR1 || mkdir -p $PLUGIN_DIR1 -if [ $? != 0 ]; then - echo "Failed to create filter plugin test directory ($PLUGIN_DIR1)" - exit $EXIT_FAILURE -fi - -test -d $PLUGIN_DIR2 || mkdir -p $PLUGIN_DIR2 -if [ $? != 0 ]; then - echo "Failed to create filter plugin test directory ($PLUGIN_DIR2)" - exit $EXIT_FAILURE -fi - -# Copy plugins for the tests. -$CP $PLUGINS_FOR_DIR1 $PLUGIN_DIR1 -if [ $? != 0 ]; then - echo "Failed to copy filter plugins ($PLUGINS_FOR_DIR1) to test directory." - exit $EXIT_FAILURE -fi - -$CP $PLUGINS_FOR_DIR2 $PLUGIN_DIR2 -if [ $? != 0 ]; then - echo "Failed to copy filter plugins ($PLUGINS_FOR_DIR2) to test directory." - exit $EXIT_FAILURE -fi - -# setup plugin path -ENVCMD="env HDF5_PLUGIN_PATH=${PLUGIN_DIR1}:${PLUGIN_DIR2}" - -# Run the test -$ENVCMD $TEST_BIN -if [ $? != 0 ]; then - nerrors=`expr $nerrors + 1` -fi - -############################################ -# HDFFV-9655 test for relative path disabled -# setup filter plugin path relative to test -# actual executable is in the .libs folder -#ENVCMD="env HDF5_PLUGIN_PATH=@/../${PLUGIN_DIR1}:@/../${PLUGIN_DIR2}" -# -# Run the test -#$ENVCMD $TEST_BIN -#if [ $? != 0 ]; then -# nerrors=`expr $nerrors + 1` -#fi -############################################# - -# print results -if test $nerrors -ne 0 ; then - echo "$nerrors errors encountered" - exit_code=$EXIT_FAILURE -else - echo "All filter plugin tests passed." - exit_code=$EXIT_SUCCESS -fi - -# Clean up temporary files/directories and leave -$RM $PLUGIN_DIR1 $PLUGIN_DIR2 - -exit $exit_code diff --git a/test/test_flush_refresh.sh.in b/test/test_flush_refresh.sh.in new file mode 100644 index 0000000..209d370 --- /dev/null +++ b/test/test_flush_refresh.sh.in @@ -0,0 +1,238 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# +# Test script for the flush/evict single objects feature. +# +# This test file doesn't actually perform any tests, rather, it acts +# as a process manager for the 'flushrefresh' test file, which is where +# the tests are actually housed. The reason this script exists is because +# the verification of this feature needs to occur in separate processes +# from the one in which the file is being manipulated in. (i.e., we have +# a single writer process, and various reader processes spawning off +# and doing the verification that individual objects are being +# correctly flushed). +# +# Programmer: +# Mike McGreevy +# Tuesday, July 20, 2010 + +############################################################################### +## test variables +############################################################################### + +# Number of errors encountered during test run. +nerrors=0 + +# Set up a function to check the current time since the epoch - ideally, we'd +# like to use Perl. If it wasn't detected by configure, then use date, though +# this is less portable and might cause problems on machines that don't +# recognize the +%s option (like Solaris). +# +# Note that PERL will resolve to true or false, not a path. +PERL=@PERL@ +if test -n "$PERL"; then + TimeStamp() + { + time=`perl -e 'print int(time)'` + echo "$time" + } +else + TimeStamp() + { + time=`date +%s` + echo "$time" + } +fi + +############################################################################### +## Main +############################################################################### +srcdir=@srcdir@ +utils_testdir=@abs_top_builddir@/@H5_UTILS_TEST_BUILDDIR@ +testdir=@abs_top_builddir@/@H5_TEST_BUILDDIR@ + +# Check to see if the VFD specified by the HDF5_DRIVER environment variable +# supports SWMR. +$utils_testdir/swmr_check_compat_vfd +rc=$? +if [ $rc -ne 0 ] ; then + echo + echo "The VFD specified by the HDF5_DRIVER environment variable" + echo "does not support SWMR." + echo + echo "flush/refresh tests skipped" + echo + exit 0 +fi + +# HDF5 has several tests that create and delete signal files to communicate +# between processes, and it seems that even though the names of the files are +# different, occasionally the wrong file is deleted, interrupting the flow of +# the test. Running each of these tests in its own directory should eliminate +# the problem. +mkdir -p flushrefresh_test +cp $testdir/flushrefresh flushrefresh_test + +# With the --disable-shared option, flushrefresh is built in the test directory, +# otherwise it is in test/.libs with a wrapper script named flushrefresh in +# the test directory. test/flushrefresh should always be copied, +# .libs/flushrefresh should be copied only if it exists. +if [ -f .libs/flushrefresh ]; then + mkdir -p flushrefresh_test/.libs + for FILE in .libs/flushrefresh*; do + case "$FILE" in + *.o) continue ;; ## don't copy the .o files + esac + if test -f "$FILE" ; then + cp $FILE flushrefresh_test/.libs + fi + done +fi +cd flushrefresh_test + +# ================================================= +# Set up/initialize some variables to be used later +# ================================================= +testfile=flushrefresh.h5 +startsignal=flushrefresh_VERIFICATION_START +endsignal=flushrefresh_VERIFICATION_DONE +timeout_length=300 +timedout=0 +verification_done=0 +if [ -e $testfile ]; then + rm $testfile +fi + +# ======================== +# Launch the Test Program. +# ======================== +$testdir/flushrefresh & +pid_main=$! + +# ======================================= +# Run flush verification on test program. +# ======================================= + +until [ $verification_done -eq 1 ]; do + + # Wait for signal from test program that verification routine can run. + before=`TimeStamp` + until [ -s $startsignal ]; do + after=`TimeStamp` + timediff=`expr $after - $before` + if [ $timediff -gt $timeout_length ]; then + nerrors=`expr $nerrors + 1` + timedout=1 + break + fi + done + + # Check to see if we timed out looking for the signal before continuing. + if [ $timedout -gt 0 ]; then + echo "timed out waiting for signal from test program (flush)." + break + fi + + # Read in test routine parameters from signal file, then delete signal file. + param1=`head -1 $startsignal` + param2=`tail -1 $startsignal` + rm $startsignal + + # Check if we're done with verifications, otherwise run the specified verification. + if [ "$param1" = "VERIFICATION_DONE" ]; then + verification_done=1 + echo "all flush verification complete" > $endsignal + else + $testdir/flushrefresh $param1 $param2 + + # Check for core dump + if [ $? -gt 0 ]; then + nerrors=`expr $nerrors + 1` + fi + + echo "verification flush process done" > $endsignal + fi + +done + +# ========================================= +# Run refresh verification on test program. +# ========================================= +if [ $timedout -eq 0 ]; then + until [ $verification_done -eq 2 ]; do + + # Wait for signal from test program that verification routine can run. + before=`TimeStamp` + until [ -s $startsignal ]; do + after=`TimeStamp` + timediff=`expr $after - $before` + if [ $timediff -gt $timeout_length ]; then + nerrors=`expr $nerrors + 1` + timedout=1 + break + fi + done + + # Check to see if we timed out looking for the signal before continuing. + if [ $timedout -gt 0 ]; then + echo "timed out waiting for signal from test program (refresh)." + break + fi + + # Read in test routine parameter from signal file, then delete signal file. + param1=`head -n 1 $startsignal` + rm $startsignal + + # Check if we're done with verifications, otherwise run the specified verification. + if [ "$param1" = "VERIFICATION_DONE" ]; then + verification_done=2 + echo "all refresh verification complete" > $endsignal + else + $testdir/flushrefresh $param1 + + # Check for core dump + if [ $? -gt 0 ]; then + nerrors=`expr $nerrors + 1` + fi + + echo "refresh verifiction process done" > $endsignal + fi + + done +fi + +# ============================================ +# Wait for main to finish up, and end testing. +# ============================================ +wait $pid_main +if test $? -ne 0; then + echo flushrefresh had error + nerrors=`expr $nerrors + 1` +fi + +############################################################################### +## Report and exit +############################################################################### + +if test $nerrors -eq 0 ; then + echo "flush/refresh objects tests passed." + if test -z "$HDF5_NOCLEANUP"; then + # delete the test directory + rm -rf flushrefresh_test + fi + exit 0 +else + echo "flush/refresh objects tests failed with $nerrors errors." + exit 1 +fi diff --git a/test/test_libinfo.sh.in b/test/test_libinfo.sh.in new file mode 100644 index 0000000..d9fee48 --- /dev/null +++ b/test/test_libinfo.sh.in @@ -0,0 +1,120 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. + + +# +# Tests for the embedded library information feature. +# Part 1: +# Verify the HDF5 library does contains an exact copy of the content of the +# libhdf5.settings file. +# Part 2: +# If executable is linked with the static hdf5 library (how to determine?), +# verify an executable indeed contains an exact copy of the content of the +# libhdf5.settings file. +# +# Programmer: Albert Cheng +# Sep 18, 2009 + +srcdir=@srcdir@ + +# Determine the configure options of the hdf5 library and executables. + +Shared_Lib=@enable_shared@ +Static_Lib=@enable_static@ +Static_exec=@STATIC_EXEC@ + + +# Print a line-line message left justified in a field of 70 characters. +# +LINEMSG() { + SPACES=" " + echo "Check file $* $SPACES" | cut -c1-70 | tr -d '\012' +} + + +# Print a "SKIP" message +SKIP() { + LINEMSG $* + echo " -SKIP-" +} + +# Function definitions +CHECK_LIBINFO(){ + LINEMSG $1 + # Some systems, like Mac, the strings command inspects library files. Older + # versions of strings may not know newer library format, resulting in + # command errors. Make it read the file as stdin to avoid the problem. + if strings < $1 | grep "SUMMARY OF THE HDF5 CONFIGURATION" > /dev/null; then + echo " PASSED" + else + echo " FAILED" + nerrors=`expr $nerrors + 1` + fi +} + + +# MAIN Body +nerrors=0 +H5_HAVE_EMBEDDED_LIBINFO=`grep '#define H5_HAVE_EMBEDDED_LIBINFO ' ../src/H5pubconf.h` + +# Skip the rest if embedded-libinfo is not enabled. +if [ -z "$H5_HAVE_EMBEDDED_LIBINFO" ]; then + echo "embedded-libinfo is not enabled. Test skipped." + exit 0 +fi + +# The location of HDF library file(s) depends on whether shared lib is +# built too. +if [ -n $Shared_Lib ]; then + h5libdir=../src/.libs + shlib=$(grep dlname ../src/libhdf5.la | sed -e "s/dlname='//" -e "s/'//") +else + h5libdir=../src +fi + +h5libsettings=../src/libhdf5.settings + +# Part 1: +# Verify the HDF5 library does contains an exact copy of the content of the +# libhdf5.settings file. +# Check dynamic library file if built. +if [ x-$Shared_Lib = x-yes ]; then + CHECK_LIBINFO ${h5libdir}/${shlib} +else + SKIP shlib +fi + +# Though rare, libhdf5.a may not have been built. +if [ x-$Static_Lib = x-yes ]; then + CHECK_LIBINFO ${h5libdir}/libhdf5.a +else + SKIP ${h5libdir}/libhdf5.a +fi + +# Check if executables has the lib information only if shared lib is not +# built or static-exec is used. (Don't care static-exec since it affects +# tools binary only.) +if [ x-$Shared_Lib != x-yes ]; then + CHECK_LIBINFO testhdf5 +else + SKIP testhdf5 +fi + + +if [ $nerrors -gt 0 ]; then + echo "***$nerrors errors encountered***" + exit 1 +else + echo "No error encountered" + exit 0 +fi diff --git a/test/test_links_env.sh.in b/test/test_links_env.sh.in new file mode 100644 index 0000000..09074c3 --- /dev/null +++ b/test/test_links_env.sh.in @@ -0,0 +1,42 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Test for external link with environment variable: HDF5_EXT_PREFIX + +srcdir=@srcdir@ + +nerrors=0 + +############################################################################## +############################################################################## +### T H E T E S T S ### +############################################################################## +############################################################################## + +# test for external links with HDF5_EXT_PREFIX +echo "Testing external link with HDF5_EXT_PREFIX" +TEST_NAME=links_env # The test name +TEST_BIN=`pwd`/$TEST_NAME # The path of the test binary +ENVCMD="env HDF5_EXT_PREFIX=.:tmp_links_env" # The environment variable & value +# +# Run the test +echo "$ENVCMD $RUNSERIAL $TEST_BIN" +$ENVCMD $RUNSERIAL $TEST_BIN +exitcode=$? +if [ $exitcode -eq 0 ]; then + echo "Test for HDF5_EXT_PREFIX PASSED" +else + nerrors="`expr $nerrors + 1`" + echo "***Error encountered for HDF5_EXT_PREFIX test***" +fi +exit $nerrors diff --git a/test/test_plugin.sh.in b/test/test_plugin.sh.in new file mode 100644 index 0000000..d958b4b --- /dev/null +++ b/test/test_plugin.sh.in @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# This shell script is for testing filter, VFD, and VOL plugins. +# +srcdir=@srcdir@ +TOP_BUILDDIR=@top_builddir@ + +EXIT_SUCCESS=0 +EXIT_FAILURE=1 + +CP="cp -p" # Use -p to preserve mode,ownership, timestamps +RM="rm -rf" + +nerrors=0 +verbose=yes +exit_code=$EXIT_SUCCESS + +# Test binary names +FILTER_TEST_NAME=filter_plugin +FILTER_TEST_BIN=`pwd`/$FILTER_TEST_NAME + +VFD_TEST_NAME=vfd_plugin +VFD_TEST_BIN=`pwd`/$VFD_TEST_NAME + +VOL_TEST_NAME=vol_plugin +VOL_TEST_BIN=`pwd`/$VOL_TEST_NAME + +# Paths to actual plugins ("libraries" in test directory are just stubs) +FROM_DIR=`pwd`/.libs +case $(uname) in + CYGWIN* ) + NULL_VFD_PLUGIN="$FROM_DIR/cygnull_vfd_plugin*" + NULL_VOL_PLUGIN="$FROM_DIR/cygnull_vol_connector*" + PLUGINS_FOR_DIR1="$FROM_DIR/cygfilter_plugin1* $FROM_DIR/cygfilter_plugin3*" + PLUGINS_FOR_DIR2="$FROM_DIR/cygfilter_plugin2* $FROM_DIR/cygfilter_plugin4*" + ;; + *) + NULL_VFD_PLUGIN="$FROM_DIR/libnull_vfd_plugin*" + NULL_VOL_PLUGIN="$FROM_DIR/libnull_vol_connector*" + PLUGINS_FOR_DIR1="$FROM_DIR/libfilter_plugin1* $FROM_DIR/libfilter_plugin3*" + PLUGINS_FOR_DIR2="$FROM_DIR/libfilter_plugin2* $FROM_DIR/libfilter_plugin4*" + ;; +esac + +# Directories where we'll copy plugins +TEMP_PLUGIN_DIR=temp_plugins +TEMP_FILTER_DIR1=temp_filter_plugin_dir1 +TEMP_FILTER_DIR2=temp_filter_plugin_dir2 + +# Function to print a line-line message left justified in a field of +# 70 characters beginning with the word "Testing". +# +TESTING() { + SPACES=" " + echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +############# +# Main Body # +############# + +# Create plugin directories +test -d $TEMP_PLUGIN_DIR || mkdir -p $TEMP_PLUGIN_DIR +if [ $? != 0 ]; then + echo "Failed to create plugin test directory ($TEMP_PLUGIN_DIR)" + exit $EXIT_FAILURE +fi +test -d $TEMP_FILTER_DIR1 || mkdir -p $TEMP_FILTER_DIR1 +if [ $? != 0 ]; then + echo "Failed to create filter plugin test directory ($TEMP_FILTER_DIR1)" + exit $EXIT_FAILURE +fi +test -d $TEMP_FILTER_DIR2 || mkdir -p $TEMP_FILTER_DIR2 +if [ $? != 0 ]; then + echo "Failed to create filter plugin test directory ($TEMP_FILTER_DIR2)" + exit $EXIT_FAILURE +fi + +# Copy plugins for the tests +$CP $NULL_VFD_PLUGIN $TEMP_PLUGIN_DIR +if [ $? != 0 ]; then + echo "Failed to copy NULL VFD plugin ($NULL_VFD_PLUGIN) to test directory." + exit $EXIT_FAILURE +fi +$CP $NULL_VOL_PLUGIN $TEMP_PLUGIN_DIR +if [ $? != 0 ]; then + echo "Failed to copy NULL VOL plugin ($NULL_VOL_PLUGIN) to test directory." + exit $EXIT_FAILURE +fi +$CP $PLUGINS_FOR_DIR1 $TEMP_FILTER_DIR1 +if [ $? != 0 ]; then + echo "Failed to copy filter plugins ($PLUGINS_FOR_DIR1) to test directory." + exit $EXIT_FAILURE +fi +$CP $PLUGINS_FOR_DIR2 $TEMP_FILTER_DIR2 +if [ $? != 0 ]; then + echo "Failed to copy filter plugins ($PLUGINS_FOR_DIR2) to test directory." + exit $EXIT_FAILURE +fi + +# Set plugin path +ENVCMD="env HDF5_PLUGIN_PATH=${TEMP_PLUGIN_DIR}:${TEMP_FILTER_DIR1}:${TEMP_FILTER_DIR2}" + +# Run the tests +$ENVCMD $FILTER_TEST_BIN +if [ $? != 0 ]; then + nerrors=`expr $nerrors + 1` +fi +$ENVCMD $VFD_TEST_BIN +if [ $? != 0 ]; then + nerrors=`expr $nerrors + 1` +fi +$ENVCMD $VOL_TEST_BIN +if [ $? != 0 ]; then + nerrors=`expr $nerrors + 1` +fi + +# Print results +if test $nerrors -ne 0 ; then + echo "$nerrors errors encountered" + exit_code=$EXIT_FAILURE +else + echo "All plugin tests passed." + exit_code=$EXIT_SUCCESS +fi + +# Clean up temporary files/directories and leave +$RM $TEMP_PLUGIN_DIR $TEMP_FILTER_DIR1 $TEMP_FILTER_DIR2 + +exit $exit_code diff --git a/test/test_swmr.pwsh.in b/test/test_swmr.pwsh.in new file mode 100644 index 0000000..de7a57a --- /dev/null +++ b/test/test_swmr.pwsh.in @@ -0,0 +1,519 @@ +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Tests for the swmr feature. + +$srcdir = '@srcdir@' +$utils_testdir=@H5_UTILS_TEST_BUILDDIR@ +$testdir=@H5_TEST_BUILDDIR@ + +############################################################################### +## test parameters +############################################################################### + +$Nreaders = 5 # number of readers to launch +$Nrdrs_spa = 3 # number of sparse readers to launch +$Nrecords = 200000 # number of records to write +$Nrecs_rem = 40000 # number of times to shrink +$Nrecs_spa = 20000 # number of records to write in the sparse test +$Nsecs_add = 5 # number of seconds per read interval +$Nsecs_rem = 3 # number of seconds per read interval +$Nsecs_addrem = 8 # number of seconds per read interval +$nerrors = 0 + +############################################################################### +## definitions for message file to coordinate test runs +############################################################################### +$WRITER_MESSAGE = 'SWMR_WRITER_MESSAGE' # The message file created by writer that the open is complete + # This should be the same as the define in "test/swmr_common.h" +$MESSAGE_TIMEOUT = 300 # Message timeout length in secs + # This should be the same as the define in "test/h5test.h" + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +function Test-WithSpaces { + $SPACES=' ' + #Write-Output "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# To wait for the writer message file or till the maximum # of seconds is reached +# $Message is the message file to wait for +# This performs similar function as the routine h5_wait_message() in test/h5test.c +function Wait-Message { + [CmdletBinding()] + param ( + [Parameter(Mandatory, ValueFromPipeline)] + [string]$TestPath, # Get the path of the message file to wait for + [Parameter(Mandatory, ValueFromPipeline)] + [string]$Message # Get the name of the message file to wait for + ) + + PROCESS { + $message = Join-Path -Path $TestPath -ChildPath $Message + $t0 = Get-Date # Get current time + $t1 = $t0 + $difft = New-Timespan -Start $t0 -End $t1 # Initialize the time difference + $mexist = 0 # Indicate whether the message file is found + while ($difft.TotalSeconds -lt $MESSAGE_TIMEOUT) { # Loop till message times out + $t1 = Get-Date # Get current time in seconds + $difft = New-Timespan -Start $t0 -End $t1 # Calculate the time difference + #Write-Output "Check for $message : time=$difft" + if ([System.IO.File]::Exists($message)) { # If message file is found: + $mexist = 1 # indicate the message file is found + Remove-Item $message # remove the message file + break # get out of the while loop + } + } + if ($mexist -eq 0) { + # Issue warning that the writer message file is not found, continue with launching the reader(s) + Write-Warning -Message "$WRITER_MESSAGE is not found after waiting $MESSAGE_TIMEOUT seconds" + } + else { + Write-Output "$WRITER_MESSAGE is found" + } + } +} + +############################################################################### +## Main +## +## Modifications: +## Vailin Choi; July 2013 +## Add waiting of message file before launching the reader(s). +## Due to the implementation of file locking, coordination +## is needed in file opening for the writer/reader tests +## to proceed as expected. +## +############################################################################### + +# Check to see if the VFD specified by the HDF5_DRIVER environment variable +# supports SWMR. +$testprog = Join-Path -Path $utils_testdir -ChildPath swmr_check_compat_vfd.exe +$rp = Start-Process -FilePath $testprog -PassThru -Wait -NoNewWindow +if ($rp.ExitCode -ne 0) { + Write-Output "" + Write-Output "The VFD specified by the HDF5_DRIVER environment variable" + Write-Output "does not support SWMR." + Write-Output "" + Write-Output "SWMR acceptance tests skipped" + Write-Output "" + exit 0 +} + +Set-Location -Path swmr_test +$testdir = Join-Path -Path $testdir -ChildPath swmr_test + +# Loop over index types +$IndexTypeLoop = '-i ea', '-i b2' +foreach ($index_type in $IndexTypeLoop) { + # Try with and without compression + $CompressLoop = '', '-c 5' + foreach ($compress in $CompressLoop) { + Write-Output "" + Write-Output "*******************************************************************************" + Write-Output "** Loop testing parameters: $index_type $compress" + Write-Output "*******************************************************************************" + Write-Output "" + Write-Output "" + Write-Output "###############################################################################" + Write-Output "## Generator test" + Write-Output "###############################################################################" + # Launch the Generator without SWMR_WRITE + Write-Output "launch the swmr_generator" + $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -ArgumentList "$compress $index_type" -PassThru -Wait + if ($rp.ExitCode -ne 0) { + Write-Warning "generator had error" + $nerrors += 1 + } + + # Launch the Generator with SWMR_WRITE + Write-Output "launch the swmr_generator with SWMR_WRITE" + $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -ArgumentList "-s $compress $index_type" -PassThru -Wait + if ($rp.ExitCode -ne 0) { + Write-Warning "generator had error" + $nerrors += 1 + } + + # Check for error and exit if one occurred + #Write-Output "nerrors=$nerrors" + if ($nerrors -ne 0) { + Write-Warning "SWMR tests failed with $nerrors errors." + exit 1 + } + + Write-Output "" + Write-Output "###############################################################################" + Write-Output "## Use H5Fstart_swmr_write() to enable SWMR writing mode" + Write-Output "###############################################################################" + + # Remove any possible writer message file before launching writer + if ([System.IO.File]::Exists($WRITER_MESSAGE)) { # If message file is found + Remove-Item $WRITER_MESSAGE + } + # + # Launch the Writer + Write-Output "launch the swmr_start_writer" + $seed = "" # Put -r command here + $rp = Start-Process -FilePath $testdir/swmr_start_write.exe -NoNewWindow -PassThru -ArgumentList "$compress $index_type $Nrecords $seed" 2>&1 |tee swmr_writer.out + $pid_writer = $rp.id + #Write-Output "pid_writer=$pid_writer" + + # Wait for message from writer process before starting reader(s) + Wait-Message $testdir $WRITER_MESSAGE + + # + # Launch the Readers + #declare -a seeds = ( ... ) + Write-Output "launch $Nreaders swmr_readers" + $pid_readers = @() + $n = 0 + while ($n -lt $Nreaders) { + #seed = "-r ${seeds[$n]}" + $seed = "" + $rp = Start-Process -FilePath $testdir/swmr_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_add $seed" 2>&1 | tee swmr_reader.out.$n + $pid_readers += $rp.id + $n += 1 + } + #Write-Output "pid_readers=$pid_readers" + + # Collect exit code of the readers first because they usually finish + # before the writer. + foreach ($xpid in $pid_readers) { + #Write-Output "checked reader $xpid" + $result = Wait-Process -Id $xpid + if ($result.ExitCode -ne 0) { + Write-Warning "reader had error" + $nerrors += 1 + } + } + + # Collect exit code of the writer + #Write-Output "checked writer $pid_writer" + $result = Wait-Process -Id $pid_writer + if ($result.ExitCode -ne 0) { + Write-Warning "writer had error" + $nerrors += 1 + } + + # Check for error and exit if one occurred + #Write-Output "nerrors=$nerrors" + if ($nerrors -ne 0) { + Write-Warning "SWMR tests failed with $nerrors errors." + Write-Warning "(Writer and reader output preserved)" + exit 1 + } + + # Clean up output files + Remove-Item swmr_writer.out + Remove-Item swmr_reader.out.* + + Write-Output "" + Write-Output "###############################################################################" + Write-Output "## Writer test - test expanding the dataset" + Write-Output "###############################################################################" + + # Launch the Generator + Write-Output "launch the swmr_generator" + $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -PassThru -Wait -ArgumentList "-s $compress $index_type" + if ($rp.ExitCode -ne 0) { + Write-Warning "generator had error" + $nerrors += 1 + } + + # Remove any possible writer message file before launching writer + Remove-Item $WRITER_MESSAGE + # + # Launch the Writer + Write-Output "launch the swmr_writer" + $seed = "" # Put -r command here + $rp = Start-Process -FilePath $testdir/swmr_writer.exe -NoNewWindow -PassThru -ArgumentList "-o $Nrecords $seed" 2>&1 |tee swmr_writer.out + $pid_writer = $rp.id + #Write-Output "pid_writer=$pid_writer" + + # Wait for message from writer process before starting reader(s) + Wait-Message $testdir $WRITER_MESSAGE + # + # Launch the Readers + #declare -a seeds = ( ... ) + Write-Output "launch $Nreaders swmr_readers" + $pid_readers = @() + $n = 0 + while ($n -lt $Nreaders) { + #seed = "-r ${seeds[$n]}" + $seed = "" + $rp = Start-Process -FilePath $testdir/swmr_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_add $seed" 2>&1 |tee swmr_reader.out.$n + $pid_readers += $rp.id + $n += 1 + } + #Write-Output "pid_readers=$pid_readers" + + # Collect exit code of the readers first because they usually finish + # before the writer. + foreach ($xpid in $pid_readers) { + #Write-Output "checked reader $xpid" + $result = Wait-Process -Id $xpid + if ($result.ExitCode -ne 0) { + Write-Warning "reader had error" + $nerrors += 1 + } + } + + # Collect exit code of the writer + #Write-Output "checked writer $pid_writer" + $result = Wait-Process -Id $pid_writer + if ($result.ExitCode -ne 0) { + Write-Warning "writer had error" + $nerrors += 1 + } + + # Check for error and exit if one occurred + #Write-Output "nerrors=$nerrors" + if ($nerrors -ne 0) { + Write-Warning "SWMR tests failed with $nerrors errors." + Write-Warning "(Writer and reader output preserved)" + exit 1 + } + + # Clean up output files + Remove-Item swmr_writer.out + Remove-Item swmr_reader.out.* + + Write-Output "" + Write-Output "###############################################################################" + Write-Output "## Remove test - test shrinking the dataset" + Write-Output "###############################################################################" + + # Remove any possible writer message file before launching writer + Remove-Item $WRITER_MESSAGE + # Launch the Remove Writer + Write-Output "launch the swmr_remove_writer" + $seed = "" # Put -r command here + $rp = Start-Process -FilePath $testdir/swmr_remove_writer.exe -NoNewWindow -PassThru -ArgumentList "-o $Nrecs_rem $seed" 2>&1 |tee swmr_writer.out + $pid_writer = $rp.id + #Write-Output "pid_writer=$pid_writer" + + # Wait for message from writer process before starting reader(s) + Wait-Message $testdir $WRITER_MESSAGE + # + # Launch the Remove Readers + #declare -a seeds = ( ... ) + $n = 0 + $pid_readers = @() + Write-Output "launch $Nreaders swmr_remove_readers" + while ($n -lt $Nreaders) { + #seed = "-r ${seeds[$n]}" + $seed = "" + $rp = Start-Process -FilePath $testdir/swmr_remove_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_rem $seed" 2>&1 |tee swmr_reader.out.$n + $pid_readers += $rp.id + $n += 1 + } + #Write-Output "pid_readers=$pid_readers" + + # Collect exit code of the readers first because they usually finish + # before the writer. + foreach ($xpid in $pid_readers) { + #Write-Output "checked reader $xpid" + $result = Wait-Process -Id $xpid + if ($result.ExitCode -ne 0) { + Write-Warning "reader had error" + $nerrors += 1 + } + } + + # Collect exit code of the writer + #Write-Output "checked writer $pid_writer" + $result = Wait-Process -Id $pid_writer + if ($result.ExitCode -ne 0) { + Write-Warning "writer had error" + $nerrors += 1 + } + + # Check for error and exit if one occurred + #Write-Output "nerrors=$nerrors" + if ($nerrors -ne 0) { + Write-Warning "SWMR tests failed with $nerrors errors." + Write-Warning "(Writer and reader output preserved)" + exit 1 + } + + # Clean up output files + Remove-Item swmr_writer.out + Remove-Item swmr_reader.out.* + + Write-Output "" + Write-Output "###############################################################################" + Write-Output "## Add/remove test - randomly grow or shrink the dataset" + Write-Output "###############################################################################" + + # Launch the Generator + Write-Output "launch the swmr_generator" + $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow-PassThru -Wait -ArgumentList "$compress $index_type" + if ($rp.ExitCode -ne 0) { + Write-Warning "generator had error" + $nerrors += 1 + } + + # Launch the Writer (not in parallel - just to rebuild the datasets) + Write-Output "launch the swmr_writer" + $seed = "" # Put -r command here + $rp = Start-Process -FilePath $testdir/swmr_writer.exe -NoNewWindow -PassThru -Wait -ArgumentList "$Nrecords $seed" + if ($rp.ExitCode -ne 0) { + Write-Warning "writer had error" + $nerrors += 1 + } + + # Remove any possible writer message file before launching writer + Remove-Item $WRITER_MESSAGE + # + # Launch the Add/Remove Writer + Write-Output "launch the swmr_addrem_writer" + $seed = "" # Put -r command here + $rp = Start-Process -FilePath $testdir/swmr_addrem_writer.exe -NoNewWindow -PassThru -ArgumentList "$Nrecords $seed" 2>&1 |tee swmr_writer.out + $pid_writer = $rp.id + #Write-Output "pid_writer=$pid_writer" + + # Wait for message from writer process before starting reader(s) + Wait-Message $testdir $WRITER_MESSAGE + # + # Launch the Add/Remove Readers + #declare -a seeds = ( ... ) + $n = 0 + $pid_readers = @() + Write-Output "launch $Nreaders swmr_remove_readers" + while ($n -lt $Nreaders) { + #seed = "-r ${seeds[$n]}" + $seed = "" + $rp = Start-Process -FilePath $testdir/swmr_remove_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_addrem $seed" 2>&1 |tee swmr_reader.out.$n + $pid_readers += $rp.id + $n += 1 + } + #Write-Output "pid_readers=$pid_readers" + + # Collect exit code of the readers first because they usually finish + # before the writer. + foreach ($xpid in $pid_readers) { + #Write-Output "checked reader $xpid" + $result = Wait-Process -Id $xpid + if ($result.ExitCode -ne 0) { + Write-Warning "reader had error" + $nerrors += 1 + } + } + + # Collect exit code of the writer + #Write-Output "checked writer $pid_writer" + $result = Wait-Process -Id $pid_writer + if ($result.ExitCode -ne 0) { + Write-Warning "writer had error" + $nerrors += 1 + } + + # Check for error and exit if one occurred + #Write-Output "nerrors=$nerrors" + if ($nerrors -ne 0) { + Write-Warning "SWMR tests failed with $nerrors errors." + Write-Warning "(Writer and reader output preserved)" + exit 1 + } + + # Clean up output files + Remove-Item swmr_writer.out + Remove-Item swmr_reader.out.* + + Write-Output "" + Write-Output "###############################################################################" + Write-Output "## Sparse writer test - test writing to random locations in the dataset" + Write-Output "###############################################################################" + + # Launch the Generator + # NOTE: Random seed is shared between readers and writers and is + # created by the generator. + Write-Output "launch the swmr_generator" + $seed = "" # Put -r command here + $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -PassThru -Wait -ArgumentList "$compress $index_type $seed" + if ($rp.ExitCode -ne 0) { + Write-Warning "generator had error" + $nerrors += 1 + } + + # Remove any possible writer message file before launching writer + Remove-Item $WRITER_MESSAGE + # Launch the Sparse writer + Write-Output "launch the swmr_sparse_writer" + $rp = Start-Process -FilePath $testdir/swmr_sparse_writer.exe -NoNewWindow -PassThru -ArgumentList "$Nrecs_spa" 2>&1 |tee swmr_writer.out + $pid_writer = $rp.Id + #Write-Output "pid_writer=$pid_writer" + + # Wait for message from writer process before starting reader(s) + Wait-Message $testdir $WRITER_MESSAGE + # + # Launch the Sparse readers + $n = 0 + $pid_readers = @() + Write-Output "launch $Nrdrs_spa swmr_sparse_readers" + while ($n -lt $Nrdrs_spa) { + # The sparse reader spits out a LOT of data so it's set to 'quiet' + $rp = Start-Process -FilePath $testdir/swmr_sparse_reader.exe -NoNewWindow -PassThru -ArgumentList "-q $Nrecs_spa" 2>&1 |tee swmr_reader.out.$n + $pid_readers += $rp.id + $n += 1 + } + #Write-Output "pid_readers=$pid_readers" + + # Collect exit code of the writer + #Write-Output "checked writer $pid_writer" + $result = Wait-Process -Id $pid_writer + if ($result.ExitCode -ne 0) { + Write-Warning "writer had error" + $nerrors += 1 + } + + # Collect exit code of the readers + foreach ($xpid in $pid_readers) { + #Write-Output "checked reader $xpid" + $result = Wait-Process -Id $xpid + if ($result.ExitCode -ne 0) { + Write-Warning "reader had error" + $nerrors += 1 + } + } + + # Check for error and exit if one occurred + #Write-Output "nerrors=$nerrors" + if ($nerrors -ne 0) { + Write-Warning "SWMR tests failed with $nerrors errors." + Write-Warning "(Writer and reader output preserved)" + exit 1 + } + + # Clean up output files + Remove-Item swmr_writer.out + Remove-Item swmr_reader.out.* + } +} + +############################################################################### +## Report and exit +############################################################################### +cd .. +#Write-Output "nerrors=$nerrors" +if ($nerrors -eq 0) { + Write-Output "SWMR tests passed." +# if test -z "$HDF5_NOCLEANUP"; then +# # delete the test directory +# Remove-Item swmr_test -Recurse +# fi + exit 0 +} +else { + Write-Warning "SWMR tests failed with $nerrors errors." + exit 1 +} + diff --git a/test/test_swmr.sh.in b/test/test_swmr.sh.in new file mode 100644 index 0000000..771fe4a --- /dev/null +++ b/test/test_swmr.sh.in @@ -0,0 +1,563 @@ +#! /bin/bash +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Tests for the swmr feature. +# +# Created: +# Albert Cheng, 2009/07/22 + +srcdir=@srcdir@ +utils_testdir=@abs_top_builddir@/@H5_UTILS_TEST_BUILDDIR@ +testdir=@abs_top_builddir@/@H5_TEST_BUILDDIR@ + +############################################################################### +## test parameters +############################################################################### + +Nreaders=5 # number of readers to launch +Nrdrs_spa=3 # number of sparse readers to launch +Nrecords=200000 # number of records to write +Nrecs_rem=40000 # number of times to shrink +Nrecs_spa=20000 # number of records to write in the sparse test +Nsecs_add=5 # number of seconds per read interval +Nsecs_rem=3 # number of seconds per read interval +Nsecs_addrem=8 # number of seconds per read interval +nerrors=0 + +############################################################################### +## definitions for message file to coordinate test runs +############################################################################### +WRITER_MESSAGE=SWMR_WRITER_MESSAGE # The message file created by writer that the open is complete + # This should be the same as the define in "test/swmr_common.h" +MESSAGE_TIMEOUT=300 # Message timeout length in secs + # This should be the same as the define in "test/h5test.h" + +############################################################################### +## short hands and function definitions +############################################################################### +DPRINT=: # Set to "echo Debug:" for debugging printing, + # else ":" for noop. +IFDEBUG=: # Set to null to turn on debugging, else ":" for noop. + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +TESTING() { + SPACES=" " + echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# To wait for the writer message file or till the maximum # of seconds is reached +# $1 is the message file to wait for +# This performs similar function as the routine h5_wait_message() in test/h5test.c +WAIT_MESSAGE() { + message=$1 # Get the name of the message file to wait for + t0=`date +%s` # Get current time in seconds + difft=0 # Initialize the time difference + mexist=0 # Indicate whether the message file is found + while [ $difft -lt $MESSAGE_TIMEOUT ] ; # Loop till message times out + do + t1=`date +%s` # Get current time in seconds + difft=`expr $t1 - $t0` # Calculate the time difference + if [ -e $message ]; then # If message file is found: + mexist=1 # indicate the message file is found + rm $message # remove the message file + break # get out of the while loop + fi + done; + if test $mexist -eq 0; then + # Issue warning that the writer message file is not found, continue with launching the reader(s) + echo warning: $WRITER_MESSAGE is not found after waiting $MESSAGE_TIMEOUT seconds + else + echo $WRITER_MESSAGE is found + fi +} + +############################################################################### +## Main +## +## Modifications: +## Vailin Choi; July 2013 +## Add waiting of message file before launching the reader(s). +## Due to the implementation of file locking, coordination +## is needed in file opening for the writer/reader tests +## to proceed as expected. +## +############################################################################### + +# Check to see if the VFD specified by the HDF5_DRIVER environment variable +# supports SWMR. +$utils_testdir/swmr_check_compat_vfd +rc=$? +if [ $rc -ne 0 ] ; then + echo + echo "The VFD specified by the HDF5_DRIVER environment variable" + echo "does not support SWMR." + echo + echo "SWMR acceptance tests skipped" + echo + exit 0 +fi + +# Parse options (none accepted at this time) +while [ $# -gt 0 ]; do + case "$1" in + *) # unknown option + echo "$0: Unknown option ($1)" + exit 1 + ;; + esac +done + +# HDF5 has several tests that create and delete signal files to communicate +# between processes, and it seems that even though the names of the files are +# different, occasionally the wrong file is deleted, interrupting the flow of +# the test. Running each of these tests in its own directory should eliminate +# the problem. +mkdir swmr_test +for FILE in swmr*; do + case "$FILE" in + *.o) continue ;; ## don't copy the .o files + esac + if test -f "$FILE" ; then + cp $FILE swmr_test + fi +done + +# With the --disable-shared option, swmr program files are built in the test +# directory, otherwise they are in test/.libs with a corresponding wrapper +# script in the test directory. The programs or wrapper scripts in test should +# always be copied, swmr files in .libs should be copied only if they exists. +if [ -f .libs/swmr ]; then + mkdir swmr_test/.libs + for FILE in .libs/swmr*; do + case "$FILE" in + *.o) continue ;; ## don't copy the .o files + esac + if test -f "$FILE" ; then + cp $FILE swmr_test/.libs + fi + done +fi + +cd swmr_test + + +# Loop over index types +for index_type in "-i ea" "-i b2" +do + # Try with and without compression + for compress in "" "-c 5" + do + echo + echo "*******************************************************************************" + echo "** Loop testing parameters: $index_type $compress" + echo "*******************************************************************************" + echo + echo + echo "###############################################################################" + echo "## Generator test" + echo "###############################################################################" + # Launch the Generator without SWMR_WRITE + echo launch the swmr_generator + $testdir/swmr_generator $compress $index_type + if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` + fi + + # Launch the Generator with SWMR_WRITE + echo launch the swmr_generator with SWMR_WRITE + $testdir/swmr_generator -s $compress $index_type + if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` + fi + + # Check for error and exit if one occurred + $DPRINT nerrors=$nerrors + if test $nerrors -ne 0 ; then + echo "SWMR tests failed with $nerrors errors." + exit 1 + fi + + echo + echo "###############################################################################" + echo "## Use H5Fstart_swmr_write() to enable SWMR writing mode" + echo "###############################################################################" + + # Remove any possible writer message file before launching writer + rm -f $WRITER_MESSAGE + # + # Launch the Writer + echo launch the swmr_start_writer + seed="" # Put -r command here + $testdir/swmr_start_write $compress $index_type $Nrecords $seed 2>&1 |tee swmr_writer.out & + pid_writer=$! + $DPRINT pid_writer=$pid_writer + + # Wait for message from writer process before starting reader(s) + WAIT_MESSAGE $WRITER_MESSAGE + + # + # Launch the Readers + #declare -a seeds=( ... ) + echo launch $Nreaders swmr_readers + pid_readers="" + n=0 + while [ $n -lt $Nreaders ]; do + #seed="-r ${seeds[$n]}" + seed="" + $testdir/swmr_reader $Nsecs_add $seed 2>&1 |tee swmr_reader.out.$n & + pid_readers="$pid_readers $!" + n=`expr $n + 1` + done + $DPRINT pid_readers=$pid_readers + $IFDEBUG ps + + # Collect exit code of the readers first because they usually finish + # before the writer. + for xpid in $pid_readers; do + $DPRINT checked reader $xpid + wait $xpid + if test $? -ne 0; then + echo reader had error + nerrors=`expr $nerrors + 1` + fi + done + + # Collect exit code of the writer + $DPRINT checked writer $pid_writer + wait $pid_writer + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi + + # Check for error and exit if one occurred + $DPRINT nerrors=$nerrors + if test $nerrors -ne 0 ; then + echo "SWMR tests failed with $nerrors errors." + echo "(Writer and reader output preserved)" + exit 1 + fi + + # Clean up output files + rm -f swmr_writer.out + rm -f swmr_reader.out.* + + echo + echo "###############################################################################" + echo "## Writer test - test expanding the dataset" + echo "###############################################################################" + + # Launch the Generator + echo launch the swmr_generator + $testdir/swmr_generator -s $compress $index_type + if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` + fi + + # Remove any possible writer message file before launching writer + rm -f $WRITER_MESSAGE + # + # Launch the Writer + echo launch the swmr_writer + seed="" # Put -r command here + $testdir/swmr_writer -o $Nrecords $seed 2>&1 |tee swmr_writer.out & + pid_writer=$! + $DPRINT pid_writer=$pid_writer + + # Wait for message from writer process before starting reader(s) + WAIT_MESSAGE $WRITER_MESSAGE + # + # Launch the Readers + #declare -a seeds=( ... ) + echo launch $Nreaders swmr_readers + pid_readers="" + n=0 + while [ $n -lt $Nreaders ]; do + #seed="-r ${seeds[$n]}" + seed="" + $testdir/swmr_reader $Nsecs_add $seed 2>&1 |tee swmr_reader.out.$n & + pid_readers="$pid_readers $!" + n=`expr $n + 1` + done + $DPRINT pid_readers=$pid_readers + $IFDEBUG ps + + # Collect exit code of the readers first because they usually finish + # before the writer. + for xpid in $pid_readers; do + $DPRINT checked reader $xpid + wait $xpid + if test $? -ne 0; then + echo reader had error + nerrors=`expr $nerrors + 1` + fi + done + + # Collect exit code of the writer + $DPRINT checked writer $pid_writer + wait $pid_writer + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi + + # Check for error and exit if one occurred + $DPRINT nerrors=$nerrors + if test $nerrors -ne 0 ; then + echo "SWMR tests failed with $nerrors errors." + echo "(Writer and reader output preserved)" + exit 1 + fi + + # Clean up output files + rm -f swmr_writer.out + rm -f swmr_reader.out.* + + echo + echo "###############################################################################" + echo "## Remove test - test shrinking the dataset" + echo "###############################################################################" + + # Remove any possible writer message file before launching writer + rm -f $WRITER_MESSAGE + # Launch the Remove Writer + echo launch the swmr_remove_writer + seed="" # Put -r command here + $testdir/swmr_remove_writer -o $Nrecs_rem $seed 2>&1 |tee swmr_writer.out & + pid_writer=$! + $DPRINT pid_writer=$pid_writer + + # Wait for message from writer process before starting reader(s) + WAIT_MESSAGE $WRITER_MESSAGE + # + # Launch the Remove Readers + #declare -a seeds=( ... ) + n=0 + pid_readers="" + echo launch $Nreaders swmr_remove_readers + while [ $n -lt $Nreaders ]; do + #seed="-r ${seeds[$n]}" + seed="" + $testdir/swmr_remove_reader $Nsecs_rem $seed 2>&1 |tee swmr_reader.out.$n & + pid_readers="$pid_readers $!" + n=`expr $n + 1` + done + $DPRINT pid_readers=$pid_readers + $IFDEBUG ps + + # Collect exit code of the readers first because they usually finish + # before the writer. + for xpid in $pid_readers; do + $DPRINT checked reader $xpid + wait $xpid + if test $? -ne 0; then + echo reader had error + nerrors=`expr $nerrors + 1` + fi + done + + # Collect exit code of the writer + $DPRINT checked writer $pid_writer + wait $pid_writer + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi + + # Check for error and exit if one occurred + $DPRINT nerrors=$nerrors + if test $nerrors -ne 0 ; then + echo "SWMR tests failed with $nerrors errors." + echo "(Writer and reader output preserved)" + exit 1 + fi + + # Clean up output files + rm -f swmr_writer.out + rm -f swmr_reader.out.* + + echo + echo "###############################################################################" + echo "## Add/remove test - randomly grow or shrink the dataset" + echo "###############################################################################" + + # Launch the Generator + echo launch the swmr_generator + $testdir/swmr_generator $compress $index_type + if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` + fi + + # Launch the Writer (not in parallel - just to rebuild the datasets) + echo launch the swmr_writer + seed="" # Put -r command here + $testdir/swmr_writer $Nrecords $seed + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi + + # Remove any possible writer message file before launching writer + rm -f $WRITER_MESSAGE + # + # Launch the Add/Remove Writer + echo launch the swmr_addrem_writer + seed="" # Put -r command here + $testdir/swmr_addrem_writer $Nrecords $seed 2>&1 |tee swmr_writer.out & + pid_writer=$! + $DPRINT pid_writer=$pid_writer + + # Wait for message from writer process before starting reader(s) + WAIT_MESSAGE $WRITER_MESSAGE + # + # Launch the Add/Remove Readers + #declare -a seeds=( ... ) + n=0 + pid_readers="" + echo launch $Nreaders swmr_remove_readers + while [ $n -lt $Nreaders ]; do + #seed="-r ${seeds[$n]}" + seed="" + $testdir/swmr_remove_reader $Nsecs_addrem $seed 2>&1 |tee swmr_reader.out.$n & + pid_readers="$pid_readers $!" + n=`expr $n + 1` + done + $DPRINT pid_readers=$pid_readers + $IFDEBUG ps + + # Collect exit code of the readers first because they usually finish + # before the writer. + for xpid in $pid_readers; do + $DPRINT checked reader $xpid + wait $xpid + if test $? -ne 0; then + echo reader had error + nerrors=`expr $nerrors + 1` + fi + done + + # Collect exit code of the writer + $DPRINT checked writer $pid_writer + wait $pid_writer + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi + + # Check for error and exit if one occurred + $DPRINT nerrors=$nerrors + if test $nerrors -ne 0 ; then + echo "SWMR tests failed with $nerrors errors." + echo "(Writer and reader output preserved)" + exit 1 + fi + + # Clean up output files + rm -f swmr_writer.out + rm -f swmr_reader.out.* + + echo + echo "###############################################################################" + echo "## Sparse writer test - test writing to random locations in the dataset" + echo "###############################################################################" + + # Launch the Generator + # NOTE: Random seed is shared between readers and writers and is + # created by the generator. + echo launch the swmr_generator + seed="" # Put -r command here + $testdir/swmr_generator $compress $index_type $seed + if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` + fi + + # Remove any possible writer message file before launching writer + rm -f $WRITER_MESSAGE + # Launch the Sparse writer + echo launch the swmr_sparse_writer + nice -n 20 $testdir/swmr_sparse_writer $Nrecs_spa 2>&1 |tee swmr_writer.out & + pid_writer=$! + $DPRINT pid_writer=$pid_writer + + # Wait for message from writer process before starting reader(s) + WAIT_MESSAGE $WRITER_MESSAGE + # + # Launch the Sparse readers + n=0 + pid_readers="" + echo launch $Nrdrs_spa swmr_sparse_readers + while [ $n -lt $Nrdrs_spa ]; do + # The sparse reader spits out a LOT of data so it's set to 'quiet' + $testdir/swmr_sparse_reader -q $Nrecs_spa 2>&1 |tee swmr_reader.out.$n & + pid_readers="$pid_readers $!" + n=`expr $n + 1` + done + $DPRINT pid_readers=$pid_readers + $IFDEBUG ps + + # Collect exit code of the writer + $DPRINT checked writer $pid_writer + wait $pid_writer + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi + + # Collect exit code of the readers + for xpid in $pid_readers; do + $DPRINT checked reader $xpid + wait $xpid + if test $? -ne 0; then + echo reader had error + nerrors=`expr $nerrors + 1` + fi + done + + # Check for error and exit if one occurred + $DPRINT nerrors=$nerrors + if test $nerrors -ne 0 ; then + echo "SWMR tests failed with $nerrors errors." + echo "(Writer and reader output preserved)" + exit 1 + fi + + # Clean up output files + rm -f swmr_writer.out + rm -f swmr_reader.out.* + done +done + +############################################################################### +## Report and exit +############################################################################### +cd .. +$DPRINT nerrors=$nerrors +if test $nerrors -eq 0 ; then + echo "SWMR tests passed." + if test -z "$HDF5_NOCLEANUP"; then + # delete the test directory + rm -rf swmr_test + fi + exit 0 +else + echo "SWMR tests failed with $nerrors errors." + exit 1 +fi + diff --git a/test/test_use_cases.sh.in b/test/test_use_cases.sh.in new file mode 100644 index 0000000..fd30afd --- /dev/null +++ b/test/test_use_cases.sh.in @@ -0,0 +1,219 @@ +#! /bin/bash +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Tests the use cases of swmr features. +# +# Created: +# Albert Cheng, 2013/06/01. +# Modified: +# + +# This is work in progress. +# For now, it shows how to run the test cases programs. It only verifies the +# exit codes are okay (0). + +############################################################################### +## test variables +############################################################################### + +# Number of errors encountered during test run. +nerrors=0 + +# Define variables +verbose=yes + +############################################################################### +## Main +############################################################################### +srcdir=@srcdir@ +utils_testdir=@abs_top_builddir@/@H5_UTILS_TEST_BUILDDIR@ +testdir=@abs_top_builddir@/@H5_TEST_BUILDDIR@ + +# Check to see if the VFD specified by the HDF5_DRIVER environment variable +# supports SWMR. +$utils_testdir/swmr_check_compat_vfd +rc=$? +if [[ $rc != 0 ]] ; then + echo + echo "The VFD specified by the HDF5_DRIVER environment variable" + echo "does not support SWMR" + echo + echo "SWMR use case tests skipped" + echo + exit 0 +fi + +# Define symbols +EXIT_SUCCESS=0 +EXIT_FAILURE=1 +EXIT_VALUE=$EXIT_SUCCESS # Default all tests succeed +RESULT_PASSED=" PASSED" +RESULT_FAILED="*FAILED*" +RESULT_SKIP="-SKIP-" +USECASES_PROGRAMS="use_append_chunk use_append_mchunks" +TESTNAME="Use Case" + +# Source in the output filter function definitions. +. $srcdir/../bin/output_filter.sh + +# Define functions +# Print a line-line message left justified in a field of 72 characters. +# Results can be " PASSED", "*FAILED*", "-SKIP-", up to 8 characters +# wide. +# SPACES should be at least 71 spaces. ($* + ' ' + 71 + 8 >= 80) +# +TESTING() { + SPACES=" " + echo "$* $SPACES" | cut -c1-72 | tr -d '\012' +} + +# Run a test and print PASS or *FAIL*. If a test fails then increment +# the `nerrors' global variable and (if $verbose is set) display the +# difference between the actual output and the expected output. The +# expected output is given as the first argument to this function and +# the actual output file is calculated by replacing the `.ddl' with +# `.out'. The actual output is not removed if $HDF5_NOCLEANUP has a +# non-zero value. +# ADD_H5_TEST +TOOLTEST() { + program=$1 + shift + + actual="$program.out" + actual_err="$program.err" + actual_sav=${actual}-sav + actual_err_sav=${actual_err}-sav + + # Run test. + TESTING $program $@ + ( + $RUNSERIAL $testdir/$program "$@" + ) >$actual 2>$actual_err + exit_code=$? + + # save actual and actual_err in case they are needed later. + cp $actual $actual_sav + STDOUT_FILTER $actual + cp $actual_err $actual_err_sav + STDERR_FILTER $actual_err + cat $actual_err >> $actual + + if [ $exit_code -eq 0 ];then + echo "$RESULT_PASSED" + test yes = "$verbose" && sed 's/^/ /' < $actual + else + echo "$RESULT_FAILED" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && sed 's/^/ /' < $actual + fi + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $actual $actual_err $actual_sav $actual_err_sav $actual_ext + fi +} + +# HDF5 has several tests that create and delete signal files to communicate +# between processes, and it seems that even though the names of the files are +# different, occasionally the wrong file is deleted, interrupting the flow of +# the test. Running each of these tests in its own directory should eliminate +# the problem. +mkdir usecases_test +cp $testdir/twriteorder usecases_test +for FILE in use_*; do + case "$FILE" in + *.o) continue ;; ## don't copy the .o files + esac + if test -f "$FILE" ; then + cp $FILE usecases_test + fi +done + +# With the --disable-shared option, swmr program files are built in the test +# directory, otherwise they are in test/.libs with a corresponding wrapper +# script in the test directory. The programs or wrapper scripts in test should +# always be copied, swmr files in .libs should be copied only if they exists. +if [ -f .libs/use_append_chunk ]; then + mkdir usecases_test/.libs + for FILE in .libs/use_*; do + case "$FILE" in + *.o) continue ;; ## don't copy the .o files + esac + if test -f "$FILE" ; then + cp $FILE usecases_test/.libs + fi + done + cp .libs/twriteorder usecases_test/.libs +fi + +cd usecases_test + + +# run tests for H5Odisable_mdc_flushes/H5Oenable_mdc_flushes/H5Oare_mdc_flushes_disabled here temporary +USECORK=use_disable_mdc_flushes +for p in $USECORK; do + TOOLTEST $p + TOOLTEST $p -y 3 + TOOLTEST $p -n 3000 + TOOLTEST $p -n 5000 +done + +# run write order test here temporary +WRITEORDER=twriteorder +for p in $WRITEORDER; do + TOOLTEST $p + TOOLTEST $p -b 1000 + TOOLTEST $p -p 3000 + TOOLTEST $p -n 2000 + TOOLTEST $p -l w + TOOLTEST $p -l r +done + +# Report test results +if test $nerrors -eq 0 ; then + echo "$WRITEORDER test passed." +else + echo "$WRITEORDER test failed with $nerrors errors." + EXIT_VALUE=$EXIT_FAILURE + nerrors=0 # reset nerror for the regular tests below. +fi + +# main body +for p in $USECASES_PROGRAMS; do + TOOLTEST $p + TOOLTEST $p -z 256 + tmpfile=/tmp/datatfile.$$ + TOOLTEST $p -f $tmpfile; rm -f $tmpfile + TOOLTEST $p -l w + TOOLTEST $p -l r + # use case 1.9, testing with multi-planes chunks + TOOLTEST $p -z 256 -y 5 # 5 planes chunks + # cleanup temp datafile + if test -z "$HDF5_NOCLEANUP"; then + rm -f $p.h5 + fi +done + +cd .. +# Report test results and exit +if test $nerrors -eq 0 ; then + echo "All $TESTNAME tests passed." + if test -z "$HDF5_NOCLEANUP"; then + # delete the test directory + rm -rf usecases_test + fi +else + echo "$TESTNAME tests failed with $nerrors errors." + EXIT_VALUE=$EXIT_FAILURE +fi + +exit $EXIT_VALUE diff --git a/test/test_usecases.sh.in b/test/test_usecases.sh.in deleted file mode 100644 index fd30afd..0000000 --- a/test/test_usecases.sh.in +++ /dev/null @@ -1,219 +0,0 @@ -#! /bin/bash -# -# Copyright by The HDF Group. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. -# -# Tests the use cases of swmr features. -# -# Created: -# Albert Cheng, 2013/06/01. -# Modified: -# - -# This is work in progress. -# For now, it shows how to run the test cases programs. It only verifies the -# exit codes are okay (0). - -############################################################################### -## test variables -############################################################################### - -# Number of errors encountered during test run. -nerrors=0 - -# Define variables -verbose=yes - -############################################################################### -## Main -############################################################################### -srcdir=@srcdir@ -utils_testdir=@abs_top_builddir@/@H5_UTILS_TEST_BUILDDIR@ -testdir=@abs_top_builddir@/@H5_TEST_BUILDDIR@ - -# Check to see if the VFD specified by the HDF5_DRIVER environment variable -# supports SWMR. -$utils_testdir/swmr_check_compat_vfd -rc=$? -if [[ $rc != 0 ]] ; then - echo - echo "The VFD specified by the HDF5_DRIVER environment variable" - echo "does not support SWMR" - echo - echo "SWMR use case tests skipped" - echo - exit 0 -fi - -# Define symbols -EXIT_SUCCESS=0 -EXIT_FAILURE=1 -EXIT_VALUE=$EXIT_SUCCESS # Default all tests succeed -RESULT_PASSED=" PASSED" -RESULT_FAILED="*FAILED*" -RESULT_SKIP="-SKIP-" -USECASES_PROGRAMS="use_append_chunk use_append_mchunks" -TESTNAME="Use Case" - -# Source in the output filter function definitions. -. $srcdir/../bin/output_filter.sh - -# Define functions -# Print a line-line message left justified in a field of 72 characters. -# Results can be " PASSED", "*FAILED*", "-SKIP-", up to 8 characters -# wide. -# SPACES should be at least 71 spaces. ($* + ' ' + 71 + 8 >= 80) -# -TESTING() { - SPACES=" " - echo "$* $SPACES" | cut -c1-72 | tr -d '\012' -} - -# Run a test and print PASS or *FAIL*. If a test fails then increment -# the `nerrors' global variable and (if $verbose is set) display the -# difference between the actual output and the expected output. The -# expected output is given as the first argument to this function and -# the actual output file is calculated by replacing the `.ddl' with -# `.out'. The actual output is not removed if $HDF5_NOCLEANUP has a -# non-zero value. -# ADD_H5_TEST -TOOLTEST() { - program=$1 - shift - - actual="$program.out" - actual_err="$program.err" - actual_sav=${actual}-sav - actual_err_sav=${actual_err}-sav - - # Run test. - TESTING $program $@ - ( - $RUNSERIAL $testdir/$program "$@" - ) >$actual 2>$actual_err - exit_code=$? - - # save actual and actual_err in case they are needed later. - cp $actual $actual_sav - STDOUT_FILTER $actual - cp $actual_err $actual_err_sav - STDERR_FILTER $actual_err - cat $actual_err >> $actual - - if [ $exit_code -eq 0 ];then - echo "$RESULT_PASSED" - test yes = "$verbose" && sed 's/^/ /' < $actual - else - echo "$RESULT_FAILED" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && sed 's/^/ /' < $actual - fi - - # Clean up output file - if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err $actual_sav $actual_err_sav $actual_ext - fi -} - -# HDF5 has several tests that create and delete signal files to communicate -# between processes, and it seems that even though the names of the files are -# different, occasionally the wrong file is deleted, interrupting the flow of -# the test. Running each of these tests in its own directory should eliminate -# the problem. -mkdir usecases_test -cp $testdir/twriteorder usecases_test -for FILE in use_*; do - case "$FILE" in - *.o) continue ;; ## don't copy the .o files - esac - if test -f "$FILE" ; then - cp $FILE usecases_test - fi -done - -# With the --disable-shared option, swmr program files are built in the test -# directory, otherwise they are in test/.libs with a corresponding wrapper -# script in the test directory. The programs or wrapper scripts in test should -# always be copied, swmr files in .libs should be copied only if they exists. -if [ -f .libs/use_append_chunk ]; then - mkdir usecases_test/.libs - for FILE in .libs/use_*; do - case "$FILE" in - *.o) continue ;; ## don't copy the .o files - esac - if test -f "$FILE" ; then - cp $FILE usecases_test/.libs - fi - done - cp .libs/twriteorder usecases_test/.libs -fi - -cd usecases_test - - -# run tests for H5Odisable_mdc_flushes/H5Oenable_mdc_flushes/H5Oare_mdc_flushes_disabled here temporary -USECORK=use_disable_mdc_flushes -for p in $USECORK; do - TOOLTEST $p - TOOLTEST $p -y 3 - TOOLTEST $p -n 3000 - TOOLTEST $p -n 5000 -done - -# run write order test here temporary -WRITEORDER=twriteorder -for p in $WRITEORDER; do - TOOLTEST $p - TOOLTEST $p -b 1000 - TOOLTEST $p -p 3000 - TOOLTEST $p -n 2000 - TOOLTEST $p -l w - TOOLTEST $p -l r -done - -# Report test results -if test $nerrors -eq 0 ; then - echo "$WRITEORDER test passed." -else - echo "$WRITEORDER test failed with $nerrors errors." - EXIT_VALUE=$EXIT_FAILURE - nerrors=0 # reset nerror for the regular tests below. -fi - -# main body -for p in $USECASES_PROGRAMS; do - TOOLTEST $p - TOOLTEST $p -z 256 - tmpfile=/tmp/datatfile.$$ - TOOLTEST $p -f $tmpfile; rm -f $tmpfile - TOOLTEST $p -l w - TOOLTEST $p -l r - # use case 1.9, testing with multi-planes chunks - TOOLTEST $p -z 256 -y 5 # 5 planes chunks - # cleanup temp datafile - if test -z "$HDF5_NOCLEANUP"; then - rm -f $p.h5 - fi -done - -cd .. -# Report test results and exit -if test $nerrors -eq 0 ; then - echo "All $TESTNAME tests passed." - if test -z "$HDF5_NOCLEANUP"; then - # delete the test directory - rm -rf usecases_test - fi -else - echo "$TESTNAME tests failed with $nerrors errors." - EXIT_VALUE=$EXIT_FAILURE -fi - -exit $EXIT_VALUE diff --git a/test/test_vds_env.sh.in b/test/test_vds_env.sh.in new file mode 100644 index 0000000..39f13a5 --- /dev/null +++ b/test/test_vds_env.sh.in @@ -0,0 +1,44 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Test for external file with environment variable: HDF5_VDS_PREFIX + +srcdir=@srcdir@ + +nerrors=0 + +############################################################################## +############################################################################## +### T H E T E S T S ### +############################################################################## +############################################################################## + +# test for VDS with HDF5_VDS_PREFIX +echo "Testing basic virtual dataset I/O via H5Pset_vds_prefix(): all selection with ENV prefix" +TEST_NAME=vds_env # The test name +TEST_BIN=`pwd`/$TEST_NAME # The path of the test binary +ENVCMD="env HDF5_VDS_PREFIX=\${ORIGIN}/tmp_vds_env" # Set the environment variable & value +UNENVCMD="unset HDF5_VDS_PREFIX" # Unset the environment variable & value +# +# Run the test +# echo "$ENVCMD $RUNSERIAL $TEST_BIN" +$ENVCMD $RUNSERIAL $TEST_BIN +exitcode=$? +if [ $exitcode -eq 0 ]; then + echo "Test prefix for HDF5_VDS_PREFIX PASSED" +else + nerrors="`expr $nerrors + 1`" + echo "***Error encountered for HDF5_VDS_PREFIX test***" +fi +$UNENVCMD +exit $nerrors diff --git a/test/test_vds_swmr.pwsh.in b/test/test_vds_swmr.pwsh.in new file mode 100644 index 0000000..bf5aabb --- /dev/null +++ b/test/test_vds_swmr.pwsh.in @@ -0,0 +1,190 @@ +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Tests for the swmr feature using virtual datasets. + +$srcdir = '@srcdir@' +$utils_testdir=@H5_UTILS_TEST_BUILDDIR@ +$testdir=@H5_TEST_BUILDDIR@ + +############################################################################### +## test parameters +############################################################################### + +$Nwriters = 6 # number of writers (1 per source dataset) +$Nreaders = 5 # number of readers to launch +$nerrors = 0 + +############################################################################### +## definitions for message file to coordinate test runs +############################################################################### +$WRITER_MESSAGE = 'SWMR_WRITER_MESSAGE' # The message file created by writer that the open is complete + # This should be the same as the define in "test/swmr_common.h" +$MESSAGE_TIMEOUT = 300 # Message timeout length in secs + # This should be the same as the define in "test/h5test.h" + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +function Test-WithSpaces { + $SPACES=' ' + #Write-Output "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# To wait for the writer message file or till the maximum # of seconds is reached +# $Message is the message file to wait for +# This performs similar function as the routine h5_wait_message() in test/h5test.c +function Wait-Message { + [CmdletBinding()] + param ( + [Parameter(Mandatory, ValueFromPipeline)] + [string]$TestPath, # Get the path of the message file to wait for + [Parameter(Mandatory, ValueFromPipeline)] + [string]$Message # Get the name of the message file to wait for + ) + + PROCESS { + $message = Join-Path -Path $TestPath -ChildPath $Message + $t0 = Get-Date # Get current time + $t1 = $t0 + $difft = New-Timespan -Start $t0 -End $t1 # Initialize the time difference + $mexist = 0 # Indicate whether the message file is found + while ($difft.TotalSeconds -lt $MESSAGE_TIMEOUT) { # Loop till message times out + $t1 = Get-Date # Get current time in seconds + $difft = New-Timespan -Start $t0 -End $t1 # Calculate the time difference + #Write-Output "Check for $message : time=$difft" + if ([System.IO.File]::Exists($message)) { # If message file is found: + $mexist = 1 # indicate the message file is found + Remove-Item $message # remove the message file + break # get out of the while loop + } + } + if ($mexist -eq 0) { + # Issue warning that the writer message file is not found, continue with launching the reader(s) + Write-Warning -Message "$WRITER_MESSAGE is not found after waiting $MESSAGE_TIMEOUT seconds" + } + else { + Write-Output "$WRITER_MESSAGE is found" + } + } +} + +############################################################################### +## Main +############################################################################### + +# Check to see if the VFD specified by the HDF5_DRIVER environment variable +# supports SWMR. +$testprog = Join-Path -Path $utils_testdir -ChildPath swmr_check_compat_vfd.exe +$rp = Start-Process -FilePath $testprog -PassThru -Wait -NoNewWindow +if ($rp.ExitCode -ne 0) { + Write-Output "" + Write-Output "The VFD specified by the HDF5_DRIVER environment variable" + Write-Output "does not support SWMR." + Write-Output "" + Write-Output "SWMR acceptance tests skipped" + Write-Output "" + exit 0 +} + +Set-Location -Path vds_swmr_test +$testdir = Join-Path -Path $testdir -ChildPath vds_swmr_test + +Write-Output "" +Write-Output "###############################################################################" +Write-Output "## Basic VDS SWMR test - writing to a tiled plane" +Write-Output "###############################################################################" + +# Launch the file generator +Write-Output "launch the swmr_generator" +$rp = Start-Process -FilePath $testdir/vds_swmr_gen.exe -NoNewWindow -PassThru -Wait +if ($rp.ExitCode -ne 0) { + Write-Warning "generator had error" + $nerrors += 1 +} + +# Check for error and exit if one occurred +#Write-Output "nerrors=$nerrors" +if ($nerrors -ne 0) { + Write-Warning "VDS SWMR tests failed with $nerrors errors." + exit 1 +} + +# Launch the writers +Write-Output "launch the $Nwriters SWMR VDS writers (1 per source)" +$pid_writers = @() +$n = 0 +while ($n -lt $Nwriters) { + $rp = Start-Process -FilePath $testdir/vds_swmr_writer.exe -NoNewWindow -PassThru -ArgumentList "$n" 2>&1 | tee swmr_writer.out.$n + $pid_writers += $rp.id + $n += 1 +} +#Write-Output "pid_writers=$pid_writers" + +# Sleep to ensure that the writers have started +Start-Sleep -Seconds 3 + +# Launch the readers +Write-Output "launch $Nreaders SWMR readers" +$pid_readers = @() +$n = 0 +while ($n -lt $Nreaders) { + $rp = Start-Process -FilePath $testdir/vds_swmr_reader.exe -NoNewWindow -PassThru 2>&1 | tee swmr_reader.out.$n + $pid_readers += $rp.id + $n += 1 +} +#Write-Output "pid_readers=$pid_readers" + +# Collect exit code of the writers +foreach ($xpid in $pid_writers) { + #Write-Output "checked writer $xpid" + $result = Wait-Process -Id $xpid + if ($result.ExitCode -ne 0) { + Write-Warning "writer had error" + $nerrors += 1 + } +} + +# Collect exit code of the readers +# (they usually finish after the writers) +foreach ($xpid in $pid_readers) { + #Write-Output "checked reader $xpid" + $result = Wait-Process -Id $xpid + if ($result.ExitCode -ne 0) { + Write-Warning "reader had error" + $nerrors += 1 + } +} + +# Check for error and exit if one occurred +#Write-Output "nerrors=$nerrors" +if ($nerrors -ne 0) { + Write-Warning "VDS SWMR tests failed with $nerrors errors." + exit 1 +} + +############################################################################### +## Report and exit +############################################################################### +cd .. +#Write-Output "nerrors=$nerrors" +if ($nerrors -eq 0) { + Write-Output "VDS SWMR tests passed." +# if test -z "$HDF5_NOCLEANUP"; then +# # delete the test directory +# Remove-Item vds_swmr_test -Recurse +# fi + exit 0 +} +else { + Write-Warning "VDS SWMR tests failed with $nerrors errors." + exit 1 +} + diff --git a/test/test_vds_swmr.sh.in b/test/test_vds_swmr.sh.in new file mode 100644 index 0000000..399fdef --- /dev/null +++ b/test/test_vds_swmr.sh.in @@ -0,0 +1,233 @@ +#! /bin/bash +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Tests for the swmr feature using virtual datasets. +# +# Created: +# Dana Robinson, November 2015 + +srcdir=@srcdir@ +utils_testdir=@abs_top_builddir@/@H5_UTILS_TEST_BUILDDIR@ +testdir=@abs_top_builddir@/@H5_TEST_BUILDDIR@ + +############################################################################### +## test parameters +############################################################################### + +Nwriters=6 # number of writers (1 per source dataset) +Nreaders=5 # number of readers to launch +nerrors=0 + +############################################################################### +## definitions for message file to coordinate test runs +############################################################################### +WRITER_MESSAGE=SWMR_WRITER_MESSAGE # The message file created by writer that the open is complete + # This should be the same as the define in "test/swmr_common.h" +MESSAGE_TIMEOUT=300 # Message timeout length in secs + # This should be the same as the define in "test/h5test.h" + +############################################################################### +## short hands and function definitions +############################################################################### +DPRINT=: # Set to "echo Debug:" for debugging printing, + # else ":" for noop. +IFDEBUG=: # Set to null to turn on debugging, else ":" for noop. + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +TESTING() { + SPACES=" " + echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# To wait for the writer message file or till the maximum # of seconds is reached +# $1 is the message file to wait for +# This performs similar function as the routine h5_wait_message() in test/h5test.c +WAIT_MESSAGE() { + message=$1 # Get the name of the message file to wait for + t0=`date +%s` # Get current time in seconds + difft=0 # Initialize the time difference + mexist=0 # Indicate whether the message file is found + while [ $difft -lt $MESSAGE_TIMEOUT ] ; # Loop till message times out + do + t1=`date +%s` # Get current time in seconds + difft=`expr $t1 - $t0` # Calculate the time difference + if [ -e $message ]; then # If message file is found: + mexist=1 # indicate the message file is found + rm $message # remove the message file + break # get out of the while loop + fi + done; + if test $mexist -eq 0; then + # Issue warning that the writer message file is not found, continue with launching the reader(s) + echo warning: $WRITER_MESSAGE is not found after waiting $MESSAGE_TIMEOUT seconds + else + echo $WRITER_MESSAGE is found + fi +} + +############################################################################### +## Main +############################################################################### + +# Check to see if the VFD specified by the HDF5_DRIVER environment variable +# supports SWMR. +$utils_testdir/swmr_check_compat_vfd +rc=$? +if [ $rc -ne 0 ] ; then + echo + echo "The VFD specified by the HDF5_DRIVER environment variable" + echo "does not support SWMR." + echo + echo "SWMR acceptance tests skipped" + echo + exit 0 +fi + +# Parse options (none accepted at this time) +while [ $# -gt 0 ]; do + case "$1" in + *) # unknown option + echo "$0: Unknown option ($1)" + exit 1 + ;; + esac +done + +# HDF5 has several tests that create and delete signal files to communicate +# between processes, and it seems that even though the names of the files are +# different, occasionally the wrong file is deleted, interrupting the flow of +# the test. Running each of these tests in its own directory should eliminate +# the problem. +mkdir vds_swmr_test +for FILE in vds_swmr*; do + case "$FILE" in + *.o) continue ;; ## don't copy the .o files + esac + if test -f "$FILE" ; then + cp $FILE vds_swmr_test + fi +done + +# With the --disable-shared option, swmr program files are built in the test +# directory, otherwise they are in test/.libs with a corresponding wrapper +# script in the test directory. The programs or wrapper scripts in test should +# always be copied, swmr files in .libs should be copied only if they exists. +if [ -f .libs/vds_swmr_writer ]; then + mkdir vds_swmr_test/.libs + for FILE in .libs/vds_swmr*; do + case "$FILE" in + *.o) continue ;; ## don't copy the .o files + esac + if test -f "$FILE" ; then + cp $FILE vds_swmr_test/.libs + fi + done +fi + +cd vds_swmr_test + + +echo +echo "###############################################################################" +echo "## Basic VDS SWMR test - writing to a tiled plane" +echo "###############################################################################" + +# Launch the file generator +echo launch the generator +$testdir/vds_swmr_gen +if test $? -ne 0; then + echo generator had error + nerrors=`expr $nerrors + 1` +fi + +# Check for error and exit if one occurred +$DPRINT nerrors=$nerrors +if test $nerrors -ne 0 ; then + echo "VDS SWMR tests failed with $nerrors errors." + exit 1 +fi + +# Launch the writers +echo "launch the $Nwriters SWMR VDS writers (1 per source)" +pid_writers="" +n=0 +while [ $n -lt $Nwriters ]; do + $testdir/vds_swmr_writer $n & + pid_writers="$pid_writers $!" + n=`expr $n + 1` +done +$DPRINT pid_writers=$pid_writers +$IFDEBUG ps + +# Sleep to ensure that the writers have started +sleep 3 + +# Launch the readers +echo launch $Nreaders SWMR readers +pid_readers="" +n=0 +while [ $n -lt $Nreaders ]; do + $testdir/vds_swmr_reader & + pid_readers="$pid_readers $!" + n=`expr $n + 1` +done +$DPRINT pid_readers=$pid_readers +$IFDEBUG ps + +# Collect exit code of the writers +for xpid in $pid_writers; do + $DPRINT checked writer $xpid + wait $xpid + if test $? -ne 0; then + echo writer had error + nerrors=`expr $nerrors + 1` + fi +done + +# Collect exit code of the readers +# (they usually finish after the writers) +for xpid in $pid_readers; do + $DPRINT checked reader $xpid + wait $xpid + if test $? -ne 0; then + echo reader had error + nerrors=`expr $nerrors + 1` + fi +done + +# Check for error and exit if one occurred +$DPRINT nerrors=$nerrors +if test $nerrors -ne 0 ; then + echo "VDS SWMR tests failed with $nerrors errors." + exit 1 +fi + +############################################################################### +## Report and exit +############################################################################### +cd .. +$DPRINT nerrors=$nerrors +if test $nerrors -eq 0 ; then + echo "VDS SWMR tests passed." + if test -z "$HDF5_NOCLEANUP"; then + # delete the test directory + rm -rf vds_swmr_test + fi + exit 0 +else + echo "VDS SWMR tests failed with $nerrors errors." + exit 1 +fi + diff --git a/test/test_vol_plugin.sh.in b/test/test_vol_plugin.sh.in deleted file mode 100644 index d31646b..0000000 --- a/test/test_vol_plugin.sh.in +++ /dev/null @@ -1,84 +0,0 @@ -#! /bin/sh -# -# Copyright by The HDF Group. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. -# -# This shell script is for testing VOL connector plugins. -# -srcdir=@srcdir@ -TOP_BUILDDIR=@top_builddir@ - -EXIT_SUCCESS=0 -EXIT_FAILURE=1 - -nerrors=0 -verbose=yes -exit_code=$EXIT_SUCCESS - -TEST_NAME=vol_plugin -TEST_BIN=`pwd`/$TEST_NAME -FROM_DIR=`pwd`/.libs -case $(uname) in - CYGWIN* ) - NULL_VOL_PLUGIN="$FROM_DIR/cygnull_vol_connector*" - ;; - *) - NULL_VOL_PLUGIN="$FROM_DIR/libnull_vol_connector*" - ;; -esac -TEMP_PLUGIN_DIR=null_vol_plugin_dir -CP="cp -p" # Use -p to preserve mode,ownership, timestamps -RM="rm -rf" - -# Print a line-line message left justified in a field of 70 characters -# beginning with the word "Testing". -# -TESTING() { - SPACES=" " - echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' -} - -# Main Body -# Create test directory if necessary. -test -d $TEMP_PLUGIN_DIR || mkdir -p $TEMP_PLUGIN_DIR -if [ $? != 0 ]; then - echo "Failed to create VOL connector plugin test directory ($TEMP_PLUGIN_DIR)" - exit $EXIT_FAILURE -fi - -# Copy plugin for the tests. -$CP $NULL_VOL_PLUGIN $TEMP_PLUGIN_DIR -if [ $? != 0 ]; then - echo "Failed to copy NULL VOL plugin ($NULL_VOL_PLUGIN) to test directory." - exit $EXIT_FAILURE -fi - -# setup plugin path -ENVCMD="env HDF5_PLUGIN_PATH=${TEMP_PLUGIN_DIR}" - -# Run the test -$ENVCMD $TEST_BIN -if [ $? != 0 ]; then - nerrors=`expr $nerrors + 1` -fi - -# print results -if test $nerrors -ne 0 ; then - echo "$nerrors errors encountered" - exit_code=$EXIT_FAILURE -else - echo "All VOL plugin tests passed." - exit_code=$EXIT_SUCCESS -fi - -# Clean up temporary files/directories and leave -$RM $TEMP_PLUGIN_DIR - -exit $exit_code diff --git a/test/testabort_fail.sh.in b/test/testabort_fail.sh.in deleted file mode 100644 index a866f9e..0000000 --- a/test/testabort_fail.sh.in +++ /dev/null @@ -1,66 +0,0 @@ -#! /bin/sh -# -# Copyright by The HDF Group. -# Copyright by the Board of Trustees of the University of Illinois. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. -# -# Test to verify that the assertion/abort failure is fixed when the application -# does not close the file. (See HDFFV-10160) -# -# Test to verify that the infinite loop closing library/abort failure is fixed -# when the application creates and removes dense attributes (See HDFFV-10659) - -srcdir=@srcdir@ - -nerrors=0 - -############################################################################## -############################################################################## -### T H E T E S T S ### -############################################################################## -############################################################################## -# -# -echo "Testing file not closed assertion/abort failure" -TEST_NAME=filenotclosed # The test name -TEST_BIN=`pwd`/$TEST_NAME # The path of the test binary -# -# Run the test -$RUNSERIAL $TEST_BIN >/dev/null 2>&1 -exitcode=$? -if [ $exitcode -eq 0 ]; then - echo "Test PASSED" -else - echo "Test FAILED" - nerrors="`expr $nerrors + 1`" -fi -# -# -echo "Testing infinite loop closing library/abort failure" -TEST_NAME=del_many_dense_attrs # The test name -TEST_BIN=`pwd`/$TEST_NAME # The path of the test binary -# Run the test -$RUNSERIAL $TEST_BIN >/dev/null 2>&1 -exitcode=$? -if [ $exitcode -eq 0 ]; then - echo "Test PASSED" -else - echo "Test FAILED" - nerrors="`expr $nerrors + 1`" -fi -# -# -if test $nerrors -eq 0 ; then - echo "All tests for abort failure passed." - exit 0 -else - echo "Tests for abort failure failed with $nerrors errors." - exit 1 -fi diff --git a/test/testcheck_version.sh.in b/test/testcheck_version.sh.in deleted file mode 100644 index 273702e..0000000 --- a/test/testcheck_version.sh.in +++ /dev/null @@ -1,261 +0,0 @@ -#! /bin/sh -# -# Copyright by The HDF Group. -# Copyright by the Board of Trustees of the University of Illinois. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. - -# -# Tests for the H5check_version function. -# -# Programmer: Albert Cheng -# Sep 28, 2009 - -srcdir=@srcdir@ - -# Variables filled in by the configure process. -# Determine the configure options of the hdf5 library and executables. -Shared_Lib=@enable_shared@ -Static_Lib=@enable_static@ -Static_exec=@STATIC_EXEC@ -h5haveexitcode=yes # default is yes - -CMP='cmp -s' -DIFF='diff -c' -RM='rm -f' - -# Function definitions -# -# Show the purpose of this test script and a note about the abort messages. -PURPOSE() { - echo "Tests for the H5check_version function." - echo "Note that abort messages may appear due to the expected termination" - echo "of the program when it is tested with mis-matched version numbers." -} - -# Print a line-line message left justified in a field of 70 characters. -# -LINEMSG() { - SPACES=" " - echo "$* $SPACES" | cut -c1-70 | tr -d '\012' -} - - -# Print a "SKIP" message -SKIP() { - LINEMSG $* - echo " -SKIP-" -} - - -# Print warning message of version mismatch. -WarnMesg(){ - echo "Warning! ***HDF5 library version mismatched error***" - echo "The HDF5 header files used to compile this application do not match" - echo "the version used by the HDF5 library to which this application is linked." - echo "Data corruption or segmentation faults may occur if the application continues." - echo "This can happen when an application was compiled by one version of HDF5 but" - echo "linked with a different version of static or shared HDF5 library." - echo "You should recompile the application or check your shared library related" - echo "settings such as 'LD_LIBRARY_PATH'." - echo "You can, at your own risk, disable this warning by setting the environment" - echo "variable 'HDF5_DISABLE_VERSION_CHECK' to a value of '1'." - echo "Setting it to 2 or higher will suppress the warning messages totally." - echo "Headers are $xxh5versmajor.$xxh5versminor.$xxh5versrelease, library is $h5versmajor.$h5versminor.$h5versrelease" - test -n "$H5_HAVE_EMBEDDED_LIBINFO" && cat $h5libsettings - echo "Bye..." -} - - -# Print warning message2 of version mismatch. -WarnMesg2(){ - echo "Warning! ***HDF5 library version mismatched error***" - echo "The HDF5 header files used to compile this application do not match" - echo "the version used by the HDF5 library to which this application is linked." - echo "Data corruption or segmentation faults may occur if the application continues." - echo "This can happen when an application was compiled by one version of HDF5 but" - echo "linked with a different version of static or shared HDF5 library." - echo "You should recompile the application or check your shared library related" - echo "settings such as 'LD_LIBRARY_PATH'." - echo "'HDF5_DISABLE_VERSION_CHECK' environment variable is set to 1, application will" - echo "continue at your own risk." - echo "Headers are $xxh5versmajor.$xxh5versminor.$xxh5versrelease, library is $h5versmajor.$h5versminor.$h5versrelease" - test -n "$H5_HAVE_EMBEDDED_LIBINFO" && cat $h5libsettings -} - - -# Run a test and print PASS or *FAIL*. If a test fails then increment -# the `nerrors' global variable and (if $verbose is set) display the -# difference between the actual output and the expected output. The -# expected output generated according to the parameter values and compared -# against actual output. -# The expected and actual output files are removed unless $HDF5_NOCLEANUP -# has a non-zero value. -# $1: the set value of $HDF5_DISABLE_VERSION_CHECK. (unset means not to set -# it at all. -# $2: Change the version number(s) to cause a mismatch. (none means no -# mismatch). -# -# Expected results: -# Value of $HDF5_DISABLE_VERSION_CHECK -# unset "" -1 0 1 2 3 -# Matched OK OK OK OK OK OK OK -# Mismatched W/A W/A W/A W/A W2/OK OK W2/OK -# Result codes: -# OK: No warning, exit 0. -# W/A: Warning, abort and exit non-0. -# W2/OK: Different Warning, exit 0. -# -# Implemented only exit code matching. Still need to match output. -TESTING() { - DEBUGPRINT command is $0 $* - TEST_NAME=tcheck_version # The test name - TEST_BIN=`pwd`/$TEST_NAME # The path of the test binary - - expect=${TEST_NAME}_expect.out - actual=${TEST_NAME}_actual.out - actual_err=${TEST_NAME}_actual.err - arguments= - - h5DisableVersion="$1" - wrongversionnumbers="$2" - xxh5versmajor=$h5versmajor - xxh5versminor=$h5versminor - xxh5versrelease=$h5versrelease - - if [ "$h5DisableVersion" = unset ]; then - envcmd="" # noop - else - envcmd="env HDF5_DISABLE_VERSION_CHECK=$h5DisableVersion" - fi - - if [ "$wrongversionnumbers" = none ]; then - # OK: No warning, exit 0 - cp /dev/null $expect - expect_code=0 - else - arguments=-t"$wrongversionnumbers" - # calculate mismatched version numbers by listing. - case $wrongversionnumbers in - "M") xxh5versmajor=`expr $h5versmajor + 1` - ;; - "m") xxh5versminor=`expr $h5versminor + 1` - ;; - "r") xxh5versrelease=`expr $h5versrelease + 1` - ;; - esac - case "$h5DisableVersion" in - 1) - # W2/OK: Different Warning, exit 0. - WarnMesg2 > $expect - expect_code=0 - ;; - [2-9]|[1-9][0-9]*) - # OK: No warning, exit 0 - cp /dev/null $expect - expect_code=0 - ;; - *) # W/A: Warning, abort and exit non-0. - WarnMesg > $expect - expect_code=6 # Signal Abort exit code (128+6) - ;; - esac - fi - - # Run test. - LINEMSG $envcmd $TEST_NAME $arguments - ( - $envcmd $RUNSERIAL $TEST_BIN $arguments - ) >$actual 2>$actual_err - ret_code=$? - cat $actual_err >> $actual - - if [ $h5haveexitcode = 'yes' -a \( $expect_code -ne $ret_code \) ]; then - echo "*FAILED*" - echo " Expected exit code ($expect_code) differs from actual code ($ret_code)" - nerrors="`expr $nerrors + 1`" - elif $CMP $expect $actual; then - echo " PASSED" - else - echo "*FAILED*" - echo " Expected result differs from actual result" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' - fi - - # Clean up output file. - # Also clean the core file generated by H5check_version's abort. - if test -z "$HDF5_NOCLEANUP"; then - $RM $expect $actual $actual_err - $RM core - fi -} - - -# Echo parameters for debugging if verbose mode is on. -DEBUGPRINT() { - if [ -n "$debugmode" ]; then - echo $* - fi -} - - -# MAIN Body -nerrors=0 -verbose=yes # default on -debugmode= # default off -H5_HAVE_EMBEDDED_LIBINFO=`grep '#define H5_HAVE_EMBEDDED_LIBINFO ' ../src/H5pubconf.h` -h5libsettings=../src/libhdf5.settings - -PURPOSE - -# Figure out library version numbers from the header file. -h5versmajor=`grep '#define H5_VERS_MAJOR' $srcdir/../src/H5public.h | awk '{print $3}'` -h5versminor=`grep '#define H5_VERS_MINOR' $srcdir/../src/H5public.h | awk '{print $3}'` -h5versrelease=`grep '#define H5_VERS_RELEASE' $srcdir/../src/H5public.h | awk '{print $3}'` -DEBUGPRINT $h5versmajor.$h5versminor.$h5versrelease -case "$h5versmajor$h5versminor$h5versrelease" in - [0-9]*) # good. noop. - ;; - *) - echo "Illegal library version numbers($h5versmajor.$h5versminor.$h5versrelease)" - echo "Test aborted" - exit 1 - ;; -esac - -# RUNSERIAL is used. Check if it can return exit code from executalbe correctly. -if [ -n "$RUNSERIAL_NOEXITCODE" ]; then - echo "***Warning*** Serial Exit Code is not passed back to shell correctly." - echo "***Warning*** Exit code checking is skipped." - h5haveexitcode=no -fi - -# Three Categories of tests: -# Normal: where the version numbers all matched (wrong_version == none). -# Mismatched version numbers (could be Major or minor version -# or release numbers or a combination of all three.) -# Test all the above with different values of the environment variable, -# HDF5_DISABLE_VERSION_CHECK, as unset, "", -1, 0, 1, 2, 3 - -for val_disable_version_check in unset "" -1 0 1 2 3; do - for wrong_version in none M m; do - TESTING "$val_disable_version_check" "$wrong_version" - done -done - - -# Check and report results. -if [ $nerrors -gt 0 ]; then - echo "***$nerrors errors encountered***" - exit 1 -else - echo "No error encountered" - exit 0 -fi diff --git a/test/testerror.sh.in b/test/testerror.sh.in deleted file mode 100644 index 4fb2a81..0000000 --- a/test/testerror.sh.in +++ /dev/null @@ -1,126 +0,0 @@ -#! /bin/sh -# -# Copyright by The HDF Group. -# Copyright by the Board of Trustees of the University of Illinois. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. -# -# Tests for test_error and err_compat - -srcdir=@srcdir@ - -# Determine backward compatibility options eneabled -DEPRECATED_SYMBOLS="@DEPRECATED_SYMBOLS@" - -CMP='cmp -s' -DIFF='diff -c' - -# Skip plugin module to test missing filter -# Also reset the VOL connector to only use the native connector, because of the -# error stack checking. QAK - 2019/03/09 -ENVCMD="env HDF5_PLUGIN_PRELOAD=:: HDF5_VOL_CONNECTOR=native" - -nerrors=0 -verbose=yes - -test -d ./testfiles || mkdir ./testfiles - -# Print a line-line message left justified in a field of 70 characters -# beginning with the word "Testing". -# -TESTING() { - SPACES=" " - echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' -} - -# Run a test and print PASS or *FAIL*. If a test fails then increment -# the `nerrors' global variable and (if $verbose is set) display the -# difference between the actual output and the expected output. The -# expected output is given as the first argument to this function and -# the actual output file is calculated by replacing the `.ddl' with -# `.out'. The actual output is not removed if $HDF5_NOCLEANUP has a -# non-zero value. -# -TEST() { - TEST_ERR=$1 # The test name - TEST_ERR_BIN=`pwd`/$TEST_ERR # The path of the test binary - - expect1="$srcdir/testfiles/$1_1" - expect2="$srcdir/testfiles/$1_2" - actual="./`basename $1`.out" - actual_err="./`basename $1`.err" - actual_ext="./`basename $1`.ext" - shift - - # Run test. - TESTING $TEST_ERR - ( - # Skip the plugin for testing missing filter. - $ENVCMD $RUNSERIAL $TEST_ERR_BIN - ) >$actual 2>$actual_err - - # Check for core dump - if [ $? != 0 ]; then - nerrors=`expr $nerrors + 1` - fi - - # Extract file name, line number, version and thread IDs because they may be different - sed -e 's/thread [0-9]*/thread (IDs)/' -e 's/: .*\.c /: (file name) /' \ - -e 's/line [0-9]*/line (number)/' \ - -e 's/v[1-9]*\.[0-9]*\./version (number)\./' \ - -e 's/[1-9]*\.[0-9]*\.[0-9]*[^)]*/version (number)/' \ - -e 's/H5Eget_auto[1-2]*/H5Eget_auto(1 or 2)/' \ - -e 's/H5Eset_auto[1-2]*/H5Eset_auto(1 or 2)/' \ - $actual_err > $actual_ext - cat $actual_ext >> $actual - - if $CMP $expect1 $actual; then - echo " PASSED" - elif $CMP $expect2 $actual; then - echo " PASSED" - else - echo "*FAILED*" - echo " Expected result differs from actual result" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect1 $actual |sed 's/^/ /' - fi - - # Clean up output file - if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err $actual_ext - fi -} - -# Print a "SKIP" message -SKIP() { - TESTING $@ - echo " -SKIP-" -} - -############################################################################## -############################################################################## -### T H E T E S T S ### -############################################################################## -############################################################################## - -# test for err_compat -if test $DEPRECATED_SYMBOLS != "yes"; then -SKIP err_compat -else -TEST err_compat -fi - -# test for error_test. Skip the plugin for testing missing filter. -TEST error_test - -if test $nerrors -eq 0 ; then - echo "All Error API tests passed." -fi - -exit $nerrors diff --git a/test/testexternal_env.sh.in b/test/testexternal_env.sh.in deleted file mode 100644 index 94fbb88..0000000 --- a/test/testexternal_env.sh.in +++ /dev/null @@ -1,42 +0,0 @@ -#! /bin/sh -# -# Copyright by The HDF Group. -# Copyright by the Board of Trustees of the University of Illinois. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. -# -# Test for external file with environment variable: HDF5_EXTFILE_PREFIX - -srcdir=@srcdir@ - -nerrors=0 - -############################################################################## -############################################################################## -### T H E T E S T S ### -############################################################################## -############################################################################## - -# test for external file with HDF5_EXTFILE_PREFIX -echo "Testing external file with HDF5_EXTFILE_PREFIX" -TEST_NAME=external_env # The test name -TEST_BIN=`pwd`/$TEST_NAME # The path of the test binary -ENVCMD="env HDF5_EXTFILE_PREFIX=\${ORIGIN}" # The environment variable & value -# -# Run the test -# echo "$ENVCMD $RUNSERIAL $TEST_BIN" -$ENVCMD $RUNSERIAL $TEST_BIN -exitcode=$? -if [ $exitcode -eq 0 ]; then - echo "Test prefix for HDF5_EXTFILE_PREFIX PASSED" -else - nerrors="`expr $nerrors + 1`" - echo "***Error encountered for HDF5_EXTFILE_PREFIX test***" -fi -exit $nerrors diff --git a/test/testflushrefresh.sh.in b/test/testflushrefresh.sh.in deleted file mode 100644 index 209d370..0000000 --- a/test/testflushrefresh.sh.in +++ /dev/null @@ -1,238 +0,0 @@ -#! /bin/sh -# -# Copyright by The HDF Group. -# Copyright by the Board of Trustees of the University of Illinois. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. -# -# -# Test script for the flush/evict single objects feature. -# -# This test file doesn't actually perform any tests, rather, it acts -# as a process manager for the 'flushrefresh' test file, which is where -# the tests are actually housed. The reason this script exists is because -# the verification of this feature needs to occur in separate processes -# from the one in which the file is being manipulated in. (i.e., we have -# a single writer process, and various reader processes spawning off -# and doing the verification that individual objects are being -# correctly flushed). -# -# Programmer: -# Mike McGreevy -# Tuesday, July 20, 2010 - -############################################################################### -## test variables -############################################################################### - -# Number of errors encountered during test run. -nerrors=0 - -# Set up a function to check the current time since the epoch - ideally, we'd -# like to use Perl. If it wasn't detected by configure, then use date, though -# this is less portable and might cause problems on machines that don't -# recognize the +%s option (like Solaris). -# -# Note that PERL will resolve to true or false, not a path. -PERL=@PERL@ -if test -n "$PERL"; then - TimeStamp() - { - time=`perl -e 'print int(time)'` - echo "$time" - } -else - TimeStamp() - { - time=`date +%s` - echo "$time" - } -fi - -############################################################################### -## Main -############################################################################### -srcdir=@srcdir@ -utils_testdir=@abs_top_builddir@/@H5_UTILS_TEST_BUILDDIR@ -testdir=@abs_top_builddir@/@H5_TEST_BUILDDIR@ - -# Check to see if the VFD specified by the HDF5_DRIVER environment variable -# supports SWMR. -$utils_testdir/swmr_check_compat_vfd -rc=$? -if [ $rc -ne 0 ] ; then - echo - echo "The VFD specified by the HDF5_DRIVER environment variable" - echo "does not support SWMR." - echo - echo "flush/refresh tests skipped" - echo - exit 0 -fi - -# HDF5 has several tests that create and delete signal files to communicate -# between processes, and it seems that even though the names of the files are -# different, occasionally the wrong file is deleted, interrupting the flow of -# the test. Running each of these tests in its own directory should eliminate -# the problem. -mkdir -p flushrefresh_test -cp $testdir/flushrefresh flushrefresh_test - -# With the --disable-shared option, flushrefresh is built in the test directory, -# otherwise it is in test/.libs with a wrapper script named flushrefresh in -# the test directory. test/flushrefresh should always be copied, -# .libs/flushrefresh should be copied only if it exists. -if [ -f .libs/flushrefresh ]; then - mkdir -p flushrefresh_test/.libs - for FILE in .libs/flushrefresh*; do - case "$FILE" in - *.o) continue ;; ## don't copy the .o files - esac - if test -f "$FILE" ; then - cp $FILE flushrefresh_test/.libs - fi - done -fi -cd flushrefresh_test - -# ================================================= -# Set up/initialize some variables to be used later -# ================================================= -testfile=flushrefresh.h5 -startsignal=flushrefresh_VERIFICATION_START -endsignal=flushrefresh_VERIFICATION_DONE -timeout_length=300 -timedout=0 -verification_done=0 -if [ -e $testfile ]; then - rm $testfile -fi - -# ======================== -# Launch the Test Program. -# ======================== -$testdir/flushrefresh & -pid_main=$! - -# ======================================= -# Run flush verification on test program. -# ======================================= - -until [ $verification_done -eq 1 ]; do - - # Wait for signal from test program that verification routine can run. - before=`TimeStamp` - until [ -s $startsignal ]; do - after=`TimeStamp` - timediff=`expr $after - $before` - if [ $timediff -gt $timeout_length ]; then - nerrors=`expr $nerrors + 1` - timedout=1 - break - fi - done - - # Check to see if we timed out looking for the signal before continuing. - if [ $timedout -gt 0 ]; then - echo "timed out waiting for signal from test program (flush)." - break - fi - - # Read in test routine parameters from signal file, then delete signal file. - param1=`head -1 $startsignal` - param2=`tail -1 $startsignal` - rm $startsignal - - # Check if we're done with verifications, otherwise run the specified verification. - if [ "$param1" = "VERIFICATION_DONE" ]; then - verification_done=1 - echo "all flush verification complete" > $endsignal - else - $testdir/flushrefresh $param1 $param2 - - # Check for core dump - if [ $? -gt 0 ]; then - nerrors=`expr $nerrors + 1` - fi - - echo "verification flush process done" > $endsignal - fi - -done - -# ========================================= -# Run refresh verification on test program. -# ========================================= -if [ $timedout -eq 0 ]; then - until [ $verification_done -eq 2 ]; do - - # Wait for signal from test program that verification routine can run. - before=`TimeStamp` - until [ -s $startsignal ]; do - after=`TimeStamp` - timediff=`expr $after - $before` - if [ $timediff -gt $timeout_length ]; then - nerrors=`expr $nerrors + 1` - timedout=1 - break - fi - done - - # Check to see if we timed out looking for the signal before continuing. - if [ $timedout -gt 0 ]; then - echo "timed out waiting for signal from test program (refresh)." - break - fi - - # Read in test routine parameter from signal file, then delete signal file. - param1=`head -n 1 $startsignal` - rm $startsignal - - # Check if we're done with verifications, otherwise run the specified verification. - if [ "$param1" = "VERIFICATION_DONE" ]; then - verification_done=2 - echo "all refresh verification complete" > $endsignal - else - $testdir/flushrefresh $param1 - - # Check for core dump - if [ $? -gt 0 ]; then - nerrors=`expr $nerrors + 1` - fi - - echo "refresh verifiction process done" > $endsignal - fi - - done -fi - -# ============================================ -# Wait for main to finish up, and end testing. -# ============================================ -wait $pid_main -if test $? -ne 0; then - echo flushrefresh had error - nerrors=`expr $nerrors + 1` -fi - -############################################################################### -## Report and exit -############################################################################### - -if test $nerrors -eq 0 ; then - echo "flush/refresh objects tests passed." - if test -z "$HDF5_NOCLEANUP"; then - # delete the test directory - rm -rf flushrefresh_test - fi - exit 0 -else - echo "flush/refresh objects tests failed with $nerrors errors." - exit 1 -fi diff --git a/test/testlibinfo.sh.in b/test/testlibinfo.sh.in deleted file mode 100644 index d9fee48..0000000 --- a/test/testlibinfo.sh.in +++ /dev/null @@ -1,120 +0,0 @@ -#! /bin/sh -# -# Copyright by The HDF Group. -# Copyright by the Board of Trustees of the University of Illinois. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. - - -# -# Tests for the embedded library information feature. -# Part 1: -# Verify the HDF5 library does contains an exact copy of the content of the -# libhdf5.settings file. -# Part 2: -# If executable is linked with the static hdf5 library (how to determine?), -# verify an executable indeed contains an exact copy of the content of the -# libhdf5.settings file. -# -# Programmer: Albert Cheng -# Sep 18, 2009 - -srcdir=@srcdir@ - -# Determine the configure options of the hdf5 library and executables. - -Shared_Lib=@enable_shared@ -Static_Lib=@enable_static@ -Static_exec=@STATIC_EXEC@ - - -# Print a line-line message left justified in a field of 70 characters. -# -LINEMSG() { - SPACES=" " - echo "Check file $* $SPACES" | cut -c1-70 | tr -d '\012' -} - - -# Print a "SKIP" message -SKIP() { - LINEMSG $* - echo " -SKIP-" -} - -# Function definitions -CHECK_LIBINFO(){ - LINEMSG $1 - # Some systems, like Mac, the strings command inspects library files. Older - # versions of strings may not know newer library format, resulting in - # command errors. Make it read the file as stdin to avoid the problem. - if strings < $1 | grep "SUMMARY OF THE HDF5 CONFIGURATION" > /dev/null; then - echo " PASSED" - else - echo " FAILED" - nerrors=`expr $nerrors + 1` - fi -} - - -# MAIN Body -nerrors=0 -H5_HAVE_EMBEDDED_LIBINFO=`grep '#define H5_HAVE_EMBEDDED_LIBINFO ' ../src/H5pubconf.h` - -# Skip the rest if embedded-libinfo is not enabled. -if [ -z "$H5_HAVE_EMBEDDED_LIBINFO" ]; then - echo "embedded-libinfo is not enabled. Test skipped." - exit 0 -fi - -# The location of HDF library file(s) depends on whether shared lib is -# built too. -if [ -n $Shared_Lib ]; then - h5libdir=../src/.libs - shlib=$(grep dlname ../src/libhdf5.la | sed -e "s/dlname='//" -e "s/'//") -else - h5libdir=../src -fi - -h5libsettings=../src/libhdf5.settings - -# Part 1: -# Verify the HDF5 library does contains an exact copy of the content of the -# libhdf5.settings file. -# Check dynamic library file if built. -if [ x-$Shared_Lib = x-yes ]; then - CHECK_LIBINFO ${h5libdir}/${shlib} -else - SKIP shlib -fi - -# Though rare, libhdf5.a may not have been built. -if [ x-$Static_Lib = x-yes ]; then - CHECK_LIBINFO ${h5libdir}/libhdf5.a -else - SKIP ${h5libdir}/libhdf5.a -fi - -# Check if executables has the lib information only if shared lib is not -# built or static-exec is used. (Don't care static-exec since it affects -# tools binary only.) -if [ x-$Shared_Lib != x-yes ]; then - CHECK_LIBINFO testhdf5 -else - SKIP testhdf5 -fi - - -if [ $nerrors -gt 0 ]; then - echo "***$nerrors errors encountered***" - exit 1 -else - echo "No error encountered" - exit 0 -fi diff --git a/test/testlinks_env.sh.in b/test/testlinks_env.sh.in deleted file mode 100644 index 09074c3..0000000 --- a/test/testlinks_env.sh.in +++ /dev/null @@ -1,42 +0,0 @@ -#! /bin/sh -# -# Copyright by The HDF Group. -# Copyright by the Board of Trustees of the University of Illinois. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. -# -# Test for external link with environment variable: HDF5_EXT_PREFIX - -srcdir=@srcdir@ - -nerrors=0 - -############################################################################## -############################################################################## -### T H E T E S T S ### -############################################################################## -############################################################################## - -# test for external links with HDF5_EXT_PREFIX -echo "Testing external link with HDF5_EXT_PREFIX" -TEST_NAME=links_env # The test name -TEST_BIN=`pwd`/$TEST_NAME # The path of the test binary -ENVCMD="env HDF5_EXT_PREFIX=.:tmp_links_env" # The environment variable & value -# -# Run the test -echo "$ENVCMD $RUNSERIAL $TEST_BIN" -$ENVCMD $RUNSERIAL $TEST_BIN -exitcode=$? -if [ $exitcode -eq 0 ]; then - echo "Test for HDF5_EXT_PREFIX PASSED" -else - nerrors="`expr $nerrors + 1`" - echo "***Error encountered for HDF5_EXT_PREFIX test***" -fi -exit $nerrors diff --git a/test/testswmr.pwsh.in b/test/testswmr.pwsh.in deleted file mode 100644 index de7a57a..0000000 --- a/test/testswmr.pwsh.in +++ /dev/null @@ -1,519 +0,0 @@ -# Copyright by The HDF Group. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. -# -# Tests for the swmr feature. - -$srcdir = '@srcdir@' -$utils_testdir=@H5_UTILS_TEST_BUILDDIR@ -$testdir=@H5_TEST_BUILDDIR@ - -############################################################################### -## test parameters -############################################################################### - -$Nreaders = 5 # number of readers to launch -$Nrdrs_spa = 3 # number of sparse readers to launch -$Nrecords = 200000 # number of records to write -$Nrecs_rem = 40000 # number of times to shrink -$Nrecs_spa = 20000 # number of records to write in the sparse test -$Nsecs_add = 5 # number of seconds per read interval -$Nsecs_rem = 3 # number of seconds per read interval -$Nsecs_addrem = 8 # number of seconds per read interval -$nerrors = 0 - -############################################################################### -## definitions for message file to coordinate test runs -############################################################################### -$WRITER_MESSAGE = 'SWMR_WRITER_MESSAGE' # The message file created by writer that the open is complete - # This should be the same as the define in "test/swmr_common.h" -$MESSAGE_TIMEOUT = 300 # Message timeout length in secs - # This should be the same as the define in "test/h5test.h" - -# Print a line-line message left justified in a field of 70 characters -# beginning with the word "Testing". -# -function Test-WithSpaces { - $SPACES=' ' - #Write-Output "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' -} - -# To wait for the writer message file or till the maximum # of seconds is reached -# $Message is the message file to wait for -# This performs similar function as the routine h5_wait_message() in test/h5test.c -function Wait-Message { - [CmdletBinding()] - param ( - [Parameter(Mandatory, ValueFromPipeline)] - [string]$TestPath, # Get the path of the message file to wait for - [Parameter(Mandatory, ValueFromPipeline)] - [string]$Message # Get the name of the message file to wait for - ) - - PROCESS { - $message = Join-Path -Path $TestPath -ChildPath $Message - $t0 = Get-Date # Get current time - $t1 = $t0 - $difft = New-Timespan -Start $t0 -End $t1 # Initialize the time difference - $mexist = 0 # Indicate whether the message file is found - while ($difft.TotalSeconds -lt $MESSAGE_TIMEOUT) { # Loop till message times out - $t1 = Get-Date # Get current time in seconds - $difft = New-Timespan -Start $t0 -End $t1 # Calculate the time difference - #Write-Output "Check for $message : time=$difft" - if ([System.IO.File]::Exists($message)) { # If message file is found: - $mexist = 1 # indicate the message file is found - Remove-Item $message # remove the message file - break # get out of the while loop - } - } - if ($mexist -eq 0) { - # Issue warning that the writer message file is not found, continue with launching the reader(s) - Write-Warning -Message "$WRITER_MESSAGE is not found after waiting $MESSAGE_TIMEOUT seconds" - } - else { - Write-Output "$WRITER_MESSAGE is found" - } - } -} - -############################################################################### -## Main -## -## Modifications: -## Vailin Choi; July 2013 -## Add waiting of message file before launching the reader(s). -## Due to the implementation of file locking, coordination -## is needed in file opening for the writer/reader tests -## to proceed as expected. -## -############################################################################### - -# Check to see if the VFD specified by the HDF5_DRIVER environment variable -# supports SWMR. -$testprog = Join-Path -Path $utils_testdir -ChildPath swmr_check_compat_vfd.exe -$rp = Start-Process -FilePath $testprog -PassThru -Wait -NoNewWindow -if ($rp.ExitCode -ne 0) { - Write-Output "" - Write-Output "The VFD specified by the HDF5_DRIVER environment variable" - Write-Output "does not support SWMR." - Write-Output "" - Write-Output "SWMR acceptance tests skipped" - Write-Output "" - exit 0 -} - -Set-Location -Path swmr_test -$testdir = Join-Path -Path $testdir -ChildPath swmr_test - -# Loop over index types -$IndexTypeLoop = '-i ea', '-i b2' -foreach ($index_type in $IndexTypeLoop) { - # Try with and without compression - $CompressLoop = '', '-c 5' - foreach ($compress in $CompressLoop) { - Write-Output "" - Write-Output "*******************************************************************************" - Write-Output "** Loop testing parameters: $index_type $compress" - Write-Output "*******************************************************************************" - Write-Output "" - Write-Output "" - Write-Output "###############################################################################" - Write-Output "## Generator test" - Write-Output "###############################################################################" - # Launch the Generator without SWMR_WRITE - Write-Output "launch the swmr_generator" - $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -ArgumentList "$compress $index_type" -PassThru -Wait - if ($rp.ExitCode -ne 0) { - Write-Warning "generator had error" - $nerrors += 1 - } - - # Launch the Generator with SWMR_WRITE - Write-Output "launch the swmr_generator with SWMR_WRITE" - $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -ArgumentList "-s $compress $index_type" -PassThru -Wait - if ($rp.ExitCode -ne 0) { - Write-Warning "generator had error" - $nerrors += 1 - } - - # Check for error and exit if one occurred - #Write-Output "nerrors=$nerrors" - if ($nerrors -ne 0) { - Write-Warning "SWMR tests failed with $nerrors errors." - exit 1 - } - - Write-Output "" - Write-Output "###############################################################################" - Write-Output "## Use H5Fstart_swmr_write() to enable SWMR writing mode" - Write-Output "###############################################################################" - - # Remove any possible writer message file before launching writer - if ([System.IO.File]::Exists($WRITER_MESSAGE)) { # If message file is found - Remove-Item $WRITER_MESSAGE - } - # - # Launch the Writer - Write-Output "launch the swmr_start_writer" - $seed = "" # Put -r command here - $rp = Start-Process -FilePath $testdir/swmr_start_write.exe -NoNewWindow -PassThru -ArgumentList "$compress $index_type $Nrecords $seed" 2>&1 |tee swmr_writer.out - $pid_writer = $rp.id - #Write-Output "pid_writer=$pid_writer" - - # Wait for message from writer process before starting reader(s) - Wait-Message $testdir $WRITER_MESSAGE - - # - # Launch the Readers - #declare -a seeds = ( ... ) - Write-Output "launch $Nreaders swmr_readers" - $pid_readers = @() - $n = 0 - while ($n -lt $Nreaders) { - #seed = "-r ${seeds[$n]}" - $seed = "" - $rp = Start-Process -FilePath $testdir/swmr_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_add $seed" 2>&1 | tee swmr_reader.out.$n - $pid_readers += $rp.id - $n += 1 - } - #Write-Output "pid_readers=$pid_readers" - - # Collect exit code of the readers first because they usually finish - # before the writer. - foreach ($xpid in $pid_readers) { - #Write-Output "checked reader $xpid" - $result = Wait-Process -Id $xpid - if ($result.ExitCode -ne 0) { - Write-Warning "reader had error" - $nerrors += 1 - } - } - - # Collect exit code of the writer - #Write-Output "checked writer $pid_writer" - $result = Wait-Process -Id $pid_writer - if ($result.ExitCode -ne 0) { - Write-Warning "writer had error" - $nerrors += 1 - } - - # Check for error and exit if one occurred - #Write-Output "nerrors=$nerrors" - if ($nerrors -ne 0) { - Write-Warning "SWMR tests failed with $nerrors errors." - Write-Warning "(Writer and reader output preserved)" - exit 1 - } - - # Clean up output files - Remove-Item swmr_writer.out - Remove-Item swmr_reader.out.* - - Write-Output "" - Write-Output "###############################################################################" - Write-Output "## Writer test - test expanding the dataset" - Write-Output "###############################################################################" - - # Launch the Generator - Write-Output "launch the swmr_generator" - $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -PassThru -Wait -ArgumentList "-s $compress $index_type" - if ($rp.ExitCode -ne 0) { - Write-Warning "generator had error" - $nerrors += 1 - } - - # Remove any possible writer message file before launching writer - Remove-Item $WRITER_MESSAGE - # - # Launch the Writer - Write-Output "launch the swmr_writer" - $seed = "" # Put -r command here - $rp = Start-Process -FilePath $testdir/swmr_writer.exe -NoNewWindow -PassThru -ArgumentList "-o $Nrecords $seed" 2>&1 |tee swmr_writer.out - $pid_writer = $rp.id - #Write-Output "pid_writer=$pid_writer" - - # Wait for message from writer process before starting reader(s) - Wait-Message $testdir $WRITER_MESSAGE - # - # Launch the Readers - #declare -a seeds = ( ... ) - Write-Output "launch $Nreaders swmr_readers" - $pid_readers = @() - $n = 0 - while ($n -lt $Nreaders) { - #seed = "-r ${seeds[$n]}" - $seed = "" - $rp = Start-Process -FilePath $testdir/swmr_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_add $seed" 2>&1 |tee swmr_reader.out.$n - $pid_readers += $rp.id - $n += 1 - } - #Write-Output "pid_readers=$pid_readers" - - # Collect exit code of the readers first because they usually finish - # before the writer. - foreach ($xpid in $pid_readers) { - #Write-Output "checked reader $xpid" - $result = Wait-Process -Id $xpid - if ($result.ExitCode -ne 0) { - Write-Warning "reader had error" - $nerrors += 1 - } - } - - # Collect exit code of the writer - #Write-Output "checked writer $pid_writer" - $result = Wait-Process -Id $pid_writer - if ($result.ExitCode -ne 0) { - Write-Warning "writer had error" - $nerrors += 1 - } - - # Check for error and exit if one occurred - #Write-Output "nerrors=$nerrors" - if ($nerrors -ne 0) { - Write-Warning "SWMR tests failed with $nerrors errors." - Write-Warning "(Writer and reader output preserved)" - exit 1 - } - - # Clean up output files - Remove-Item swmr_writer.out - Remove-Item swmr_reader.out.* - - Write-Output "" - Write-Output "###############################################################################" - Write-Output "## Remove test - test shrinking the dataset" - Write-Output "###############################################################################" - - # Remove any possible writer message file before launching writer - Remove-Item $WRITER_MESSAGE - # Launch the Remove Writer - Write-Output "launch the swmr_remove_writer" - $seed = "" # Put -r command here - $rp = Start-Process -FilePath $testdir/swmr_remove_writer.exe -NoNewWindow -PassThru -ArgumentList "-o $Nrecs_rem $seed" 2>&1 |tee swmr_writer.out - $pid_writer = $rp.id - #Write-Output "pid_writer=$pid_writer" - - # Wait for message from writer process before starting reader(s) - Wait-Message $testdir $WRITER_MESSAGE - # - # Launch the Remove Readers - #declare -a seeds = ( ... ) - $n = 0 - $pid_readers = @() - Write-Output "launch $Nreaders swmr_remove_readers" - while ($n -lt $Nreaders) { - #seed = "-r ${seeds[$n]}" - $seed = "" - $rp = Start-Process -FilePath $testdir/swmr_remove_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_rem $seed" 2>&1 |tee swmr_reader.out.$n - $pid_readers += $rp.id - $n += 1 - } - #Write-Output "pid_readers=$pid_readers" - - # Collect exit code of the readers first because they usually finish - # before the writer. - foreach ($xpid in $pid_readers) { - #Write-Output "checked reader $xpid" - $result = Wait-Process -Id $xpid - if ($result.ExitCode -ne 0) { - Write-Warning "reader had error" - $nerrors += 1 - } - } - - # Collect exit code of the writer - #Write-Output "checked writer $pid_writer" - $result = Wait-Process -Id $pid_writer - if ($result.ExitCode -ne 0) { - Write-Warning "writer had error" - $nerrors += 1 - } - - # Check for error and exit if one occurred - #Write-Output "nerrors=$nerrors" - if ($nerrors -ne 0) { - Write-Warning "SWMR tests failed with $nerrors errors." - Write-Warning "(Writer and reader output preserved)" - exit 1 - } - - # Clean up output files - Remove-Item swmr_writer.out - Remove-Item swmr_reader.out.* - - Write-Output "" - Write-Output "###############################################################################" - Write-Output "## Add/remove test - randomly grow or shrink the dataset" - Write-Output "###############################################################################" - - # Launch the Generator - Write-Output "launch the swmr_generator" - $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow-PassThru -Wait -ArgumentList "$compress $index_type" - if ($rp.ExitCode -ne 0) { - Write-Warning "generator had error" - $nerrors += 1 - } - - # Launch the Writer (not in parallel - just to rebuild the datasets) - Write-Output "launch the swmr_writer" - $seed = "" # Put -r command here - $rp = Start-Process -FilePath $testdir/swmr_writer.exe -NoNewWindow -PassThru -Wait -ArgumentList "$Nrecords $seed" - if ($rp.ExitCode -ne 0) { - Write-Warning "writer had error" - $nerrors += 1 - } - - # Remove any possible writer message file before launching writer - Remove-Item $WRITER_MESSAGE - # - # Launch the Add/Remove Writer - Write-Output "launch the swmr_addrem_writer" - $seed = "" # Put -r command here - $rp = Start-Process -FilePath $testdir/swmr_addrem_writer.exe -NoNewWindow -PassThru -ArgumentList "$Nrecords $seed" 2>&1 |tee swmr_writer.out - $pid_writer = $rp.id - #Write-Output "pid_writer=$pid_writer" - - # Wait for message from writer process before starting reader(s) - Wait-Message $testdir $WRITER_MESSAGE - # - # Launch the Add/Remove Readers - #declare -a seeds = ( ... ) - $n = 0 - $pid_readers = @() - Write-Output "launch $Nreaders swmr_remove_readers" - while ($n -lt $Nreaders) { - #seed = "-r ${seeds[$n]}" - $seed = "" - $rp = Start-Process -FilePath $testdir/swmr_remove_reader.exe -NoNewWindow -PassThru -ArgumentList "$Nsecs_addrem $seed" 2>&1 |tee swmr_reader.out.$n - $pid_readers += $rp.id - $n += 1 - } - #Write-Output "pid_readers=$pid_readers" - - # Collect exit code of the readers first because they usually finish - # before the writer. - foreach ($xpid in $pid_readers) { - #Write-Output "checked reader $xpid" - $result = Wait-Process -Id $xpid - if ($result.ExitCode -ne 0) { - Write-Warning "reader had error" - $nerrors += 1 - } - } - - # Collect exit code of the writer - #Write-Output "checked writer $pid_writer" - $result = Wait-Process -Id $pid_writer - if ($result.ExitCode -ne 0) { - Write-Warning "writer had error" - $nerrors += 1 - } - - # Check for error and exit if one occurred - #Write-Output "nerrors=$nerrors" - if ($nerrors -ne 0) { - Write-Warning "SWMR tests failed with $nerrors errors." - Write-Warning "(Writer and reader output preserved)" - exit 1 - } - - # Clean up output files - Remove-Item swmr_writer.out - Remove-Item swmr_reader.out.* - - Write-Output "" - Write-Output "###############################################################################" - Write-Output "## Sparse writer test - test writing to random locations in the dataset" - Write-Output "###############################################################################" - - # Launch the Generator - # NOTE: Random seed is shared between readers and writers and is - # created by the generator. - Write-Output "launch the swmr_generator" - $seed = "" # Put -r command here - $rp = Start-Process -FilePath $testdir/swmr_generator.exe -NoNewWindow -PassThru -Wait -ArgumentList "$compress $index_type $seed" - if ($rp.ExitCode -ne 0) { - Write-Warning "generator had error" - $nerrors += 1 - } - - # Remove any possible writer message file before launching writer - Remove-Item $WRITER_MESSAGE - # Launch the Sparse writer - Write-Output "launch the swmr_sparse_writer" - $rp = Start-Process -FilePath $testdir/swmr_sparse_writer.exe -NoNewWindow -PassThru -ArgumentList "$Nrecs_spa" 2>&1 |tee swmr_writer.out - $pid_writer = $rp.Id - #Write-Output "pid_writer=$pid_writer" - - # Wait for message from writer process before starting reader(s) - Wait-Message $testdir $WRITER_MESSAGE - # - # Launch the Sparse readers - $n = 0 - $pid_readers = @() - Write-Output "launch $Nrdrs_spa swmr_sparse_readers" - while ($n -lt $Nrdrs_spa) { - # The sparse reader spits out a LOT of data so it's set to 'quiet' - $rp = Start-Process -FilePath $testdir/swmr_sparse_reader.exe -NoNewWindow -PassThru -ArgumentList "-q $Nrecs_spa" 2>&1 |tee swmr_reader.out.$n - $pid_readers += $rp.id - $n += 1 - } - #Write-Output "pid_readers=$pid_readers" - - # Collect exit code of the writer - #Write-Output "checked writer $pid_writer" - $result = Wait-Process -Id $pid_writer - if ($result.ExitCode -ne 0) { - Write-Warning "writer had error" - $nerrors += 1 - } - - # Collect exit code of the readers - foreach ($xpid in $pid_readers) { - #Write-Output "checked reader $xpid" - $result = Wait-Process -Id $xpid - if ($result.ExitCode -ne 0) { - Write-Warning "reader had error" - $nerrors += 1 - } - } - - # Check for error and exit if one occurred - #Write-Output "nerrors=$nerrors" - if ($nerrors -ne 0) { - Write-Warning "SWMR tests failed with $nerrors errors." - Write-Warning "(Writer and reader output preserved)" - exit 1 - } - - # Clean up output files - Remove-Item swmr_writer.out - Remove-Item swmr_reader.out.* - } -} - -############################################################################### -## Report and exit -############################################################################### -cd .. -#Write-Output "nerrors=$nerrors" -if ($nerrors -eq 0) { - Write-Output "SWMR tests passed." -# if test -z "$HDF5_NOCLEANUP"; then -# # delete the test directory -# Remove-Item swmr_test -Recurse -# fi - exit 0 -} -else { - Write-Warning "SWMR tests failed with $nerrors errors." - exit 1 -} - diff --git a/test/testswmr.sh.in b/test/testswmr.sh.in deleted file mode 100644 index 771fe4a..0000000 --- a/test/testswmr.sh.in +++ /dev/null @@ -1,563 +0,0 @@ -#! /bin/bash -# -# Copyright by The HDF Group. -# Copyright by the Board of Trustees of the University of Illinois. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. -# -# Tests for the swmr feature. -# -# Created: -# Albert Cheng, 2009/07/22 - -srcdir=@srcdir@ -utils_testdir=@abs_top_builddir@/@H5_UTILS_TEST_BUILDDIR@ -testdir=@abs_top_builddir@/@H5_TEST_BUILDDIR@ - -############################################################################### -## test parameters -############################################################################### - -Nreaders=5 # number of readers to launch -Nrdrs_spa=3 # number of sparse readers to launch -Nrecords=200000 # number of records to write -Nrecs_rem=40000 # number of times to shrink -Nrecs_spa=20000 # number of records to write in the sparse test -Nsecs_add=5 # number of seconds per read interval -Nsecs_rem=3 # number of seconds per read interval -Nsecs_addrem=8 # number of seconds per read interval -nerrors=0 - -############################################################################### -## definitions for message file to coordinate test runs -############################################################################### -WRITER_MESSAGE=SWMR_WRITER_MESSAGE # The message file created by writer that the open is complete - # This should be the same as the define in "test/swmr_common.h" -MESSAGE_TIMEOUT=300 # Message timeout length in secs - # This should be the same as the define in "test/h5test.h" - -############################################################################### -## short hands and function definitions -############################################################################### -DPRINT=: # Set to "echo Debug:" for debugging printing, - # else ":" for noop. -IFDEBUG=: # Set to null to turn on debugging, else ":" for noop. - -# Print a line-line message left justified in a field of 70 characters -# beginning with the word "Testing". -# -TESTING() { - SPACES=" " - echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' -} - -# To wait for the writer message file or till the maximum # of seconds is reached -# $1 is the message file to wait for -# This performs similar function as the routine h5_wait_message() in test/h5test.c -WAIT_MESSAGE() { - message=$1 # Get the name of the message file to wait for - t0=`date +%s` # Get current time in seconds - difft=0 # Initialize the time difference - mexist=0 # Indicate whether the message file is found - while [ $difft -lt $MESSAGE_TIMEOUT ] ; # Loop till message times out - do - t1=`date +%s` # Get current time in seconds - difft=`expr $t1 - $t0` # Calculate the time difference - if [ -e $message ]; then # If message file is found: - mexist=1 # indicate the message file is found - rm $message # remove the message file - break # get out of the while loop - fi - done; - if test $mexist -eq 0; then - # Issue warning that the writer message file is not found, continue with launching the reader(s) - echo warning: $WRITER_MESSAGE is not found after waiting $MESSAGE_TIMEOUT seconds - else - echo $WRITER_MESSAGE is found - fi -} - -############################################################################### -## Main -## -## Modifications: -## Vailin Choi; July 2013 -## Add waiting of message file before launching the reader(s). -## Due to the implementation of file locking, coordination -## is needed in file opening for the writer/reader tests -## to proceed as expected. -## -############################################################################### - -# Check to see if the VFD specified by the HDF5_DRIVER environment variable -# supports SWMR. -$utils_testdir/swmr_check_compat_vfd -rc=$? -if [ $rc -ne 0 ] ; then - echo - echo "The VFD specified by the HDF5_DRIVER environment variable" - echo "does not support SWMR." - echo - echo "SWMR acceptance tests skipped" - echo - exit 0 -fi - -# Parse options (none accepted at this time) -while [ $# -gt 0 ]; do - case "$1" in - *) # unknown option - echo "$0: Unknown option ($1)" - exit 1 - ;; - esac -done - -# HDF5 has several tests that create and delete signal files to communicate -# between processes, and it seems that even though the names of the files are -# different, occasionally the wrong file is deleted, interrupting the flow of -# the test. Running each of these tests in its own directory should eliminate -# the problem. -mkdir swmr_test -for FILE in swmr*; do - case "$FILE" in - *.o) continue ;; ## don't copy the .o files - esac - if test -f "$FILE" ; then - cp $FILE swmr_test - fi -done - -# With the --disable-shared option, swmr program files are built in the test -# directory, otherwise they are in test/.libs with a corresponding wrapper -# script in the test directory. The programs or wrapper scripts in test should -# always be copied, swmr files in .libs should be copied only if they exists. -if [ -f .libs/swmr ]; then - mkdir swmr_test/.libs - for FILE in .libs/swmr*; do - case "$FILE" in - *.o) continue ;; ## don't copy the .o files - esac - if test -f "$FILE" ; then - cp $FILE swmr_test/.libs - fi - done -fi - -cd swmr_test - - -# Loop over index types -for index_type in "-i ea" "-i b2" -do - # Try with and without compression - for compress in "" "-c 5" - do - echo - echo "*******************************************************************************" - echo "** Loop testing parameters: $index_type $compress" - echo "*******************************************************************************" - echo - echo - echo "###############################################################################" - echo "## Generator test" - echo "###############################################################################" - # Launch the Generator without SWMR_WRITE - echo launch the swmr_generator - $testdir/swmr_generator $compress $index_type - if test $? -ne 0; then - echo generator had error - nerrors=`expr $nerrors + 1` - fi - - # Launch the Generator with SWMR_WRITE - echo launch the swmr_generator with SWMR_WRITE - $testdir/swmr_generator -s $compress $index_type - if test $? -ne 0; then - echo generator had error - nerrors=`expr $nerrors + 1` - fi - - # Check for error and exit if one occurred - $DPRINT nerrors=$nerrors - if test $nerrors -ne 0 ; then - echo "SWMR tests failed with $nerrors errors." - exit 1 - fi - - echo - echo "###############################################################################" - echo "## Use H5Fstart_swmr_write() to enable SWMR writing mode" - echo "###############################################################################" - - # Remove any possible writer message file before launching writer - rm -f $WRITER_MESSAGE - # - # Launch the Writer - echo launch the swmr_start_writer - seed="" # Put -r command here - $testdir/swmr_start_write $compress $index_type $Nrecords $seed 2>&1 |tee swmr_writer.out & - pid_writer=$! - $DPRINT pid_writer=$pid_writer - - # Wait for message from writer process before starting reader(s) - WAIT_MESSAGE $WRITER_MESSAGE - - # - # Launch the Readers - #declare -a seeds=( ... ) - echo launch $Nreaders swmr_readers - pid_readers="" - n=0 - while [ $n -lt $Nreaders ]; do - #seed="-r ${seeds[$n]}" - seed="" - $testdir/swmr_reader $Nsecs_add $seed 2>&1 |tee swmr_reader.out.$n & - pid_readers="$pid_readers $!" - n=`expr $n + 1` - done - $DPRINT pid_readers=$pid_readers - $IFDEBUG ps - - # Collect exit code of the readers first because they usually finish - # before the writer. - for xpid in $pid_readers; do - $DPRINT checked reader $xpid - wait $xpid - if test $? -ne 0; then - echo reader had error - nerrors=`expr $nerrors + 1` - fi - done - - # Collect exit code of the writer - $DPRINT checked writer $pid_writer - wait $pid_writer - if test $? -ne 0; then - echo writer had error - nerrors=`expr $nerrors + 1` - fi - - # Check for error and exit if one occurred - $DPRINT nerrors=$nerrors - if test $nerrors -ne 0 ; then - echo "SWMR tests failed with $nerrors errors." - echo "(Writer and reader output preserved)" - exit 1 - fi - - # Clean up output files - rm -f swmr_writer.out - rm -f swmr_reader.out.* - - echo - echo "###############################################################################" - echo "## Writer test - test expanding the dataset" - echo "###############################################################################" - - # Launch the Generator - echo launch the swmr_generator - $testdir/swmr_generator -s $compress $index_type - if test $? -ne 0; then - echo generator had error - nerrors=`expr $nerrors + 1` - fi - - # Remove any possible writer message file before launching writer - rm -f $WRITER_MESSAGE - # - # Launch the Writer - echo launch the swmr_writer - seed="" # Put -r command here - $testdir/swmr_writer -o $Nrecords $seed 2>&1 |tee swmr_writer.out & - pid_writer=$! - $DPRINT pid_writer=$pid_writer - - # Wait for message from writer process before starting reader(s) - WAIT_MESSAGE $WRITER_MESSAGE - # - # Launch the Readers - #declare -a seeds=( ... ) - echo launch $Nreaders swmr_readers - pid_readers="" - n=0 - while [ $n -lt $Nreaders ]; do - #seed="-r ${seeds[$n]}" - seed="" - $testdir/swmr_reader $Nsecs_add $seed 2>&1 |tee swmr_reader.out.$n & - pid_readers="$pid_readers $!" - n=`expr $n + 1` - done - $DPRINT pid_readers=$pid_readers - $IFDEBUG ps - - # Collect exit code of the readers first because they usually finish - # before the writer. - for xpid in $pid_readers; do - $DPRINT checked reader $xpid - wait $xpid - if test $? -ne 0; then - echo reader had error - nerrors=`expr $nerrors + 1` - fi - done - - # Collect exit code of the writer - $DPRINT checked writer $pid_writer - wait $pid_writer - if test $? -ne 0; then - echo writer had error - nerrors=`expr $nerrors + 1` - fi - - # Check for error and exit if one occurred - $DPRINT nerrors=$nerrors - if test $nerrors -ne 0 ; then - echo "SWMR tests failed with $nerrors errors." - echo "(Writer and reader output preserved)" - exit 1 - fi - - # Clean up output files - rm -f swmr_writer.out - rm -f swmr_reader.out.* - - echo - echo "###############################################################################" - echo "## Remove test - test shrinking the dataset" - echo "###############################################################################" - - # Remove any possible writer message file before launching writer - rm -f $WRITER_MESSAGE - # Launch the Remove Writer - echo launch the swmr_remove_writer - seed="" # Put -r command here - $testdir/swmr_remove_writer -o $Nrecs_rem $seed 2>&1 |tee swmr_writer.out & - pid_writer=$! - $DPRINT pid_writer=$pid_writer - - # Wait for message from writer process before starting reader(s) - WAIT_MESSAGE $WRITER_MESSAGE - # - # Launch the Remove Readers - #declare -a seeds=( ... ) - n=0 - pid_readers="" - echo launch $Nreaders swmr_remove_readers - while [ $n -lt $Nreaders ]; do - #seed="-r ${seeds[$n]}" - seed="" - $testdir/swmr_remove_reader $Nsecs_rem $seed 2>&1 |tee swmr_reader.out.$n & - pid_readers="$pid_readers $!" - n=`expr $n + 1` - done - $DPRINT pid_readers=$pid_readers - $IFDEBUG ps - - # Collect exit code of the readers first because they usually finish - # before the writer. - for xpid in $pid_readers; do - $DPRINT checked reader $xpid - wait $xpid - if test $? -ne 0; then - echo reader had error - nerrors=`expr $nerrors + 1` - fi - done - - # Collect exit code of the writer - $DPRINT checked writer $pid_writer - wait $pid_writer - if test $? -ne 0; then - echo writer had error - nerrors=`expr $nerrors + 1` - fi - - # Check for error and exit if one occurred - $DPRINT nerrors=$nerrors - if test $nerrors -ne 0 ; then - echo "SWMR tests failed with $nerrors errors." - echo "(Writer and reader output preserved)" - exit 1 - fi - - # Clean up output files - rm -f swmr_writer.out - rm -f swmr_reader.out.* - - echo - echo "###############################################################################" - echo "## Add/remove test - randomly grow or shrink the dataset" - echo "###############################################################################" - - # Launch the Generator - echo launch the swmr_generator - $testdir/swmr_generator $compress $index_type - if test $? -ne 0; then - echo generator had error - nerrors=`expr $nerrors + 1` - fi - - # Launch the Writer (not in parallel - just to rebuild the datasets) - echo launch the swmr_writer - seed="" # Put -r command here - $testdir/swmr_writer $Nrecords $seed - if test $? -ne 0; then - echo writer had error - nerrors=`expr $nerrors + 1` - fi - - # Remove any possible writer message file before launching writer - rm -f $WRITER_MESSAGE - # - # Launch the Add/Remove Writer - echo launch the swmr_addrem_writer - seed="" # Put -r command here - $testdir/swmr_addrem_writer $Nrecords $seed 2>&1 |tee swmr_writer.out & - pid_writer=$! - $DPRINT pid_writer=$pid_writer - - # Wait for message from writer process before starting reader(s) - WAIT_MESSAGE $WRITER_MESSAGE - # - # Launch the Add/Remove Readers - #declare -a seeds=( ... ) - n=0 - pid_readers="" - echo launch $Nreaders swmr_remove_readers - while [ $n -lt $Nreaders ]; do - #seed="-r ${seeds[$n]}" - seed="" - $testdir/swmr_remove_reader $Nsecs_addrem $seed 2>&1 |tee swmr_reader.out.$n & - pid_readers="$pid_readers $!" - n=`expr $n + 1` - done - $DPRINT pid_readers=$pid_readers - $IFDEBUG ps - - # Collect exit code of the readers first because they usually finish - # before the writer. - for xpid in $pid_readers; do - $DPRINT checked reader $xpid - wait $xpid - if test $? -ne 0; then - echo reader had error - nerrors=`expr $nerrors + 1` - fi - done - - # Collect exit code of the writer - $DPRINT checked writer $pid_writer - wait $pid_writer - if test $? -ne 0; then - echo writer had error - nerrors=`expr $nerrors + 1` - fi - - # Check for error and exit if one occurred - $DPRINT nerrors=$nerrors - if test $nerrors -ne 0 ; then - echo "SWMR tests failed with $nerrors errors." - echo "(Writer and reader output preserved)" - exit 1 - fi - - # Clean up output files - rm -f swmr_writer.out - rm -f swmr_reader.out.* - - echo - echo "###############################################################################" - echo "## Sparse writer test - test writing to random locations in the dataset" - echo "###############################################################################" - - # Launch the Generator - # NOTE: Random seed is shared between readers and writers and is - # created by the generator. - echo launch the swmr_generator - seed="" # Put -r command here - $testdir/swmr_generator $compress $index_type $seed - if test $? -ne 0; then - echo generator had error - nerrors=`expr $nerrors + 1` - fi - - # Remove any possible writer message file before launching writer - rm -f $WRITER_MESSAGE - # Launch the Sparse writer - echo launch the swmr_sparse_writer - nice -n 20 $testdir/swmr_sparse_writer $Nrecs_spa 2>&1 |tee swmr_writer.out & - pid_writer=$! - $DPRINT pid_writer=$pid_writer - - # Wait for message from writer process before starting reader(s) - WAIT_MESSAGE $WRITER_MESSAGE - # - # Launch the Sparse readers - n=0 - pid_readers="" - echo launch $Nrdrs_spa swmr_sparse_readers - while [ $n -lt $Nrdrs_spa ]; do - # The sparse reader spits out a LOT of data so it's set to 'quiet' - $testdir/swmr_sparse_reader -q $Nrecs_spa 2>&1 |tee swmr_reader.out.$n & - pid_readers="$pid_readers $!" - n=`expr $n + 1` - done - $DPRINT pid_readers=$pid_readers - $IFDEBUG ps - - # Collect exit code of the writer - $DPRINT checked writer $pid_writer - wait $pid_writer - if test $? -ne 0; then - echo writer had error - nerrors=`expr $nerrors + 1` - fi - - # Collect exit code of the readers - for xpid in $pid_readers; do - $DPRINT checked reader $xpid - wait $xpid - if test $? -ne 0; then - echo reader had error - nerrors=`expr $nerrors + 1` - fi - done - - # Check for error and exit if one occurred - $DPRINT nerrors=$nerrors - if test $nerrors -ne 0 ; then - echo "SWMR tests failed with $nerrors errors." - echo "(Writer and reader output preserved)" - exit 1 - fi - - # Clean up output files - rm -f swmr_writer.out - rm -f swmr_reader.out.* - done -done - -############################################################################### -## Report and exit -############################################################################### -cd .. -$DPRINT nerrors=$nerrors -if test $nerrors -eq 0 ; then - echo "SWMR tests passed." - if test -z "$HDF5_NOCLEANUP"; then - # delete the test directory - rm -rf swmr_test - fi - exit 0 -else - echo "SWMR tests failed with $nerrors errors." - exit 1 -fi - diff --git a/test/testvds_env.sh.in b/test/testvds_env.sh.in deleted file mode 100644 index 39f13a5..0000000 --- a/test/testvds_env.sh.in +++ /dev/null @@ -1,44 +0,0 @@ -#! /bin/sh -# -# Copyright by The HDF Group. -# Copyright by the Board of Trustees of the University of Illinois. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. -# -# Test for external file with environment variable: HDF5_VDS_PREFIX - -srcdir=@srcdir@ - -nerrors=0 - -############################################################################## -############################################################################## -### T H E T E S T S ### -############################################################################## -############################################################################## - -# test for VDS with HDF5_VDS_PREFIX -echo "Testing basic virtual dataset I/O via H5Pset_vds_prefix(): all selection with ENV prefix" -TEST_NAME=vds_env # The test name -TEST_BIN=`pwd`/$TEST_NAME # The path of the test binary -ENVCMD="env HDF5_VDS_PREFIX=\${ORIGIN}/tmp_vds_env" # Set the environment variable & value -UNENVCMD="unset HDF5_VDS_PREFIX" # Unset the environment variable & value -# -# Run the test -# echo "$ENVCMD $RUNSERIAL $TEST_BIN" -$ENVCMD $RUNSERIAL $TEST_BIN -exitcode=$? -if [ $exitcode -eq 0 ]; then - echo "Test prefix for HDF5_VDS_PREFIX PASSED" -else - nerrors="`expr $nerrors + 1`" - echo "***Error encountered for HDF5_VDS_PREFIX test***" -fi -$UNENVCMD -exit $nerrors diff --git a/test/testvdsswmr.pwsh.in b/test/testvdsswmr.pwsh.in deleted file mode 100644 index bf5aabb..0000000 --- a/test/testvdsswmr.pwsh.in +++ /dev/null @@ -1,190 +0,0 @@ -# Copyright by The HDF Group. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. -# -# Tests for the swmr feature using virtual datasets. - -$srcdir = '@srcdir@' -$utils_testdir=@H5_UTILS_TEST_BUILDDIR@ -$testdir=@H5_TEST_BUILDDIR@ - -############################################################################### -## test parameters -############################################################################### - -$Nwriters = 6 # number of writers (1 per source dataset) -$Nreaders = 5 # number of readers to launch -$nerrors = 0 - -############################################################################### -## definitions for message file to coordinate test runs -############################################################################### -$WRITER_MESSAGE = 'SWMR_WRITER_MESSAGE' # The message file created by writer that the open is complete - # This should be the same as the define in "test/swmr_common.h" -$MESSAGE_TIMEOUT = 300 # Message timeout length in secs - # This should be the same as the define in "test/h5test.h" - -# Print a line-line message left justified in a field of 70 characters -# beginning with the word "Testing". -# -function Test-WithSpaces { - $SPACES=' ' - #Write-Output "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' -} - -# To wait for the writer message file or till the maximum # of seconds is reached -# $Message is the message file to wait for -# This performs similar function as the routine h5_wait_message() in test/h5test.c -function Wait-Message { - [CmdletBinding()] - param ( - [Parameter(Mandatory, ValueFromPipeline)] - [string]$TestPath, # Get the path of the message file to wait for - [Parameter(Mandatory, ValueFromPipeline)] - [string]$Message # Get the name of the message file to wait for - ) - - PROCESS { - $message = Join-Path -Path $TestPath -ChildPath $Message - $t0 = Get-Date # Get current time - $t1 = $t0 - $difft = New-Timespan -Start $t0 -End $t1 # Initialize the time difference - $mexist = 0 # Indicate whether the message file is found - while ($difft.TotalSeconds -lt $MESSAGE_TIMEOUT) { # Loop till message times out - $t1 = Get-Date # Get current time in seconds - $difft = New-Timespan -Start $t0 -End $t1 # Calculate the time difference - #Write-Output "Check for $message : time=$difft" - if ([System.IO.File]::Exists($message)) { # If message file is found: - $mexist = 1 # indicate the message file is found - Remove-Item $message # remove the message file - break # get out of the while loop - } - } - if ($mexist -eq 0) { - # Issue warning that the writer message file is not found, continue with launching the reader(s) - Write-Warning -Message "$WRITER_MESSAGE is not found after waiting $MESSAGE_TIMEOUT seconds" - } - else { - Write-Output "$WRITER_MESSAGE is found" - } - } -} - -############################################################################### -## Main -############################################################################### - -# Check to see if the VFD specified by the HDF5_DRIVER environment variable -# supports SWMR. -$testprog = Join-Path -Path $utils_testdir -ChildPath swmr_check_compat_vfd.exe -$rp = Start-Process -FilePath $testprog -PassThru -Wait -NoNewWindow -if ($rp.ExitCode -ne 0) { - Write-Output "" - Write-Output "The VFD specified by the HDF5_DRIVER environment variable" - Write-Output "does not support SWMR." - Write-Output "" - Write-Output "SWMR acceptance tests skipped" - Write-Output "" - exit 0 -} - -Set-Location -Path vds_swmr_test -$testdir = Join-Path -Path $testdir -ChildPath vds_swmr_test - -Write-Output "" -Write-Output "###############################################################################" -Write-Output "## Basic VDS SWMR test - writing to a tiled plane" -Write-Output "###############################################################################" - -# Launch the file generator -Write-Output "launch the swmr_generator" -$rp = Start-Process -FilePath $testdir/vds_swmr_gen.exe -NoNewWindow -PassThru -Wait -if ($rp.ExitCode -ne 0) { - Write-Warning "generator had error" - $nerrors += 1 -} - -# Check for error and exit if one occurred -#Write-Output "nerrors=$nerrors" -if ($nerrors -ne 0) { - Write-Warning "VDS SWMR tests failed with $nerrors errors." - exit 1 -} - -# Launch the writers -Write-Output "launch the $Nwriters SWMR VDS writers (1 per source)" -$pid_writers = @() -$n = 0 -while ($n -lt $Nwriters) { - $rp = Start-Process -FilePath $testdir/vds_swmr_writer.exe -NoNewWindow -PassThru -ArgumentList "$n" 2>&1 | tee swmr_writer.out.$n - $pid_writers += $rp.id - $n += 1 -} -#Write-Output "pid_writers=$pid_writers" - -# Sleep to ensure that the writers have started -Start-Sleep -Seconds 3 - -# Launch the readers -Write-Output "launch $Nreaders SWMR readers" -$pid_readers = @() -$n = 0 -while ($n -lt $Nreaders) { - $rp = Start-Process -FilePath $testdir/vds_swmr_reader.exe -NoNewWindow -PassThru 2>&1 | tee swmr_reader.out.$n - $pid_readers += $rp.id - $n += 1 -} -#Write-Output "pid_readers=$pid_readers" - -# Collect exit code of the writers -foreach ($xpid in $pid_writers) { - #Write-Output "checked writer $xpid" - $result = Wait-Process -Id $xpid - if ($result.ExitCode -ne 0) { - Write-Warning "writer had error" - $nerrors += 1 - } -} - -# Collect exit code of the readers -# (they usually finish after the writers) -foreach ($xpid in $pid_readers) { - #Write-Output "checked reader $xpid" - $result = Wait-Process -Id $xpid - if ($result.ExitCode -ne 0) { - Write-Warning "reader had error" - $nerrors += 1 - } -} - -# Check for error and exit if one occurred -#Write-Output "nerrors=$nerrors" -if ($nerrors -ne 0) { - Write-Warning "VDS SWMR tests failed with $nerrors errors." - exit 1 -} - -############################################################################### -## Report and exit -############################################################################### -cd .. -#Write-Output "nerrors=$nerrors" -if ($nerrors -eq 0) { - Write-Output "VDS SWMR tests passed." -# if test -z "$HDF5_NOCLEANUP"; then -# # delete the test directory -# Remove-Item vds_swmr_test -Recurse -# fi - exit 0 -} -else { - Write-Warning "VDS SWMR tests failed with $nerrors errors." - exit 1 -} - diff --git a/test/testvdsswmr.sh.in b/test/testvdsswmr.sh.in deleted file mode 100644 index 399fdef..0000000 --- a/test/testvdsswmr.sh.in +++ /dev/null @@ -1,233 +0,0 @@ -#! /bin/bash -# -# Copyright by The HDF Group. -# Copyright by the Board of Trustees of the University of Illinois. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. -# -# Tests for the swmr feature using virtual datasets. -# -# Created: -# Dana Robinson, November 2015 - -srcdir=@srcdir@ -utils_testdir=@abs_top_builddir@/@H5_UTILS_TEST_BUILDDIR@ -testdir=@abs_top_builddir@/@H5_TEST_BUILDDIR@ - -############################################################################### -## test parameters -############################################################################### - -Nwriters=6 # number of writers (1 per source dataset) -Nreaders=5 # number of readers to launch -nerrors=0 - -############################################################################### -## definitions for message file to coordinate test runs -############################################################################### -WRITER_MESSAGE=SWMR_WRITER_MESSAGE # The message file created by writer that the open is complete - # This should be the same as the define in "test/swmr_common.h" -MESSAGE_TIMEOUT=300 # Message timeout length in secs - # This should be the same as the define in "test/h5test.h" - -############################################################################### -## short hands and function definitions -############################################################################### -DPRINT=: # Set to "echo Debug:" for debugging printing, - # else ":" for noop. -IFDEBUG=: # Set to null to turn on debugging, else ":" for noop. - -# Print a line-line message left justified in a field of 70 characters -# beginning with the word "Testing". -# -TESTING() { - SPACES=" " - echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' -} - -# To wait for the writer message file or till the maximum # of seconds is reached -# $1 is the message file to wait for -# This performs similar function as the routine h5_wait_message() in test/h5test.c -WAIT_MESSAGE() { - message=$1 # Get the name of the message file to wait for - t0=`date +%s` # Get current time in seconds - difft=0 # Initialize the time difference - mexist=0 # Indicate whether the message file is found - while [ $difft -lt $MESSAGE_TIMEOUT ] ; # Loop till message times out - do - t1=`date +%s` # Get current time in seconds - difft=`expr $t1 - $t0` # Calculate the time difference - if [ -e $message ]; then # If message file is found: - mexist=1 # indicate the message file is found - rm $message # remove the message file - break # get out of the while loop - fi - done; - if test $mexist -eq 0; then - # Issue warning that the writer message file is not found, continue with launching the reader(s) - echo warning: $WRITER_MESSAGE is not found after waiting $MESSAGE_TIMEOUT seconds - else - echo $WRITER_MESSAGE is found - fi -} - -############################################################################### -## Main -############################################################################### - -# Check to see if the VFD specified by the HDF5_DRIVER environment variable -# supports SWMR. -$utils_testdir/swmr_check_compat_vfd -rc=$? -if [ $rc -ne 0 ] ; then - echo - echo "The VFD specified by the HDF5_DRIVER environment variable" - echo "does not support SWMR." - echo - echo "SWMR acceptance tests skipped" - echo - exit 0 -fi - -# Parse options (none accepted at this time) -while [ $# -gt 0 ]; do - case "$1" in - *) # unknown option - echo "$0: Unknown option ($1)" - exit 1 - ;; - esac -done - -# HDF5 has several tests that create and delete signal files to communicate -# between processes, and it seems that even though the names of the files are -# different, occasionally the wrong file is deleted, interrupting the flow of -# the test. Running each of these tests in its own directory should eliminate -# the problem. -mkdir vds_swmr_test -for FILE in vds_swmr*; do - case "$FILE" in - *.o) continue ;; ## don't copy the .o files - esac - if test -f "$FILE" ; then - cp $FILE vds_swmr_test - fi -done - -# With the --disable-shared option, swmr program files are built in the test -# directory, otherwise they are in test/.libs with a corresponding wrapper -# script in the test directory. The programs or wrapper scripts in test should -# always be copied, swmr files in .libs should be copied only if they exists. -if [ -f .libs/vds_swmr_writer ]; then - mkdir vds_swmr_test/.libs - for FILE in .libs/vds_swmr*; do - case "$FILE" in - *.o) continue ;; ## don't copy the .o files - esac - if test -f "$FILE" ; then - cp $FILE vds_swmr_test/.libs - fi - done -fi - -cd vds_swmr_test - - -echo -echo "###############################################################################" -echo "## Basic VDS SWMR test - writing to a tiled plane" -echo "###############################################################################" - -# Launch the file generator -echo launch the generator -$testdir/vds_swmr_gen -if test $? -ne 0; then - echo generator had error - nerrors=`expr $nerrors + 1` -fi - -# Check for error and exit if one occurred -$DPRINT nerrors=$nerrors -if test $nerrors -ne 0 ; then - echo "VDS SWMR tests failed with $nerrors errors." - exit 1 -fi - -# Launch the writers -echo "launch the $Nwriters SWMR VDS writers (1 per source)" -pid_writers="" -n=0 -while [ $n -lt $Nwriters ]; do - $testdir/vds_swmr_writer $n & - pid_writers="$pid_writers $!" - n=`expr $n + 1` -done -$DPRINT pid_writers=$pid_writers -$IFDEBUG ps - -# Sleep to ensure that the writers have started -sleep 3 - -# Launch the readers -echo launch $Nreaders SWMR readers -pid_readers="" -n=0 -while [ $n -lt $Nreaders ]; do - $testdir/vds_swmr_reader & - pid_readers="$pid_readers $!" - n=`expr $n + 1` -done -$DPRINT pid_readers=$pid_readers -$IFDEBUG ps - -# Collect exit code of the writers -for xpid in $pid_writers; do - $DPRINT checked writer $xpid - wait $xpid - if test $? -ne 0; then - echo writer had error - nerrors=`expr $nerrors + 1` - fi -done - -# Collect exit code of the readers -# (they usually finish after the writers) -for xpid in $pid_readers; do - $DPRINT checked reader $xpid - wait $xpid - if test $? -ne 0; then - echo reader had error - nerrors=`expr $nerrors + 1` - fi -done - -# Check for error and exit if one occurred -$DPRINT nerrors=$nerrors -if test $nerrors -ne 0 ; then - echo "VDS SWMR tests failed with $nerrors errors." - exit 1 -fi - -############################################################################### -## Report and exit -############################################################################### -cd .. -$DPRINT nerrors=$nerrors -if test $nerrors -eq 0 ; then - echo "VDS SWMR tests passed." - if test -z "$HDF5_NOCLEANUP"; then - # delete the test directory - rm -rf vds_swmr_test - fi - exit 0 -else - echo "VDS SWMR tests failed with $nerrors errors." - exit 1 -fi - -- cgit v0.12 From 1fb3743f04df101c7245ce045b67d371d462fd70 Mon Sep 17 00:00:00 2001 From: David Young Date: Thu, 10 Mar 2022 13:54:20 -0600 Subject: Fix JNI bug, fix warnings noticed in MSVC CI, be modern & portable (#1480) --- java/src/jni/h5util.c | 8 ++++---- src/H5VMprivate.h | 4 ++-- src/H5system.c | 4 ++-- test/external.c | 7 +++---- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/java/src/jni/h5util.c b/java/src/jni/h5util.c index 5a01dc9..2de5d87 100644 --- a/java/src/jni/h5util.c +++ b/java/src/jni/h5util.c @@ -813,7 +813,7 @@ h5str_sprintf(JNIEnv *env, h5str_t *out_str, hid_t container, hid_t tid, void *i if (NULL == (this_str = (char *)HDmalloc(this_len))) H5_OUT_OF_MEMORY_ERROR(ENVONLY, "h5str_sprintf: failed to allocate string buffer"); - if (HDsnprintf(this_str, "%g", this_len, tmp_double) < 0) + if (HDsnprintf(this_str, this_len, "%g", tmp_double) < 0) H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); break; @@ -1160,11 +1160,11 @@ h5str_sprintf(JNIEnv *env, h5str_t *out_str, hid_t container, hid_t tid, void *i H5Otoken_to_str(tid, &oi.token, &token_str); - size_t this_len = 64 + strlen(token_str) + 1; - if (NULL == (this_str = (char *)HDmalloc(this_len))) + size_t that_len = 64 + strlen(token_str) + 1; + if (NULL == (this_str = HDmalloc(that_len))) H5_OUT_OF_MEMORY_ERROR( ENVONLY, "h5str_sprintf: failed to allocate string buffer"); - if (HDsnprintf(this_str, this_len, "%lu:%s", oi.fileno, token_str) < + if (HDsnprintf(this_str, that_len, "%lu:%s", oi.fileno, token_str) < 0) H5_JNI_FATAL_ERROR(ENVONLY, "h5str_sprintf: HDsnprintf failure"); diff --git a/src/H5VMprivate.h b/src/H5VMprivate.h index 0d3bd0f..e773bae 100644 --- a/src/H5VMprivate.h +++ b/src/H5VMprivate.h @@ -393,8 +393,8 @@ static const unsigned char LogTable256[] = { static inline unsigned H5_ATTR_UNUSED H5VM_log2_gen(uint64_t n) { - unsigned r; /* r will be log2(n) */ - register unsigned int t, tt, ttt; /* temporaries */ + unsigned r; /* r will be log2(n) */ + unsigned int t, tt, ttt; /* temporaries */ if ((ttt = (unsigned)(n >> 32))) if ((tt = (unsigned)(n >> 48))) diff --git a/src/H5system.c b/src/H5system.c index ee9077e..a369e3d 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -862,7 +862,7 @@ H5_nanosleep(uint64_t nanosec) #else - const uint64_t nanosec_per_sec = 1000 * 1000 * 1000; + const uint64_t nanosec_per_sec = 1000 * 1000L * 1000; struct timespec sleeptime; /* Struct to hold time to sleep */ /* Set up time to sleep @@ -1033,7 +1033,7 @@ H5_get_option(int argc, const char *const *argv, const char *opts, const struct HDfree(arg); } else { - register char *cp; /* pointer into current token */ + char *cp; /* pointer into current token */ /* short command line option */ optchar = argv[H5_optind][sp]; diff --git a/test/external.c b/test/external.c index 11185b3..bb86cde 100644 --- a/test/external.c +++ b/test/external.c @@ -166,8 +166,7 @@ test_non_extendible(hid_t file) if (file_size != (max_size[0] * sizeof(int))) { H5_FAILED(); HDputs(" Wrong file size."); - HDprintf(" got: %lu\n ans: %lu\n", (unsigned long)file_size, - (unsigned long)max_size[0] * sizeof(int)); + HDprintf(" got: %" PRIuHSIZE "\n ans: %" PRIuHSIZE "\n", file_size, max_size[0] * sizeof(int)); goto error; } @@ -986,11 +985,11 @@ test_path_absolute(hid_t fapl) if (NULL == HDgetcwd(cwdpath, sizeof(cwdpath))) TEST_ERROR for (i = 0; i < N_EXT_FILES; i++) { - HDsnprintf(filename, sizeof(filename), "%s%sextern_%dr.raw", cwdpath, H5_DIR_SEPS, (int)i + 1); + HDsnprintf(filename, sizeof(filename), "%s%sextern_%zur.raw", cwdpath, H5_DIR_SEPS, i + 1); #if defined(H5_HAVE_WINDOW_PATH) /* For windows, test path-absolute case (\dir\file.raw) for the second file */ if (i == 1) - HDsnprintf(filename, sizeof(filename), "%s%sextern_%dr.raw", cwdpath + 2, H5_DIR_SEPS, i + 1); + HDsnprintf(filename, sizeof(filename), "%s%sextern_%zur.raw", cwdpath + 2, H5_DIR_SEPS, i + 1); #endif if (H5Pset_external(dcpl, filename, (off_t)(i * GARBAGE_PER_FILE), (hsize_t)sizeof(part)) < 0) FAIL_STACK_ERROR -- cgit v0.12 From 01209d277767cc66d11d4c1ef874ed493000f78c Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Thu, 10 Mar 2022 14:14:05 -0800 Subject: Removes remaining register keywords (#1481) --- hl/tools/gif2h5/gif2mem.c | 32 +++++++++--------- hl/tools/gif2h5/hdfgifwr.c | 14 ++++---- tools/lib/h5tools.c | 2 +- tools/lib/h5tools_str.c | 8 ++--- tools/src/h5perf/pio_engine.c | 2 +- tools/src/h5perf/pio_perf.c | 65 ++++++++++++++++++------------------- tools/src/h5perf/sio_engine.c | 2 +- tools/src/h5perf/sio_perf.c | 43 ++++++++++++------------ tools/test/h5dump/h5dumpgentest.c | 6 ++-- tools/test/perform/pio_standalone.c | 2 +- tools/test/perform/sio_standalone.c | 2 +- tools/test/perform/zip_perf.c | 6 ++-- 12 files changed, 91 insertions(+), 93 deletions(-) diff --git a/hl/tools/gif2h5/gif2mem.c b/hl/tools/gif2h5/gif2mem.c index 2c2225e..44e18e0 100644 --- a/hl/tools/gif2h5/gif2mem.c +++ b/hl/tools/gif2h5/gif2mem.c @@ -49,22 +49,22 @@ Gif2Mem(GIFBYTE *MemGif, GIFTOMEM *GifMemoryStruct) GIFCOMMENT ** gifComment; /* Comment Extension structure */ GIFGRAPHICCONTROL **gifGraphicControl; /* Graphic Control Extension strct */ - register GIFWORD i; /* Loop counter */ - GIFBYTE Identifier; /* Extension block identifier holder */ - GIFBYTE Label; /* Extension block label holder */ - GIFBYTE ImageCount; /* Count of the number of images in the file */ - GIFBYTE ImageArray; /* Keep the size of the array to store Images */ - GIFBYTE CommentCount; - GIFBYTE CommentArray; - GIFBYTE ApplicationCount; - GIFBYTE ApplicationArray; - GIFBYTE PlainTextCount; - GIFBYTE PlainTextArray; - GIFBYTE GCEflag; - GIFBYTE aTemp; - GIFBYTE j; - GIFBYTE w; /* Two more variables needed only while testing */ - GIFBYTE * b; /* Endian Ordering */ + GIFWORD i; /* Loop counter */ + GIFBYTE Identifier; /* Extension block identifier holder */ + GIFBYTE Label; /* Extension block label holder */ + GIFBYTE ImageCount; /* Count of the number of images in the file */ + GIFBYTE ImageArray; /* Keep the size of the array to store Images */ + GIFBYTE CommentCount; + GIFBYTE CommentArray; + GIFBYTE ApplicationCount; + GIFBYTE ApplicationArray; + GIFBYTE PlainTextCount; + GIFBYTE PlainTextArray; + GIFBYTE GCEflag; + GIFBYTE aTemp; + GIFBYTE j; + GIFBYTE w; /* Two more variables needed only while testing */ + GIFBYTE *b; /* Endian Ordering */ /* Allocate memory for the GIF structures */ /* Plug the structs into GifMemoryStruct at the end */ diff --git a/hl/tools/gif2h5/hdfgifwr.c b/hl/tools/gif2h5/hdfgifwr.c index 21b14d1..7126da9 100644 --- a/hl/tools/gif2h5/hdfgifwr.c +++ b/hl/tools/gif2h5/hdfgifwr.c @@ -188,13 +188,13 @@ hdfWriteGIF(FILE *fp, byte *pic, int ptype, int w, int h, const byte *rmap, cons static void compress(int init_bits, FILE *outfile, byte *data, int len) { - register long fcode; - register int i = 0; - register int c; - register int ent; - register int disp; - register int hsize_reg; - register int hshift; + long fcode; + int i = 0; + int c; + int ent; + int disp; + int hsize_reg; + int hshift; /* * Set up the globals: g_init_bits - initial number of bits g_outfile - diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c index da55136..c7dba25 100644 --- a/tools/lib/h5tools.c +++ b/tools/lib/h5tools.c @@ -1052,7 +1052,7 @@ done: H5_ATTR_PURE static size_t h5tools_count_ncols(const char *s) { - register size_t i; + size_t i; for (i = 0; *s; s++) if (*s >= ' ') diff --git a/tools/lib/h5tools_str.c b/tools/lib/h5tools_str.c index 1fb2c68..7ad6a65 100644 --- a/tools/lib/h5tools_str.c +++ b/tools/lib/h5tools_str.c @@ -1401,10 +1401,10 @@ h5tools_str_sprint_reference(h5tools_str_t *str, H5R_ref_t *ref_vp) static char * h5tools_escape(char *s /*in,out*/, size_t size) { - register size_t i; - const char * escape; - char octal[8]; - size_t n = HDstrlen(s); + size_t i; + const char *escape; + char octal[8]; + size_t n = HDstrlen(s); for (i = 0; i < n; i++) { switch (s[i]) { diff --git a/tools/src/h5perf/pio_engine.c b/tools/src/h5perf/pio_engine.c index e316245..1baaca2 100644 --- a/tools/src/h5perf/pio_engine.c +++ b/tools/src/h5perf/pio_engine.c @@ -420,7 +420,7 @@ pio_create_filename(iotype iot, const char *base_name, char *fullname, size_t si /* If the prefix specifies the HDF5_PARAPREFIX directory, then * default to using the "/tmp/$USER" or "/tmp/$LOGIN" * directory instead. */ - register char *user, *login, *subdir; + char *user, *login, *subdir; user = HDgetenv("USER"); login = HDgetenv("LOGIN"); diff --git a/tools/src/h5perf/pio_perf.c b/tools/src/h5perf/pio_perf.c index c233684..003c4f2 100644 --- a/tools/src/h5perf/pio_perf.c +++ b/tools/src/h5perf/pio_perf.c @@ -200,7 +200,7 @@ static void output_results(const struct options *options, const char *name, min off_t data_size); static void output_times(const struct options *options, const char *name, minmax *table, int table_size); static void output_report(const char *fmt, ...); -static void print_indent(register int indent); +static void print_indent(int indent); static void usage(const char *prog); static void report_parameters(struct options *opts); static off_t squareo(off_t); @@ -336,7 +336,7 @@ run_test_loop(struct options *opts) /* start with max_num_procs and decrement it by half for each loop. */ /* if performance needs restart, fewer processes may be needed. */ for (num_procs = opts->max_num_procs; num_procs >= opts->min_num_procs; num_procs >>= 1) { - register size_t buf_size; + size_t buf_size; parms.num_procs = num_procs; @@ -411,34 +411,34 @@ run_test_loop(struct options *opts) static int run_test(iotype iot, parameters parms, struct options *opts) { - results res; - register int i, ret_value = SUCCESS; - int comm_size; - off_t raw_size; - minmax * write_mpi_mm_table = NULL; - minmax * write_mm_table = NULL; - minmax * write_gross_mm_table = NULL; - minmax * write_raw_mm_table = NULL; - minmax * read_mpi_mm_table = NULL; - minmax * read_mm_table = NULL; - minmax * read_gross_mm_table = NULL; - minmax * read_raw_mm_table = NULL; - minmax * read_open_mm_table = NULL; - minmax * read_close_mm_table = NULL; - minmax * write_open_mm_table = NULL; - minmax * write_close_mm_table = NULL; - minmax write_mpi_mm = {0.0, 0.0, 0.0, 0}; - minmax write_mm = {0.0, 0.0, 0.0, 0}; - minmax write_gross_mm = {0.0, 0.0, 0.0, 0}; - minmax write_raw_mm = {0.0, 0.0, 0.0, 0}; - minmax read_mpi_mm = {0.0, 0.0, 0.0, 0}; - minmax read_mm = {0.0, 0.0, 0.0, 0}; - minmax read_gross_mm = {0.0, 0.0, 0.0, 0}; - minmax read_raw_mm = {0.0, 0.0, 0.0, 0}; - minmax read_open_mm = {0.0, 0.0, 0.0, 0}; - minmax read_close_mm = {0.0, 0.0, 0.0, 0}; - minmax write_open_mm = {0.0, 0.0, 0.0, 0}; - minmax write_close_mm = {0.0, 0.0, 0.0, 0}; + results res; + int i, ret_value = SUCCESS; + int comm_size; + off_t raw_size; + minmax *write_mpi_mm_table = NULL; + minmax *write_mm_table = NULL; + minmax *write_gross_mm_table = NULL; + minmax *write_raw_mm_table = NULL; + minmax *read_mpi_mm_table = NULL; + minmax *read_mm_table = NULL; + minmax *read_gross_mm_table = NULL; + minmax *read_raw_mm_table = NULL; + minmax *read_open_mm_table = NULL; + minmax *read_close_mm_table = NULL; + minmax *write_open_mm_table = NULL; + minmax *write_close_mm_table = NULL; + minmax write_mpi_mm = {0.0, 0.0, 0.0, 0}; + minmax write_mm = {0.0, 0.0, 0.0, 0}; + minmax write_gross_mm = {0.0, 0.0, 0.0, 0}; + minmax write_raw_mm = {0.0, 0.0, 0.0, 0}; + minmax read_mpi_mm = {0.0, 0.0, 0.0, 0}; + minmax read_mm = {0.0, 0.0, 0.0, 0}; + minmax read_gross_mm = {0.0, 0.0, 0.0, 0}; + minmax read_raw_mm = {0.0, 0.0, 0.0, 0}; + minmax read_open_mm = {0.0, 0.0, 0.0, 0}; + minmax read_close_mm = {0.0, 0.0, 0.0, 0}; + minmax write_open_mm = {0.0, 0.0, 0.0, 0}; + minmax write_close_mm = {0.0, 0.0, 0.0, 0}; raw_size = parms.num_files * (off_t)parms.num_dsets * (off_t)parms.num_bytes; parms.io_type = iot; @@ -1098,10 +1098,9 @@ output_report(const char *fmt, ...) * things. * Return: Nothing * Programmer: Bill Wendling, 29. October 2001 - * Modifications: */ static void -print_indent(register int indent) +print_indent(int indent) { int myrank; @@ -1278,7 +1277,7 @@ report_parameters(struct options *opts) static struct options * parse_command_line(int argc, const char *const *argv) { - register int opt; + int opt; struct options *cl_opts; cl_opts = (struct options *)malloc(sizeof(struct options)); diff --git a/tools/src/h5perf/sio_engine.c b/tools/src/h5perf/sio_engine.c index 95ec3ce..bdd2c08 100644 --- a/tools/src/h5perf/sio_engine.c +++ b/tools/src/h5perf/sio_engine.c @@ -328,7 +328,7 @@ sio_create_filename(iotype iot, const char *base_name, char *fullname, size_t si /* If the prefix specifies the HDF5_PREFIX directory, then * default to using the "/tmp/$USER" or "/tmp/$LOGIN" * directory instead. */ - register char *user, *login, *subdir; + char *user, *login, *subdir; user = HDgetenv("USER"); login = HDgetenv("LOGIN"); diff --git a/tools/src/h5perf/sio_perf.c b/tools/src/h5perf/sio_perf.c index 87b9d65..dc2e15e 100644 --- a/tools/src/h5perf/sio_perf.c +++ b/tools/src/h5perf/sio_perf.c @@ -173,7 +173,7 @@ static void accumulate_minmax_stuff(const minmax *mm, int count, minm static void output_results(const struct options *options, const char *name, minmax *table, int table_size, off_t data_size); static void output_report(const char *fmt, ...); -static void print_indent(register int indent); +static void print_indent(int indent); static void usage(const char *prog); static void report_parameters(struct options *opts); @@ -299,25 +299,25 @@ run_test_loop(struct options *opts) static int run_test(iotype iot, parameters parms, struct options *opts) { - results res; - register int i, ret_value = SUCCESS; - off_t raw_size; - minmax * write_sys_mm_table = NULL; - minmax * write_mm_table = NULL; - minmax * write_gross_mm_table = NULL; - minmax * write_raw_mm_table = NULL; - minmax * read_sys_mm_table = NULL; - minmax * read_mm_table = NULL; - minmax * read_gross_mm_table = NULL; - minmax * read_raw_mm_table = NULL; - minmax write_sys_mm = {0.0, 0.0, 0.0, 0}; - minmax write_mm = {0.0, 0.0, 0.0, 0}; - minmax write_gross_mm = {0.0, 0.0, 0.0, 0}; - minmax write_raw_mm = {0.0, 0.0, 0.0, 0}; - minmax read_sys_mm = {0.0, 0.0, 0.0, 0}; - minmax read_mm = {0.0, 0.0, 0.0, 0}; - minmax read_gross_mm = {0.0, 0.0, 0.0, 0}; - minmax read_raw_mm = {0.0, 0.0, 0.0, 0}; + results res; + int i, ret_value = SUCCESS; + off_t raw_size; + minmax *write_sys_mm_table = NULL; + minmax *write_mm_table = NULL; + minmax *write_gross_mm_table = NULL; + minmax *write_raw_mm_table = NULL; + minmax *read_sys_mm_table = NULL; + minmax *read_mm_table = NULL; + minmax *read_gross_mm_table = NULL; + minmax *read_raw_mm_table = NULL; + minmax write_sys_mm = {0.0, 0.0, 0.0, 0}; + minmax write_mm = {0.0, 0.0, 0.0, 0}; + minmax write_gross_mm = {0.0, 0.0, 0.0, 0}; + minmax write_raw_mm = {0.0, 0.0, 0.0, 0}; + minmax read_sys_mm = {0.0, 0.0, 0.0, 0}; + minmax read_mm = {0.0, 0.0, 0.0, 0}; + minmax read_gross_mm = {0.0, 0.0, 0.0, 0}; + minmax read_raw_mm = {0.0, 0.0, 0.0, 0}; raw_size = (off_t)parms.num_bytes; parms.io_type = iot; @@ -657,10 +657,9 @@ output_report(const char *fmt, ...) * things. * Return: Nothing * Programmer: Bill Wendling, 29. October 2001 - * Modifications: */ static void -print_indent(register int indent) +print_indent(int indent) { indent *= TAB_SPACE; diff --git a/tools/test/h5dump/h5dumpgentest.c b/tools/test/h5dump/h5dumpgentest.c index 5258211..339b563 100644 --- a/tools/test/h5dump/h5dumpgentest.c +++ b/tools/test/h5dump/h5dumpgentest.c @@ -3914,9 +3914,9 @@ gent_multi(void) static void gent_large_objname(void) { - hid_t fid, group, group2; - char grp_name[128]; - register int i; + hid_t fid, group, group2; + char grp_name[128]; + int i; fid = H5Fcreate(FILE37, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); diff --git a/tools/test/perform/pio_standalone.c b/tools/test/perform/pio_standalone.c index 032bfba..a074d2c 100644 --- a/tools/test/perform/pio_standalone.c +++ b/tools/test/perform/pio_standalone.c @@ -92,7 +92,7 @@ get_option(int argc, const char **argv, const char *opts, const struct h5_long_o sp = 1; } else { - register char *cp; /* pointer into current token */ + char *cp; /* pointer into current token */ /* short command line option */ opt_opt = argv[H5_optind][sp]; diff --git a/tools/test/perform/sio_standalone.c b/tools/test/perform/sio_standalone.c index 7c22b53..353dad3 100644 --- a/tools/test/perform/sio_standalone.c +++ b/tools/test/perform/sio_standalone.c @@ -92,7 +92,7 @@ get_option(int argc, const char **argv, const char *opts, const struct h5_long_o sp = 1; } else { - register char *cp; /* pointer into current token */ + char *cp; /* pointer into current token */ /* short command line option */ opt_opt = argv[H5_optind][sp]; diff --git a/tools/test/perform/zip_perf.c b/tools/test/perform/zip_perf.c index 74b1685..87a7122 100644 --- a/tools/test/perform/zip_perf.c +++ b/tools/test/perform/zip_perf.c @@ -336,8 +336,8 @@ parse_size_directive(const char *size) static void fill_with_random_data(Bytef *src, uLongf src_len) { - register unsigned u; - h5_stat_t stat_buf; + unsigned u; + h5_stat_t stat_buf; if (HDstat("/dev/urandom", &stat_buf) == 0) { uLongf len = src_len; @@ -386,7 +386,7 @@ do_write_test(unsigned long file_size, unsigned long min_buf_size, unsigned long Bytef * src; for (src_len = min_buf_size; src_len <= max_buf_size; src_len <<= 1) { - register unsigned long i, iters; + unsigned long i, iters; iters = file_size / src_len; src = (Bytef *)HDcalloc(1, sizeof(Bytef) * src_len); -- cgit v0.12 From 1f9228316c3177fa632979fa89e8753d35770efb Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Fri, 11 Mar 2022 11:10:20 -0800 Subject: Fixes for format string warnings raised by -Wformat=2/-Wformat-security (#1489) --- config/clang-warnings/developer-general | 1 - config/clang-warnings/general | 16 +++++----------- config/gnu-warnings/cxx-general | 1 - config/gnu-warnings/developer-general | 3 --- config/gnu-warnings/general | 14 ++++---------- tools/lib/h5tools_str.c | 2 +- tools/src/h5ls/h5ls.c | 6 +++--- 7 files changed, 13 insertions(+), 30 deletions(-) diff --git a/config/clang-warnings/developer-general b/config/clang-warnings/developer-general index 74d8404..b80552f 100644 --- a/config/clang-warnings/developer-general +++ b/config/clang-warnings/developer-general @@ -1,4 +1,3 @@ --Wformat-nonliteral -Wmissing-noreturn -Wsometimes-uninitialized -Wswitch-enum diff --git a/config/clang-warnings/general b/config/clang-warnings/general index f0c9b93..433dff9 100644 --- a/config/clang-warnings/general +++ b/config/clang-warnings/general @@ -8,19 +8,13 @@ -Wformat=2 -Wframe-larger-than=16384 -Wimplicit-fallthrough -# -# NOTE: Due to the divergence in the C and C++, we're dropping support for -# compiling the C library with a C++ compiler and dropping the -Wc++-compat -# warning. -# --Wno-c++-compat -# -# NOTE: Disable the -Wformat-nonliteral from -Wformat=2 here and re-add -# it to the developer flags. -# --Wno-format-nonliteral -Wnull-dereference -Wunused-const-variable -Wwrite-strings -Wpedantic -Wvolatile-register-var +# NOTE: Due to the divergence in the C and C++, we're dropping support for +# compiling the C library with a C++ compiler and dropping the -Wc++-compat +# warning. +# +-Wno-c++-compat diff --git a/config/gnu-warnings/cxx-general b/config/gnu-warnings/cxx-general index 9548cc0..1626524 100644 --- a/config/gnu-warnings/cxx-general +++ b/config/gnu-warnings/cxx-general @@ -17,7 +17,6 @@ -Winit-self -Winvalid-pch -Wmissing-include-dirs --Wno-format-nonliteral -Wnon-virtual-dtor -Wold-style-cast -Woverloaded-virtual diff --git a/config/gnu-warnings/developer-general b/config/gnu-warnings/developer-general index b34c4b7..460b874 100644 --- a/config/gnu-warnings/developer-general +++ b/config/gnu-warnings/developer-general @@ -1,9 +1,6 @@ # (suggestions from gcc, not code problems) -# NOTE: -Wformat-nonliteral added back in here (from being disabled in -# H5_CFLAGS) -Waggregate-return -Wdisabled-optimization --Wformat-nonliteral -Winline -Wmissing-format-attribute -Wmissing-noreturn diff --git a/config/gnu-warnings/general b/config/gnu-warnings/general index a7a20b7..df4c613 100644 --- a/config/gnu-warnings/general +++ b/config/gnu-warnings/general @@ -15,18 +15,12 @@ -Winit-self -Winvalid-pch -Wmissing-include-dirs -# +-Wshadow +-Wundef +-Wwrite-strings +-pedantic # NOTE: Due to the divergence in the C and C++, we're dropping support for # compiling the C library with a C++ compiler and dropping the -Wc++-compat # warning. # -Wno-c++-compat -# -# NOTE: Disable the -Wformat-nonliteral from -Wformat=2 here and re-add -# it to the developer flags. -# --Wno-format-nonliteral --Wshadow --Wundef --Wwrite-strings --pedantic diff --git a/tools/lib/h5tools_str.c b/tools/lib/h5tools_str.c index 7ad6a65..5976044 100644 --- a/tools/lib/h5tools_str.c +++ b/tools/lib/h5tools_str.c @@ -1051,7 +1051,7 @@ h5tools_str_sprint(h5tools_str_t *str, const h5tool_format_t *info, hid_t contai H5TOOLS_DEBUG("H5T_ENUM"); if (H5Tenum_nameof(type, vp, enum_name, sizeof enum_name) >= 0) - h5tools_str_append(str, h5tools_escape(enum_name, sizeof(enum_name))); + h5tools_str_append(str, "%s", h5tools_escape(enum_name, sizeof(enum_name))); else { size_t i; if (1 == nsize) diff --git a/tools/src/h5ls/h5ls.c b/tools/src/h5ls/h5ls.c index a27a8cc..59cd974 100644 --- a/tools/src/h5ls/h5ls.c +++ b/tools/src/h5ls/h5ls.c @@ -2299,7 +2299,7 @@ list_lnk(const char *name, const H5L_info2_t *linfo, void *_iter) iter->symlink_list->dangle_link = TRUE; h5tools_str_append(&buffer, "Soft Link {"); - h5tools_str_append(&buffer, buf); + h5tools_str_append(&buffer, "%s", buf); h5tools_str_append(&buffer, "}"); h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); @@ -2367,11 +2367,11 @@ list_lnk(const char *name, const H5L_info2_t *linfo, void *_iter) goto done; h5tools_str_append(&buffer, "External Link {"); - h5tools_str_append(&buffer, filename); + h5tools_str_append(&buffer, "%s", filename); h5tools_str_append(&buffer, "/"); if (*path != '/') h5tools_str_append(&buffer, "/"); - h5tools_str_append(&buffer, path); + h5tools_str_append(&buffer, "%s", path); h5tools_str_append(&buffer, "}"); h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); -- cgit v0.12 From 872b87d5d69fdd3b1b75a7047b713b1adfd16248 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Fri, 11 Mar 2022 13:17:26 -0600 Subject: Update release note (#1485) * Flip inits to correct ifdef section * rework ifdef to be simpler * format issue * Reformat ifdef inits * remove static attribute * format compliance * Update names * Revert because logic relies on float not being int * Changes noticed from creating merge of #412 * Double underscore change * Correct compiler version variable used * Remove header guard underscores * Whitespace cleanup * Split format source and commit changes on repo push * remove pre-split setting * Change windows TS to use older VS. * correct window os name * HDFFV-11212 JNI export util and Javadoc * Suggested review changes * Another change found * Committing clang-format changes * HDFFV-11113 long double in tools * HDFFV-11113 add note * Disable long double tests for now * HDFFV-11228 remove arbitrary CMake warning groups. * Make each flag a string * Some Javadoc warning fixes * Updated javadoc fixes * # WARNING: head commit changed in the meantime HDFFV-11229 Fix long double usage in tools and java Changed h5dump and h5ls to just print 128-bit for long double type. Added test and file for dataset and attributes with all three float types. * Committing clang-format changes * HDFFV-11229 Add release note * HDFFV-11229 windows testfile needed * fix typo * Remove non supported message text * HDFFV-11229 - change ldouble test to check both native and general * HDFFV-11229 add second file compare * HDFFV-11229 fix reference file * HDFFV-11229 autotools check two refs * HDFFV-11229 revert back to removal of NATIVE_LDOUBLE in tools output * Committing clang-format changes * Update release note * Update attribute type of ref file * Change source of ninja for macs * try port instead of brew * Recommended is to use brew. * Undo non long double changes * remove unneeded file * Fix perl and doxygen CMake code * Add "option" command for clang options * Rework CMake add_custom to use the BYPRODUCTS argument * Add stamp files to BYPRODUCTS * Only one copy of file exists * Fix custom cmmand depends targets * Fix fortran custom command DEPENDS * Add LD_LIBRARY_PATH to tests * Add custom target for DEPENDS * Add h5detect conditionaly to generated target DEPENDS * Correct DEPENDS targets * Parallel builds need the mpi compiler for pkgconfig scripts. * install only if MPI build * Fortran target depends * Remove incorrect source attribute * doxygen adjustments * doxygen build updates * Correct version * Correct function version - function has been merged to 1.12 * Correct version string for map functions * Cleanup warnings for tools debug builds * TRILAB-227 - fix indexing for h5diff selections * Correct location of pos to index function call * TRILAB-227 Enable test * Quote subset args * Use MATCHES because of AppleClang * if blocks needed for build modes * Update list of DT platforms * VS2019 correctly displays float values * revert VS2019 change * Issue #669 remove version from pkgcfg filename * remove version from h5cc script * Java reference functions updated enabled fortran in cmake gcc action yaml file java reference test changed to correctly test refs jni reference functions that create ids changed to use jni id logging * Correct BYPRODUCTS argument * Correct more genereated files BYPRODUCTS * BYPRODUCTS must have unique locations * Fix typo * Fix fortran configure checks * Rework H5_PAC_C_MAX_REAL_PRECISION setting logic * Add note about fortran configure change * Adds a quick for for some egregious chunk_info badness (#722) * Fixes issue with ccmake that prevents building Fortran (#723) ccmake runs iteratively, and the check_fortran_source_runs macros were clobbering a single output file that did not get updated on further configure iterations * Fix conflicts with merge * Move MAX_PRECISION back to HDF5UseFortran.cmake * Use STREQUAL to test macro argument * Move C language test to ConfigureChecks from HDF5UseFortran * MAX_PRECISION defines must be defined * Organize flags and align autotools and cmake * Fix comment in no-error-general warnings files. * Flag cleanup and fix typos * Add comment * Correct VAR used to find configure time file * Set the path correctly * Update missing release note info. * Update code owners * Correct JIRA note * add known problem. * Use only core library for testing dynamic plugins. * Committing clang-format changes * Update main.yml * Update HISTORY-1_13.txt Missing release note added for changed location of CMake configuration files. * Update HISTORY-1_13.txt Fix typo * Update main.yml revert change Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Dana Robinson <43805+derobins@users.noreply.github.com> --- release_docs/HISTORY-1_13.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/release_docs/HISTORY-1_13.txt b/release_docs/HISTORY-1_13.txt index 91320a1..bc9ae3f 100644 --- a/release_docs/HISTORY-1_13.txt +++ b/release_docs/HISTORY-1_13.txt @@ -1810,6 +1810,17 @@ New Features Configuration: ------------- + - Reworked corrected path searched by CMake find_package command + + The install path for cmake find_package files had been changed to use + "share/cmake" + for all platforms. However setting the HDF5_ROOT variable failed to locate + the configuration files. The build variable HDF5_INSTALL_CMAKE_DIR is now + set to the /cmake folder. The location of the configuration + files can still be specified by the "HDF5_DIR" variable. + + (ADB - 2022/02/02) + - CPack will now generate RPM/DEB packages. Enabled the RPM and DEB CPack generators on linux. In addition to -- cgit v0.12 From 377560fdd8a840a86be496f0d5e5f05094063605 Mon Sep 17 00:00:00 2001 From: Neil Fortner Date: Mon, 14 Mar 2022 19:22:57 -0500 Subject: Fix warnings in selection I/O code --- src/H5FDint.c | 95 +++++++++++++++++++++++++++++++++++++++---------------- src/H5FDmpio.c | 16 +++++----- src/H5FDprivate.h | 4 +-- src/H5private.h | 10 ++++++ 4 files changed, 87 insertions(+), 38 deletions(-) diff --git a/src/H5FDint.c b/src/H5FDint.c index f624aa8..fabcedf 100644 --- a/src/H5FDint.c +++ b/src/H5FDint.c @@ -34,6 +34,7 @@ #include "H5Eprivate.h" /* Error handling */ #include "H5Fprivate.h" /* File access */ #include "H5FDpkg.h" /* File Drivers */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ #include "H5PLprivate.h" /* Plugins */ @@ -116,6 +117,9 @@ static herr_t H5FD__write_selection_translate(H5FD_t *file, H5FD_mem_t type, hid /* Local Variables */ /*******************/ +/* Declare extern free list to manage the H5S_sel_iter_t struct */ +H5FL_EXTERN(H5S_sel_iter_t); + /*------------------------------------------------------------------------- * Function: H5FD_locate_signature * @@ -761,8 +765,10 @@ H5FD__read_selection_translate(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, uin size_t nelmts; hssize_t hss_nelmts; size_t seq_nelem; - H5S_sel_iter_t file_iter; - H5S_sel_iter_t mem_iter; + H5S_sel_iter_t *file_iter = NULL; + H5S_sel_iter_t *mem_iter = NULL; + hbool_t file_iter_init = FALSE; + hbool_t mem_iter_init = FALSE; H5FD_mem_t types[2] = {type, H5FD_MEM_NOLIST}; size_t vec_arr_nalloc = sizeof(addrs_static) / sizeof(addrs_static[0]); size_t vec_arr_nused = 0; @@ -821,11 +827,19 @@ H5FD__read_selection_translate(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, uin } } + /* Allocate sequence lists for memory and file spaces */ + if (NULL == (file_iter = H5FL_MALLOC(H5S_sel_iter_t))) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "couldn't allocate file selection iterator") + if (NULL == (mem_iter = H5FL_MALLOC(H5S_sel_iter_t))) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "couldn't allocate memory selection iterator") + /* Initialize sequence lists for memory and file spaces */ - if (H5S_select_iter_init(&file_iter, file_spaces[i], element_size, 0) < 0) + if (H5S_select_iter_init(file_iter, file_spaces[i], element_size, 0) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "can't initialize sequence list for file space") - if (H5S_select_iter_init(&mem_iter, mem_spaces[i], element_size, 0) < 0) + file_iter_init = TRUE; + if (H5S_select_iter_init(mem_iter, mem_spaces[i], element_size, 0) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "can't initialize sequence list for memory space") + mem_iter_init = TRUE; /* Get the number of elements in selection */ if ((hss_nelmts = (hssize_t)H5S_GET_SELECT_NPOINTS(file_spaces[i])) < 0) @@ -852,7 +866,7 @@ H5FD__read_selection_translate(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, uin while (file_seq_i < file_nseq || nelmts > 0) { /* Fill/refill file sequence list if necessary */ if (file_seq_i == H5FD_SEQ_LIST_LEN) { - if (H5S_SELECT_ITER_GET_SEQ_LIST(&file_iter, H5FD_SEQ_LIST_LEN, SIZE_MAX, &file_nseq, + if (H5S_SELECT_ITER_GET_SEQ_LIST(file_iter, H5FD_SEQ_LIST_LEN, SIZE_MAX, &file_nseq, &seq_nelem, file_off, file_len) < 0) HGOTO_ERROR(H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed") HDassert(file_nseq > 0); @@ -864,7 +878,7 @@ H5FD__read_selection_translate(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, uin /* Fill/refill memory sequence list if necessary */ if (mem_seq_i == H5FD_SEQ_LIST_LEN) { - if (H5S_SELECT_ITER_GET_SEQ_LIST(&mem_iter, H5FD_SEQ_LIST_LEN, SIZE_MAX, &mem_nseq, + if (H5S_SELECT_ITER_GET_SEQ_LIST(mem_iter, H5FD_SEQ_LIST_LEN, SIZE_MAX, &mem_nseq, &seq_nelem, mem_off, mem_len) < 0) HGOTO_ERROR(H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed") HDassert(mem_nseq > 0); @@ -953,14 +967,9 @@ H5FD__read_selection_translate(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, uin } } + /* Make sure both memory and file sequences terminated at the same time */ if (mem_seq_i < mem_nseq) HGOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, "file selection terminated before memory selection") - - /* Terminate iterators */ - if (H5S_SELECT_ITER_RELEASE(&file_iter) < 0) - HGOTO_ERROR(H5E_INTERNAL, H5E_CANTFREE, FAIL, "can't release file selection iterator") - if (H5S_SELECT_ITER_RELEASE(&mem_iter) < 0) - HGOTO_ERROR(H5E_INTERNAL, H5E_CANTFREE, FAIL, "can't release memory selection iterator") } /* Issue vector read call if appropriate */ @@ -972,7 +981,19 @@ H5FD__read_selection_translate(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, uin } done: - /* Cleanup */ + /* Terminate iterators */ + if (file_iter) { + if (file_iter_init && H5S_SELECT_ITER_RELEASE(file_iter) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_CANTFREE, FAIL, "can't release file selection iterator") + file_iter = H5FL_FREE(H5S_sel_iter_t, file_iter); + } + if (mem_iter) { + if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_CANTFREE, FAIL, "can't release memory selection iterator") + mem_iter = H5FL_FREE(H5S_sel_iter_t, mem_iter); + } + + /* Cleanup vector arrays */ if (use_vector) { if (addrs != addrs_static) addrs = H5MM_xfree(addrs); @@ -1379,8 +1400,10 @@ H5FD__write_selection_translate(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, ui size_t nelmts; hssize_t hss_nelmts; size_t seq_nelem; - H5S_sel_iter_t file_iter; - H5S_sel_iter_t mem_iter; + H5S_sel_iter_t *file_iter = NULL; + H5S_sel_iter_t *mem_iter = NULL; + hbool_t file_iter_init = FALSE; + hbool_t mem_iter_init = FALSE; H5FD_mem_t types[2] = {type, H5FD_MEM_NOLIST}; size_t vec_arr_nalloc = sizeof(addrs_static) / sizeof(addrs_static[0]); size_t vec_arr_nused = 0; @@ -1439,11 +1462,19 @@ H5FD__write_selection_translate(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, ui } } + /* Allocate sequence lists for memory and file spaces */ + if (NULL == (file_iter = H5FL_MALLOC(H5S_sel_iter_t))) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "couldn't allocate file selection iterator") + if (NULL == (mem_iter = H5FL_MALLOC(H5S_sel_iter_t))) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "couldn't allocate memory selection iterator") + /* Initialize sequence lists for memory and file spaces */ - if (H5S_select_iter_init(&file_iter, file_spaces[i], element_size, 0) < 0) + if (H5S_select_iter_init(file_iter, file_spaces[i], element_size, 0) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "can't initialize sequence list for file space") - if (H5S_select_iter_init(&mem_iter, mem_spaces[i], element_size, 0) < 0) + file_iter_init = TRUE; + if (H5S_select_iter_init(mem_iter, mem_spaces[i], element_size, 0) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "can't initialize sequence list for memory space") + mem_iter_init = TRUE; /* Get the number of elements in selection */ if ((hss_nelmts = (hssize_t)H5S_GET_SELECT_NPOINTS(file_spaces[i])) < 0) @@ -1470,7 +1501,7 @@ H5FD__write_selection_translate(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, ui while (file_seq_i < file_nseq || nelmts > 0) { /* Fill/refill file sequence list if necessary */ if (file_seq_i == H5FD_SEQ_LIST_LEN) { - if (H5S_SELECT_ITER_GET_SEQ_LIST(&file_iter, H5FD_SEQ_LIST_LEN, SIZE_MAX, &file_nseq, + if (H5S_SELECT_ITER_GET_SEQ_LIST(file_iter, H5FD_SEQ_LIST_LEN, SIZE_MAX, &file_nseq, &seq_nelem, file_off, file_len) < 0) HGOTO_ERROR(H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed") HDassert(file_nseq > 0); @@ -1482,7 +1513,7 @@ H5FD__write_selection_translate(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, ui /* Fill/refill memory sequence list if necessary */ if (mem_seq_i == H5FD_SEQ_LIST_LEN) { - if (H5S_SELECT_ITER_GET_SEQ_LIST(&mem_iter, H5FD_SEQ_LIST_LEN, SIZE_MAX, &mem_nseq, + if (H5S_SELECT_ITER_GET_SEQ_LIST(mem_iter, H5FD_SEQ_LIST_LEN, SIZE_MAX, &mem_nseq, &seq_nelem, mem_off, mem_len) < 0) HGOTO_ERROR(H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed") HDassert(mem_nseq > 0); @@ -1571,14 +1602,10 @@ H5FD__write_selection_translate(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, ui } } + /* Make sure both memory and file sequences terminated at the same time */ if (mem_seq_i < mem_nseq) HGOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, "file selection terminated before memory selection") - /* Terminate iterators */ - if (H5S_SELECT_ITER_RELEASE(&file_iter) < 0) - HGOTO_ERROR(H5E_INTERNAL, H5E_CANTFREE, FAIL, "can't release file selection iterator") - if (H5S_SELECT_ITER_RELEASE(&mem_iter) < 0) - HGOTO_ERROR(H5E_INTERNAL, H5E_CANTFREE, FAIL, "can't release memory selection iterator") } /* Issue vector write call if appropriate */ @@ -1590,7 +1617,19 @@ H5FD__write_selection_translate(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, ui } done: - /* Cleanup */ + /* Terminate iterators */ + if (file_iter) { + if (file_iter_init && H5S_SELECT_ITER_RELEASE(file_iter) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_CANTFREE, FAIL, "can't release file selection iterator") + file_iter = H5FL_FREE(H5S_sel_iter_t, file_iter); + } + if (mem_iter) { + if (mem_iter_init && H5S_SELECT_ITER_RELEASE(mem_iter) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_CANTFREE, FAIL, "can't release memory selection iterator") + mem_iter = H5FL_FREE(H5S_sel_iter_t, mem_iter); + } + + /* Cleanup vector arrays */ if (use_vector) { if (addrs != addrs_static) addrs = H5MM_xfree(addrs); @@ -2127,8 +2166,8 @@ H5FD__vsrt_tmp_cmp(const void *element_1, const void *element_2) herr_t H5FD_sort_vector_io_req(hbool_t *vector_was_sorted, uint32_t _count, H5FD_mem_t types[], haddr_t addrs[], - size_t sizes[], const void *bufs[], H5FD_mem_t **s_types_ptr, haddr_t **s_addrs_ptr, - size_t **s_sizes_ptr, void ***s_bufs_ptr) + size_t sizes[], H5_flexible_const_ptr_t bufs[], H5FD_mem_t **s_types_ptr, haddr_t **s_addrs_ptr, + size_t **s_sizes_ptr, H5_flexible_const_ptr_t **s_bufs_ptr) { herr_t ret_value = SUCCEED; /* Return value */ size_t count = (size_t)_count; @@ -2229,7 +2268,7 @@ H5FD_sort_vector_io_req(hbool_t *vector_was_sorted, uint32_t _count, H5FD_mem_t if ((NULL == (*s_types_ptr = (H5FD_mem_t *)HDmalloc(count * sizeof(H5FD_mem_t)))) || (NULL == (*s_addrs_ptr = (haddr_t *)HDmalloc(count * sizeof(haddr_t)))) || (NULL == (*s_sizes_ptr = (size_t *)HDmalloc(count * sizeof(size_t)))) || - (NULL == (*s_bufs_ptr = (void *)HDmalloc(count * sizeof(void *))))) { + (NULL == (*s_bufs_ptr = (H5_flexible_const_ptr_t *)HDmalloc(count * sizeof(H5_flexible_const_ptr_t))))) { HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't alloc sorted vector(s)") } diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index 211da9c..74ab34e 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -1390,7 +1390,7 @@ H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNU #ifdef H5FDmpio_DEBUG if (H5FD_mpio_debug_r_flag) HDfprintf(stderr, "%s: (%d) mpi_off = %ld bytes_read = %lld\n", __func__, file->mpi_rank, - (long)mpi_off, bytes_read); + (long)mpi_off, (long long)bytes_read); #endif /* @@ -1611,7 +1611,7 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, h #ifdef H5FDmpio_DEBUG if (H5FD_mpio_debug_w_flag) HDfprintf(stderr, "%s: (%d) mpi_off = %ld bytes_written = %lld\n", __func__, file->mpi_rank, - (long)mpi_off, bytes_written); + (long)mpi_off, (long long)bytes_written); #endif /* Each process will keep track of its perceived EOF value locally, and @@ -1811,8 +1811,8 @@ H5FD__mpio_read_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t cou * are allocated, populated, and returned in s_types, s_addrs, s_sizes, and s_bufs respectively. * In this case, this function must free the memory allocated for the sorted vectors. */ - if (H5FD_sort_vector_io_req(&vector_was_sorted, count, types, addrs, sizes, (const void **)bufs, - &s_types, &s_addrs, &s_sizes, &s_bufs) < 0) + if (H5FD_sort_vector_io_req(&vector_was_sorted, count, types, addrs, sizes, (H5_flexible_const_ptr_t *)bufs, + &s_types, &s_addrs, &s_sizes, (H5_flexible_const_ptr_t **)&s_bufs) < 0) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "can't sort vector I/O request") if ((NULL == (mpi_block_lengths = (int *)HDmalloc((size_t)count * sizeof(int)))) || @@ -2385,10 +2385,10 @@ H5FD__mpio_write_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t co H5FD_mem_t * s_types = NULL; haddr_t * s_addrs = NULL; size_t * s_sizes = NULL; - void ** s_bufs = NULL; + const void ** s_bufs = NULL; int * mpi_block_lengths = NULL; char unused = 0; /* Unused, except for non-NULL pointer value */ - void * mpi_bufs_base = NULL; + const void * mpi_bufs_base = NULL; MPI_Aint mpi_bufs_base_Aint; MPI_Aint * mpi_bufs = NULL; MPI_Aint * mpi_displacements = NULL; @@ -2446,8 +2446,8 @@ H5FD__mpio_write_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t co * are allocated, populated, and returned in s_types, s_addrs, s_sizes, and s_bufs respectively. * In this case, this function must free the memory allocated for the sorted vectors. */ - if (H5FD_sort_vector_io_req(&vector_was_sorted, count, types, addrs, sizes, bufs, &s_types, &s_addrs, - &s_sizes, &s_bufs) < 0) + if (H5FD_sort_vector_io_req(&vector_was_sorted, count, types, addrs, sizes, (H5_flexible_const_ptr_t *)bufs, &s_types, &s_addrs, + &s_sizes, (H5_flexible_const_ptr_t **)&s_bufs) < 0) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "can't sort vector I/O request") /* Get the transfer mode from the API context diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h index 3b6e23c..bcbc693 100644 --- a/src/H5FDprivate.h +++ b/src/H5FDprivate.h @@ -173,9 +173,9 @@ H5_DLL haddr_t H5FD_get_base_addr(const H5FD_t *file); H5_DLL herr_t H5FD_set_paged_aggr(H5FD_t *file, hbool_t paged); H5_DLL herr_t H5FD_sort_vector_io_req(hbool_t *vector_was_sorted, uint32_t count, H5FD_mem_t types[], - haddr_t addrs[], size_t sizes[], const void *bufs[], + haddr_t addrs[], size_t sizes[], H5_flexible_const_ptr_t bufs[], H5FD_mem_t **s_types_ptr, haddr_t **s_addrs_ptr, size_t **s_sizes_ptr, - void ***s_bufs_ptr); + H5_flexible_const_ptr_t **s_bufs_ptr); H5_DLL herr_t H5FD_init(void); /* Function prototypes for MPI based VFDs*/ diff --git a/src/H5private.h b/src/H5private.h index d71fca6..4a9b203 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -2507,6 +2507,16 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); #define HDcompile_assert(e) do { typedef struct { unsigned int b: (e); } x; } while(0) */ +/* Private typedefs */ + +/* Union for const/non-const pointer for use by functions that manipulate + * pointers but do not write to their targets or return pointers to const + * specified locations. This helps us avoid compiler warnings. */ +typedef union { + void *vp; + const void *cvp; +} H5_flexible_const_ptr_t; + /* Private functions, not part of the publicly documented API */ H5_DLL herr_t H5_init_library(void); H5_DLL void H5_term_library(void); -- cgit v0.12 From b9846d174972ecaa1e236d65d6579444e382b74b Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Mon, 14 Mar 2022 20:47:32 -0700 Subject: Removes the MANIFEST file and unused release files (#1497) * Removes the MANIFEST file and unused release files * Updated tar command --- .github/workflows/main.yml | 1 - .github/workflows/pr-check.yml | 1 - MANIFEST | 3897 ---------------------------------------- bin/bbrelease | 238 +-- bin/chkmanifest | 154 -- bin/locate_sw | 238 --- bin/release | 80 +- bin/runtest | 966 ---------- bin/snapshot | 837 --------- bin/snapshot_version | 19 - bin/timekeeper | 129 -- 11 files changed, 102 insertions(+), 6458 deletions(-) delete mode 100644 MANIFEST delete mode 100755 bin/chkmanifest delete mode 100755 bin/locate_sw delete mode 100755 bin/runtest delete mode 100755 bin/snapshot delete mode 100644 bin/snapshot_version delete mode 100755 bin/timekeeper diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7b1d0c4..b99361e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -184,7 +184,6 @@ jobs: if: matrix.generator == 'autogen' run: | sh ./autogen.sh - sh ./bin/chkmanifest mkdir "${{ runner.workspace }}/build" cd "${{ runner.workspace }}/build" $GITHUB_WORKSPACE/configure --enable-shared --${{ matrix.ts }}-threadsafe --${{ matrix.hl }}-hl --${{ matrix.parallel }}-parallel --${{ matrix.cpp }}-cxx --${{ matrix.fortran }}-fortran --${{ matrix.java }}-java diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index 79d5c83..65bf42b 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -179,7 +179,6 @@ jobs: if: matrix.generator == 'autogen' run: | sh ./autogen.sh - sh ./bin/chkmanifest mkdir "${{ runner.workspace }}/build" cd "${{ runner.workspace }}/build" $GITHUB_WORKSPACE/configure --enable-shared --${{ matrix.ts }}-threadsafe --${{ matrix.hl }}-hl --${{ matrix.parallel }}-parallel --${{ matrix.cpp }}-cxx --${{ matrix.fortran }}-fortran --${{ matrix.java }}-java diff --git a/MANIFEST b/MANIFEST deleted file mode 100644 index cd6d086..0000000 --- a/MANIFEST +++ /dev/null @@ -1,3897 +0,0 @@ -# -# Copyright by The HDF Group. -# Copyright by the Board of Trustees of the University of Illinois. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. -# -#------------------------------------------------------------------------------ -# This is the list of files that are part of HDF5 source distribution. -# All files have a `./' prefix and appear in lexicographic order. -# Lines that end with _DO_NOT_DISTRIBUTE_ will not be included in a -# release. Blank lines and comments are ignored. Comments must start -# in column one with a '#'. -#------------------------------------------------------------------------------ - -./.gitattributes _DO_NOT_DISTRIBUTE_ -./.gitignore _DO_NOT_DISTRIBUTE_ -./.autom4te.cfg _DO_NOT_DISTRIBUTE_ -./.h5chkright.ini _DO_NOT_DISTRIBUTE_ -./ACKNOWLEDGMENTS -./COPYING -./COPYING_LBNL_HDF5 -./MANIFEST -./Makefile.dist -./Makefile.am -./README.md -./acsite.m4 -./autogen.sh -./configure.ac - -./.clang-format -./.github/CODEOWNERS _DO_NOT_DISTRIBUTE_ -./.github/workflows/clang-format-fix.yml _DO_NOT_DISTRIBUTE_ -./.github/workflows/clang-format-check.yml _DO_NOT_DISTRIBUTE_ -./.github/workflows/main.yml _DO_NOT_DISTRIBUTE_ -./.github/workflows/pr-check.yml _DO_NOT_DISTRIBUTE_ -./.github/workflows/codespell.yml _DO_NOT_DISTRIBUTE_ -./.github/FUNDING.yml _DO_NOT_DISTRIBUTE_ - -./m4/aclocal_fc.m4 -./m4/aclocal_fc.f90 -./m4/ax_check_class.m4 -./m4/ax_check_classpath.m4 -./m4/ax_check_java_home.m4 -./m4/ax_check_junit.m4 -./m4/ax_check_rqrd_class.m4 -./m4/ax_java_check_class.m4 -./m4/ax_java_options.m4 -./m4/ax_jni_include_dir.m4 -./m4/ax_prog_doxygen.m4 -./m4/ax_prog_jar.m4 -./m4/ax_prog_java_cc.m4 -./m4/ax_prog_java_works.m4 -./m4/ax_prog_java.m4 -./m4/ax_prog_javac_works.m4 -./m4/ax_prog_javac.m4 -./m4/ax_prog_javadoc.m4 -./m4/ax_prog_javah.m4 -./m4/ax_try_compile_java.m4 -./m4/ax_try_run_java.m4 - -./bin/bbrelease _DO_NOT_DISTRIBUTE_ -./bin/buildhdf5 -./bin/checkapi _DO_NOT_DISTRIBUTE_ -./bin/checkposix _DO_NOT_DISTRIBUTE_ -./bin/chkconfigure _DO_NOT_DISTRIBUTE_ -./bin/chkcopyright _DO_NOT_DISTRIBUTE_ -./bin/chkmanifest -./bin/cmakehdf5 -./bin/debug-ohdr _DO_NOT_DISTRIBUTE_ -./bin/dependencies -./bin/deploy -./bin/distdep -./bin/errors _DO_NOT_DISTRIBUTE_ -./bin/format_source -./bin/format_source_patch -./bin/genparser -./bin/gcov_script _DO_NOT_DISTRIBUTE_ -./bin/h5cc.in -./bin/h5redeploy.in -./bin/h5vers -./bin/iostats -./bin/locate_sw -./bin/make_err -./bin/make_overflow -./bin/make_vers -./bin/Makefile.am -./bin/makehelp -./bin/mkdirs -./bin/newer -./bin/output_filter.sh -./bin/README _DO_NOT_DISTRIBUTE_ -./bin/release -./bin/restore.sh -./bin/runtest _DO_NOT_DISTRIBUTE_ -./bin/runbkgprog _DO_NOT_DISTRIBUTE_ -./bin/snapshot -./bin/snapshot_version _DO_NOT_DISTRIBUTE_ -./bin/switch_maint_mode _DO_NOT_DISTRIBUTE_ -./bin/timekeeper _DO_NOT_DISTRIBUTE_ -./bin/trace -./bin/warnhist _DO_NOT_DISTRIBUTE_ -./bin/yodconfigure -./bin/batch/ctestP.lsf.in.cmake -./bin/batch/ctestP.sl.in.cmake -./bin/batch/ctestS.lsf.in.cmake -./bin/batch/ctestS.sl.in.cmake -./bin/batch/knl_ctestP.sl.in.cmake -./bin/batch/knl_ctestS.sl.in.cmake -./bin/batch/knl_H5detect.sl.in.cmake -./bin/batch/ctest.qsub.in.cmake -./bin/batch/ray_ctestP.lsf.in.cmake -./bin/batch/ray_ctestS.lsf.in.cmake -./bin/batch/raybsub -./bin/pkgscrpts/h5rmflags _DO_NOT_DISTRIBUTE_ -./bin/pkgscrpts/makeHDF5BinaryTarfiles.pl _DO_NOT_DISTRIBUTE_ -./bin/pkgscrpts/makeInternalREADME.pl _DO_NOT_DISTRIBUTE_ -./bin/pkgscrpts/makeOuterREADME.pl _DO_NOT_DISTRIBUTE_ - -./config/BlankForm -./config/apple -./config/cce-fflags -./config/cce-flags -./config/commence.am -./config/conclude.am -./config/conclude_fc.am -./config/examples.am -./config/freebsd -./config/gnu-cxxflags -./config/gnu-fflags -./config/gnu-flags -./config/cygwin -./config/ibm-aix -./config/ibm-flags -./config/intel-cxxflags -./config/intel-fflags -./config/intel-flags -./config/libhdf5.pc.in -./config/linux-gnu -./config/linux-gnuaout -./config/linux-gnueabihf -./config/linux-gnulibc1 -./config/linux-gnulibc2 -./config/lt_vers.am -./config/Makefile.am.blank -./config/netbsd -./config/pgi-cxxflags -./config/pgi-fflags -./config/pgi-flags -./config/solaris - -#warnings files for both autotools and CMake -./config/clang-cxxflags -./config/clang-flags -./config/clang-warnings/developer-general -./config/clang-warnings/error-general -./config/clang-warnings/general -./config/clang-warnings/no-developer-general -./config/gnu-warnings/4.8-4.last -./config/gnu-warnings/4.8 -./config/gnu-warnings/4.9 -./config/gnu-warnings/5 -./config/gnu-warnings/6 -./config/gnu-warnings/7 -./config/gnu-warnings/8 -./config/gnu-warnings/9 -./config/gnu-warnings/9.3 -./config/gnu-warnings/cxx-general -./config/gnu-warnings/cxx-4.8 -./config/gnu-warnings/cxx-4.9 -./config/gnu-warnings/cxx-5 -./config/gnu-warnings/cxx-9 -./config/gnu-warnings/cxx-developer-4.8 -./config/gnu-warnings/cxx-error-5 -./config/gnu-warnings/cxx-error-general -./config/gnu-warnings/developer-4.8 -./config/gnu-warnings/developer-7 -./config/gnu-warnings/developer-8 -./config/gnu-warnings/developer-10 -./config/gnu-warnings/developer-general -./config/gnu-warnings/error-5 -./config/gnu-warnings/error-8 -./config/gnu-warnings/error-general -./config/gnu-warnings/general -./config/gnu-warnings/gfort-general -./config/gnu-warnings/gfort-4.8 -./config/gnu-warnings/developer-gfort-5 -./config/gnu-warnings/gfort-6 -./config/gnu-warnings/gfort-8 -./config/gnu-warnings/no-cxx-developer-4.8 -./config/gnu-warnings/no-developer-4.8 -./config/gnu-warnings/no-developer-8 -./config/gnu-warnings/no-developer-general -./config/intel-warnings/15 -./config/intel-warnings/18 -./config/intel-warnings/developer-general -./config/intel-warnings/general -./config/intel-warnings/win-developer-general -./config/intel-warnings/win-general -./config/intel-warnings/ifort-general - -./config/cmake/FindMFU.cmake -./config/cmake/FindDTCMP.cmake -./config/cmake/FindCIRCLE.cmake - -./config/site-specific/BlankForm - -./doc/branches-explained.md -./doc/code-conventions.md -./doc/contributing.md -./doc/library-init-shutdown.md -./doxygen/aliases -./doxygen/CMakeLists.txt -./doxygen/Doxyfile.in -./doxygen/dox/APIVersions.dox -./doxygen/dox/About.dox -./doxygen/dox/Cookbook.dox -./doxygen/dox/DDLBNF110.dox -./doxygen/dox/DDLBNF112.dox -./doxygen/dox/FTS.dox -./doxygen/dox/GettingStarted.dox -./doxygen/dox/Glossary.dox -./doxygen/dox/H5AC_cache_config_t.dox -./doxygen/dox/MetadataCachingInHDF5.dox -./doxygen/dox/Overview.dox -./doxygen/dox/ReferenceManual.dox -./doxygen/dox/RFC.dox -./doxygen/dox/Specifications.dox -./doxygen/dox/TechnicalNotes.dox -./doxygen/dox/api-compat-macros.dox -./doxygen/dox/maybe_metadata_reads.dox -./doxygen/dox/rm-template.dox -./doxygen/dox/cookbook/Accessibility.c -./doxygen/dox/cookbook/Accessibility.dox -./doxygen/dox/cookbook/Attributes.c -./doxygen/dox/cookbook/Attributes.dox -./doxygen/dox/cookbook/Files.c -./doxygen/dox/cookbook/Files.dox -./doxygen/dox/cookbook/Performance.dox -./doxygen/examples/DebuggingHDF5Applications.html -./doxygen/examples/FF-IH_FileGroup.gif -./doxygen/examples/FF-IH_FileObject.gif -./doxygen/examples/FileFormat.html -./doxygen/examples/FileFormatSpecChunkDiagram.jpg -./doxygen/examples/Filters.html -./doxygen/examples/H5Pset_metadata_read_attempts.c -./doxygen/examples/H5Pset_object_flush_cb.c -./doxygen/examples/H5.format.1.0.html -./doxygen/examples/H5.format.1.1.html -./doxygen/examples/H5.format.2.0.html -./doxygen/examples/H5.format.html -./doxygen/examples/H5A_examples.c -./doxygen/examples/H5D_examples.c -./doxygen/examples/H5E_examples.c -./doxygen/examples/H5Fclose.c -./doxygen/examples/H5Fcreate.c -./doxygen/examples/H5F_examples.c -./doxygen/examples/H5G_examples.c -./doxygen/examples/H5I_examples.c -./doxygen/examples/H5L_examples.c -./doxygen/examples/H5O_examples.c -./doxygen/examples/H5PL_examples.c -./doxygen/examples/H5Pget_metadata_read_attempts.1.c -./doxygen/examples/H5Pget_metadata_read_attempts.2.c -./doxygen/examples/H5Pget_metadata_read_attempts.3.c -./doxygen/examples/H5Pget_object_flush_cb.c -./doxygen/examples/H5P_examples.c -./doxygen/examples/H5R_examples.c -./doxygen/examples/H5S_examples.c -./doxygen/examples/H5T_examples.c -./doxygen/examples/H5Z_examples.c -./doxygen/examples/H5_examples.c -./doxygen/examples/ImageSpec.html -./doxygen/examples/IOFlow.html -./doxygen/examples/PaletteExample1.gif -./doxygen/examples/Palettes.fm.anc.gif -./doxygen/examples/TableSpec.html -./doxygen/examples/ThreadSafeLibrary.html -./doxygen/examples/VFL.html -./doxygen/examples/hello_hdf5.c -./doxygen/hdf5_footer.html -./doxygen/hdf5_header.html -./doxygen/hdf5_navtree_hacks.js -./doxygen/hdf5doxy.css -./doxygen/hdf5doxy_layout.xml -./doxygen/img/FF-IH_FileGroup.gif -./doxygen/img/FF-IH_FileObject.gif -./doxygen/img/FileFormatSpecChunkDiagram.jpg -./doxygen/img/HDF5.png -./doxygen/img/HDFG-logo.png -./doxygen/img/IOFlow.gif -./doxygen/img/IOFlow2.gif -./doxygen/img/IOFlow3.gif -./doxygen/img/PaletteExample1.gif -./doxygen/img/Palettes.fm.anc.gif -./doxygen/img/ftv2node.png -./doxygen/img/ftv2pnode.png - -./examples/Attributes.txt -./examples/Makefile.am -./examples/h5_chunk_read.c -./examples/h5_compound.c -./examples/h5_crtgrpd.c -./examples/h5_debug_trace.c -./examples/h5_subset.c -./examples/h5_cmprss.c -./examples/h5_rdwt.c -./examples/h5_crtgrpar.c -./examples/h5_extend.c -./examples/h5_crtatt.c -./examples/h5_crtgrp.c -./examples/h5_crtdat.c -./examples/h5_drivers.c -./examples/h5_dtransform.c -./examples/h5_elink_unix2win.c -./examples/h5_extend_write.c -./examples/h5_extlink.c -./examples/h5_group.c -./examples/h5_interm_group.c -./examples/h5_read.c -./examples/h5_write.c -./examples/h5_select.c -./examples/h5_attribute.c -./examples/h5_mount.c -./examples/h5_ref_compat.c -./examples/h5_ref_extern.c -./examples/h5_reference_deprec.c -./examples/h5_ref2reg_deprec.c -./examples/h5_shared_mesg.c -./examples/ph5example.c -./examples/ph5_filtered_writes.c -./examples/ph5_filtered_writes_no_sel.c -./examples/h5_vds.c -./examples/h5_vds-exc.c -./examples/h5_vds-exclim.c -./examples/h5_vds-eiger.c -./examples/h5_vds-simpleIO.c -./examples/h5_vds-percival.c -./examples/h5_vds-percival-unlim.c -./examples/h5_vds-percival-unlim-maxmin.c -./examples/testh5cc.sh.in -./examples/README - - -#------------------------------------------------------------------------------ -# -# Begin Fortran interface -# -#------------------------------------------------------------------------------ - -./fortran/Makefile.am -./fortran/robodoc.rc - -./fortran/examples/Makefile.am -./fortran/examples/compound.f90 -./fortran/examples/compound_fortran2003.f90 -./fortran/examples/compound_complex_fortran2003.f90 -./fortran/examples/h5_cmprss.f90 -./fortran/examples/h5_crtatt.f90 -./fortran/examples/h5_crtdat.f90 -./fortran/examples/h5_crtgrp.f90 -./fortran/examples/h5_crtgrpar.f90 -./fortran/examples/h5_crtgrpd.f90 -./hl/fortran/examples/exlite.f90 -./fortran/examples/h5_extend.f90 -./hl/fortran/examples/ex_ds1.f90 -./fortran/examples/h5_rdwt.f90 -./fortran/examples/h5_subset.f90 -./fortran/examples/hyperslab.f90 -./fortran/examples/mountexample.f90 -./fortran/examples/ph5example.f90 -./fortran/examples/refobjexample.f90 -./fortran/examples/refregexample.f90 -./fortran/examples/run-fortran-ex.sh.in -./fortran/examples/selectele.f90 -./fortran/examples/testh5fc.sh.in -./fortran/examples/nested_derived_type.f90 -./fortran/examples/rwdset_fortran2003.f90 - -./fortran/src/H5_buildiface.F90 -./fortran/src/H5_f.c -./fortran/src/H5_ff.F90 -./fortran/src/H5Af.c -./fortran/src/H5Aff.F90 -./fortran/src/H5Df.c -./fortran/src/H5Dff.F90 -./fortran/src/H5Ef.c -./fortran/src/H5Eff.F90 -./fortran/src/H5Ff.c -./fortran/src/H5Fff.F90 -./fortran/src/H5Gf.c -./fortran/src/H5Gff.F90 -./fortran/src/H5If.c -./fortran/src/H5Iff.F90 -./fortran/src/H5Lf.c -./fortran/src/H5Lff.F90 -./fortran/src/H5Of.c -./fortran/src/H5Off.F90 -./fortran/src/H5Pf.c -./fortran/src/H5Pff.F90 -./fortran/src/H5Rf.c -./fortran/src/H5Rff.F90 -./fortran/src/H5Sf.c -./fortran/src/H5Sff.F90 -./fortran/src/H5Tf.c -./fortran/src/H5Tff.F90 -./fortran/src/H5VLff.F90 -./fortran/src/H5Zf.c -./fortran/src/H5Zff.F90 -./fortran/src/H5config_f.inc.cmake -./fortran/src/H5config_f.inc.in -./fortran/src/H5fort_type_defines.h.cmake -./fortran/src/H5fort_type_defines.h.in -./fortran/src/H5f90.h -./fortran/src/H5f90global.F90 -./fortran/src/H5f90i.h -./fortran/src/H5f90kit.c -./fortran/src/H5fortkit.F90 -./fortran/src/H5f90proto.h -./fortran/src/H5match_types.c -./fortran/src/HDF5.F90 -./fortran/src/Makefile.am -./fortran/src/README -./fortran/src/h5fc.in -./fortran/src/hdf5_fortrandll.def.in - -./fortran/test/Makefile.am -./fortran/test/vol_connector.F90 -./fortran/test/fflush1.F90 -./fortran/test/fflush2.F90 -./fortran/test/fortranlib_test.F90 -./fortran/test/fortranlib_test_1_8.F90 -./fortran/test/fortranlib_test_F03.F90 -./fortran/test/t.c -./fortran/test/t.h -./fortran/test/tf.F90 -./fortran/test/tH5A.F90 -./fortran/test/tH5A_1_8.F90 -./fortran/test/tH5D.F90 -./fortran/test/tH5E_F03.F90 -./fortran/test/tH5E.F90 -./fortran/test/tH5F.F90 -./fortran/test/tH5F_F03.F90 -./fortran/test/tH5G.F90 -./fortran/test/tH5G_1_8.F90 -./fortran/test/tH5I.F90 -./fortran/test/tH5L_F03.F90 -./fortran/test/tH5MISC_1_8.F90 -./fortran/test/tH5O.F90 -./fortran/test/tH5O_F03.F90 -./fortran/test/tH5P_F03.F90 -./fortran/test/tH5P.F90 -./fortran/test/tH5R.F90 -./fortran/test/tH5S.F90 -./fortran/test/tH5Sselect.F90 -./fortran/test/tH5T_F03.F90 -./fortran/test/tH5T.F90 -./fortran/test/tH5VL.F90 -./fortran/test/tH5Z.F90 -./fortran/test/tHDF5_1_8.F90 -./fortran/test/tHDF5_F03.F90 -./fortran/test/tHDF5.F90 -./fortran/test/H5_test_buildiface.F90 - -./fortran/testpar/Makefile.am -./fortran/testpar/ptest.f90 -./fortran/testpar/hyper.f90 -./fortran/testpar/mdset.f90 - -#------------------------------------------------------------------------------ -# -# End Fortran interface -# -#------------------------------------------------------------------------------ - -#------------------------------------------------------------------------------ -# -# Begin C++ interface -# -#------------------------------------------------------------------------------ - -./c++/Makefile.am - -./c++/examples/chunks.cpp -./c++/examples/compound.cpp -./c++/examples/create.cpp -./c++/examples/expected.out -./c++/examples/extend_ds.cpp -./c++/examples/h5group.cpp -./c++/examples/h5tutr_cmprss.cpp -./c++/examples/h5tutr_crtatt.cpp -./c++/examples/h5tutr_crtdat.cpp -./c++/examples/h5tutr_crtgrpar.cpp -./c++/examples/h5tutr_crtgrp.cpp -./c++/examples/h5tutr_crtgrpd.cpp -./c++/examples/h5tutr_extend.cpp -./c++/examples/h5tutr_rdwt.cpp -./c++/examples/h5tutr_subset.cpp -./c++/examples/readdata.cpp -./c++/examples/testh5c++.sh.in -./c++/examples/writedata.cpp -./c++/examples/Makefile.am -./c++/examples/run-c++-ex.sh.in - -./c++/src/H5AbstractDs.cpp -./c++/src/H5AbstractDs.h -./c++/src/H5Alltypes.h -./c++/src/H5ArrayType.cpp -./c++/src/H5ArrayType.h -./c++/src/H5AtomType.cpp -./c++/src/H5AtomType.h -./c++/src/H5Attribute.cpp -./c++/src/H5Attribute.h -./c++/src/H5Classes.h -./c++/src/H5CommonFG.cpp -./c++/src/H5CommonFG.h -./c++/src/H5CompType.cpp -./c++/src/H5CompType.h -./c++/src/H5Cpp.h -./c++/src/H5CppDoc.h -./c++/src/H5DataSet.cpp -./c++/src/H5DataSet.h -./c++/src/H5DataSpace.cpp -./c++/src/H5DataSpace.h -./c++/src/H5DataType.cpp -./c++/src/H5DataType.h -./c++/src/H5DaccProp.cpp -./c++/src/H5DaccProp.h -./c++/src/H5DcreatProp.cpp -./c++/src/H5DcreatProp.h -./c++/src/H5DxferProp.cpp -./c++/src/H5DxferProp.h -./c++/src/H5EnumType.cpp -./c++/src/H5EnumType.h -./c++/src/H5Exception.cpp -./c++/src/H5Exception.h -./c++/src/H5FaccProp.cpp -./c++/src/H5FaccProp.h -./c++/src/H5FcreatProp.cpp -./c++/src/H5FcreatProp.h -./c++/src/H5File.cpp -./c++/src/H5File.h -./c++/src/H5FloatType.cpp -./c++/src/H5FloatType.h -./c++/src/H5Group.cpp -./c++/src/H5Group.h -./c++/src/H5IdComponent.cpp -./c++/src/H5IdComponent.h -./c++/src/H5Include.h -./c++/src/H5IntType.cpp -./c++/src/H5IntType.h -./c++/src/H5LaccProp.cpp -./c++/src/H5LaccProp.h -./c++/src/H5LcreatProp.cpp -./c++/src/H5LcreatProp.h -./c++/src/H5Library.cpp -./c++/src/H5Library.h -./c++/src/H5Location.cpp -./c++/src/H5Location.h -./c++/src/H5Object.cpp -./c++/src/H5Object.h -./c++/src/H5OcreatProp.cpp -./c++/src/H5OcreatProp.h -./c++/src/H5PredType.cpp -./c++/src/H5PredType.h -./c++/src/H5PropList.cpp -./c++/src/H5PropList.h -./c++/src/H5StrType.cpp -./c++/src/H5StrType.h -./c++/src/H5VarLenType.cpp -./c++/src/H5VarLenType.h -./c++/src/Makefile.am -./c++/src/RM_stylesheet.css -./c++/src/C2Cppfunction_map.htm -./c++/src/cpp_doc_config -./c++/src/h5c++.in -./c++/src/footer.html -./c++/src/header.html -./c++/src/header_files/filelist.xml -./c++/src/header_files/hdf_logo.jpg -./c++/src/header_files/help.jpg -./c++/src/header_files/image001.jpg -./c++/src/header_files/image002.jpg - -./c++/test/H5srcdir_str.h.in -./c++/test/Makefile.am -./c++/test/dsets.cpp -./c++/test/h5cpputil.cpp -./c++/test/h5cpputil.h -./c++/test/tarray.cpp -./c++/test/tattr.cpp -./c++/test/tcompound.cpp -./c++/test/tdspl.cpp -./c++/test/testhdf5.cpp -./c++/test/tfile.cpp -./c++/test/tfilter.cpp -./c++/test/th5s.cpp -./c++/test/th5s.h5 -./c++/test/titerate.cpp -./c++/test/tlinks.cpp -./c++/test/tobject.cpp -./c++/test/ttypes.cpp -./c++/test/trefer.cpp -./c++/test/tvlstr.cpp - -#------------------------------------------------------------------------------ -# -# End C++ interface -# -#------------------------------------------------------------------------------ - -./release_docs/HISTORY-1_0-1_8_0_rc3.txt -./release_docs/HISTORY-1_8_0-1_10_0.txt -./release_docs/HISTORY-1_10_0-1_12_0.txt -./release_docs/HISTORY-1_8.txt -./release_docs/HISTORY-1_10.txt -./release_docs/HISTORY-1_12.txt -./release_docs/HISTORY-1_13.txt -./release_docs/INSTALL -./release_docs/INSTALL_CMake.txt -./release_docs/INSTALL_Cygwin.txt -./release_docs/INSTALL_parallel -./release_docs/INSTALL_Warnings.txt -./release_docs/INSTALL_Windows.txt -./release_docs/README_HDF5_CMake -./release_docs/README_HPC -./release_docs/RELEASE.txt -./release_docs/USING_HDF5_CMake.txt -./release_docs/USING_HDF5_VS.txt - -./src/.indent.pro _DO_NOT_DISTRIBUTE_ -./src/hdf5.lnt _DO_NOT_DISTRIBUTE_ -./src/hdf5-win.lnt _DO_NOT_DISTRIBUTE_ -./src/hdf5-lin.lnt _DO_NOT_DISTRIBUTE_ -./src/H5.c -./src/H5checksum.c -./src/H5dbg.c -./src/H5api_adpt.h -./src/H5err.txt -./src/H5detect.c -./src/H5make_libsettings.c -./src/H5module.h -./src/H5mpi.c -./src/H5overflow.txt -./src/H5private.h -./src/H5public.h -./src/H5system.c -./src/H5timer.c -./src/H5trace.c -./src/H5vers.txt -./src/H5A.c -./src/H5Abtree2.c -./src/H5Adense.c -./src/H5Adeprec.c -./src/H5Aint.c -./src/H5Amodule.h -./src/H5Atest.c -./src/H5Apkg.h -./src/H5Aprivate.h -./src/H5Apublic.h -./src/H5AC.c -./src/H5ACdbg.c -./src/H5ACmodule.h -./src/H5ACmpio.c -./src/H5ACpkg.h -./src/H5ACprivate.h -./src/H5ACpublic.h -./src/H5ACproxy_entry.c -./src/H5B.c -./src/H5Bcache.c -./src/H5Bdbg.c -./src/H5Bmodule.h -./src/H5Bpkg.h -./src/H5Bprivate.h -./src/H5B2.c -./src/H5B2cache.c -./src/H5B2dbg.c -./src/H5B2hdr.c -./src/H5B2int.c -./src/H5B2internal.c -./src/H5B2leaf.c -./src/H5B2module.h -./src/H5B2pkg.h -./src/H5B2private.h -./src/H5B2stat.c -./src/H5B2test.c -./src/H5C.c -./src/H5Cdbg.c -./src/H5Cepoch.c -./src/H5Cimage.c -./src/H5Clog.c -./src/H5Clog.h -./src/H5Clog_json.c -./src/H5Clog_trace.c -./src/H5Cmodule.h -./src/H5Cmpio.c -./src/H5Cpkg.h -./src/H5Cprefetched.c -./src/H5Cprivate.h -./src/H5Cpublic.h -./src/H5Cquery.c -./src/H5Ctag.c -./src/H5Ctest.c -./src/H5CS.c -./src/H5CSprivate.h -./src/H5CX.c -./src/H5CXmodule.h -./src/H5CXprivate.h -./src/H5D.c -./src/H5Dbtree.c -./src/H5Dbtree2.c -./src/H5Dchunk.c -./src/H5Dcompact.c -./src/H5Dcontig.c -./src/H5Ddbg.c -./src/H5Ddeprec.c -./src/H5Dearray.c -./src/H5Defl.c -./src/H5Dfarray.c -./src/H5Dfill.c -./src/H5Dint.c -./src/H5Dio.c -./src/H5Dlayout.c -./src/H5Dmodule.h -./src/H5Dmpio.c -./src/H5Dnone.c -./src/H5Doh.c -./src/H5Dpkg.h -./src/H5Dprivate.h -./src/H5Dpublic.h -./src/H5Dscatgath.c -./src/H5Dselect.c -./src/H5Dsingle.c -./src/H5Dtest.c -./src/H5Dvirtual.c -./src/H5E.c -./src/H5Edeprec.c -./src/H5Eint.c -./src/H5Emodule.h -./src/H5Epkg.h -./src/H5Eprivate.h -./src/H5Epublic.h -./src/H5EA.c -./src/H5EAcache.c -./src/H5EAdbg.c -./src/H5EAdblkpage.c -./src/H5EAdblock.c -./src/H5EAhdr.c -./src/H5EAiblock.c -./src/H5EAint.c -./src/H5EAmodule.h -./src/H5EApkg.h -./src/H5EAprivate.h -./src/H5EAsblock.c -./src/H5EAstat.c -./src/H5EAtest.c -./src/H5ES.c -./src/H5ESdevelop.h -./src/H5ESevent.c -./src/H5ESint.c -./src/H5ESlist.c -./src/H5ESmodule.h -./src/H5ESpkg.h -./src/H5ESprivate.h -./src/H5ESpublic.h -./src/H5F.c -./src/H5Faccum.c -./src/H5Fcwfs.c -./src/H5Fdbg.c -./src/H5Fdeprec.c -./src/H5Fefc.c -./src/H5Ffake.c -./src/H5Fint.c -./src/H5Fio.c -./src/H5Fmodule.h -./src/H5Fmount.c -./src/H5Fmpi.c -./src/H5Fquery.c -./src/H5Fsfile.c -./src/H5Fspace.c -./src/H5Fsuper.c -./src/H5Fsuper_cache.c -./src/H5Fpkg.h -./src/H5Fprivate.h -./src/H5Fpublic.h -./src/H5Ftest.c -./src/H5FA.c -./src/H5FAcache.c -./src/H5FAdbg.c -./src/H5FAdblkpage.c -./src/H5FAdblock.c -./src/H5FAhdr.c -./src/H5FAint.c -./src/H5FAmodule.h -./src/H5FApkg.h -./src/H5FAprivate.h -./src/H5FAstat.c -./src/H5FAtest.c -./src/H5FD.c -./src/H5FDcore.c -./src/H5FDcore.h -./src/H5FDdevelop.h -./src/H5FDdirect.c -./src/H5FDdirect.h -./src/H5FDdrvr_module.h -./src/H5FDfamily.c -./src/H5FDfamily.h -./src/H5FDhdfs.c -./src/H5FDhdfs.h -./src/H5FDint.c -./src/H5FDlog.c -./src/H5FDlog.h -./src/H5FDmirror.c -./src/H5FDmirror.h -./src/H5FDmirror_priv.h -./src/H5FDmodule.h -./src/H5FDmpi.c -./src/H5FDmpi.h -./src/H5FDmpio.c -./src/H5FDmpio.h -./src/H5FDmulti.c -./src/H5FDmulti.h -./src/H5FDperform.c -./src/H5FDros3.c -./src/H5FDros3.h -./src/H5FDpkg.h -./src/H5FDprivate.h -./src/H5FDpublic.h -./src/H5FDs3comms.h -./src/H5FDs3comms.c -./src/H5FDsec2.c -./src/H5FDsec2.h -./src/H5FDspace.c -./src/H5FDsplitter.c -./src/H5FDsplitter.h -./src/H5FDstdio.c -./src/H5FDstdio.h -./src/H5FDtest.c -./src/H5FDwindows.c -./src/H5FDwindows.h -./src/H5FL.c -./src/H5FLmodule.h -./src/H5FLprivate.h -./src/H5FO.c -./src/H5FOprivate.h -./src/H5FS.c -./src/H5FScache.c -./src/H5FSdbg.c -./src/H5FSint.c -./src/H5FSmodule.h -./src/H5FSpkg.h -./src/H5FSprivate.h -./src/H5FSsection.c -./src/H5FSstat.c -./src/H5FStest.c -./src/H5G.c -./src/H5Gbtree2.c -./src/H5Gcache.c -./src/H5Gcompact.c -./src/H5Gdense.c -./src/H5Gdeprec.c -./src/H5Gent.c -./src/H5Gint.c -./src/H5Glink.c -./src/H5Gloc.c -./src/H5Gmodule.h -./src/H5Gname.c -./src/H5Gnode.c -./src/H5Gobj.c -./src/H5Goh.c -./src/H5Gpkg.h -./src/H5Gprivate.h -./src/H5Gpublic.h -./src/H5Groot.c -./src/H5Gstab.c -./src/H5Gtest.c -./src/H5Gtraverse.c -./src/H5HF.c -./src/H5HFbtree2.c -./src/H5HFcache.c -./src/H5HFdbg.c -./src/H5HFdblock.c -./src/H5HFdtable.c -./src/H5HFhdr.c -./src/H5HFhuge.c -./src/H5HFiblock.c -./src/H5HFiter.c -./src/H5HFman.c -./src/H5HFmodule.h -./src/H5HFpkg.h -./src/H5HFprivate.h -./src/H5HFsection.c -./src/H5HFspace.c -./src/H5HFstat.c -./src/H5HFtest.c -./src/H5HFtiny.c -./src/H5HG.c -./src/H5HGcache.c -./src/H5HGdbg.c -./src/H5HGmodule.h -./src/H5HGpkg.h -./src/H5HGprivate.h -./src/H5HGquery.c -./src/H5HL.c -./src/H5HLcache.c -./src/H5HLdbg.c -./src/H5HLdblk.c -./src/H5HLint.c -./src/H5HLmodule.h -./src/H5HLpkg.h -./src/H5HLprfx.c -./src/H5HLprivate.h -./src/H5HP.c -./src/H5HPprivate.h -./src/H5I.c -./src/H5Idbg.c -./src/H5Idevelop.h -./src/H5Iint.c -./src/H5Imodule.h -./src/H5Ipkg.h -./src/H5Iprivate.h -./src/H5Ipublic.h -./src/H5Itest.c -./src/H5L.c -./src/H5Ldeprec.c -./src/H5Ldevelop.h -./src/H5Lexternal.c -./src/H5Lint.c -./src/H5Lmodule.h -./src/H5Lpkg.h -./src/H5Lprivate.h -./src/H5Lpublic.h -./src/H5M.c -./src/H5Mmodule.h -./src/H5Mpkg.h -./src/H5Mprivate.h -./src/H5Mpublic.h -./src/H5MF.c -./src/H5MFaggr.c -./src/H5MFdbg.c -./src/H5MFmodule.h -./src/H5MFsection.c -./src/H5MFpkg.h -./src/H5MFprivate.h -./src/H5MM.c -./src/H5MMprivate.h -./src/H5MMpublic.h -./src/H5MP.c -./src/H5MPmodule.h -./src/H5MPpkg.h -./src/H5MPprivate.h -./src/H5MPtest.c -./src/H5O.c -./src/H5Oainfo.c -./src/H5Oalloc.c -./src/H5Oattr.c -./src/H5Oattribute.c -./src/H5Obogus.c -./src/H5Obtreek.c -./src/H5Ocache.c -./src/H5Ocache_image.c -./src/H5Ochunk.c -./src/H5Ocont.c -./src/H5Ocopy.c -./src/H5Ocopy_ref.c -./src/H5Odbg.c -./src/H5Odeprec.c -./src/H5Odrvinfo.c -./src/H5Odtype.c -./src/H5Oefl.c -./src/H5Ofill.c -./src/H5Oflush.c -./src/H5Ofsinfo.c -./src/H5Oginfo.c -./src/H5Oint.c -./src/H5Olayout.c -./src/H5Olinfo.c -./src/H5Olink.c -./src/H5Omessage.c -./src/H5Omodule.h -./src/H5Omtime.c -./src/H5Oname.c -./src/H5Onull.c -./src/H5Opkg.h -./src/H5Opline.c -./src/H5Oprivate.h -./src/H5Opublic.h -./src/H5Orefcount.c -./src/H5Osdspace.c -./src/H5Oshared.c -./src/H5Oshared.h -./src/H5Oshmesg.c -./src/H5Ostab.c -./src/H5Otest.c -./src/H5Ounknown.c -./src/H5P.c -./src/H5Pacpl.c -./src/H5Pdapl.c -./src/H5Pdcpl.c -./src/H5Pdeprec.c -./src/H5Pdxpl.c -./src/H5Pencdec.c -./src/H5Pfapl.c -./src/H5Pfcpl.c -./src/H5Pfmpl.c -./src/H5Pgcpl.c -./src/H5Pint.c -./src/H5Plapl.c -./src/H5Plcpl.c -./src/H5Pmapl.c -./src/H5Pmcpl.c -./src/H5Pmodule.h -./src/H5Pocpl.c -./src/H5Pocpypl.c -./src/H5Ppkg.h -./src/H5Pprivate.h -./src/H5Ppublic.h -./src/H5Pstrcpl.c -./src/H5Ptest.c -./src/H5PB.c -./src/H5PBmodule.h -./src/H5PBpkg.h -./src/H5PBprivate.h -./src/H5PL.c -./src/H5PLint.c -./src/H5PLmodule.h -./src/H5PLpath.c -./src/H5PLpkg.h -./src/H5PLplugin_cache.c -./src/H5PLprivate.h -./src/H5PLpublic.h -./src/H5PLextern.h -./src/H5R.c -./src/H5Rdeprec.c -./src/H5Rint.c -./src/H5Rmodule.h -./src/H5Rpkg.h -./src/H5Rprivate.h -./src/H5Rpublic.h -./src/H5RS.c -./src/H5RSmodule.h -./src/H5RSprivate.h -./src/H5S.c -./src/H5Sall.c -./src/H5Sdbg.c -./src/H5Sdeprec.c -./src/H5Shyper.c -./src/H5Smodule.h -./src/H5Smpio.c -./src/H5Snone.c -./src/H5Spkg.h -./src/H5Spoint.c -./src/H5Sprivate.h -./src/H5Spublic.h -./src/H5Sselect.c -./src/H5Stest.c -./src/H5SL.c -./src/H5SLmodule.h -./src/H5SLprivate.h -./src/H5SM.c -./src/H5SMbtree2.c -./src/H5SMcache.c -./src/H5SMmessage.c -./src/H5SMmodule.h -./src/H5SMpkg.h -./src/H5SMprivate.h -./src/H5SMtest.c -./src/H5T.c -./src/H5Tarray.c -./src/H5Tbit.c -./src/H5Tcommit.c -./src/H5Tcompound.c -./src/H5Tconv.c -./src/H5Tcset.c -./src/H5Tdbg.c -./src/H5Tdeprec.c -./src/H5Tdevelop.h -./src/H5Tenum.c -./src/H5Tfields.c -./src/H5Tfixed.c -./src/H5Tfloat.c -./src/H5Tmodule.h -./src/H5Tnative.c -./src/H5Toffset.c -./src/H5Topaque.c -./src/H5Torder.c -./src/H5Toh.c -./src/H5Tpad.c -./src/H5Tpkg.h -./src/H5Tprecis.c -./src/H5Tprivate.h -./src/H5Tpublic.h -./src/H5Tref.c -./src/H5Tstrpad.c -./src/H5Tvisit.c -./src/H5Tvlen.c -./src/H5TS.c -./src/H5TSdevelop.h -./src/H5TSprivate.h -./src/H5UC.c -./src/H5UCprivate.h -./src/H5VL.c -./src/H5VLcallback.c -./src/H5VLconnector.h -./src/H5VLconnector_passthru.h -./src/H5VLdyn_ops.c -./src/H5VLint.c -./src/H5VLmodule.h -./src/H5VLnative.c -./src/H5VLnative.h -./src/H5VLnative_attr.c -./src/H5VLnative_blob.c -./src/H5VLnative_dataset.c -./src/H5VLnative_datatype.c -./src/H5VLnative_file.c -./src/H5VLnative_group.c -./src/H5VLnative_link.c -./src/H5VLnative_introspect.c -./src/H5VLnative_object.c -./src/H5VLnative_token.c -./src/H5VLnative_private.h -./src/H5VLpassthru.c -./src/H5VLpassthru.h -./src/H5VLpkg.h -./src/H5VLprivate.h -./src/H5VLpublic.h -./src/H5VLtest.c -./src/H5VM.c -./src/H5VMprivate.h -./src/H5WB.c -./src/H5WBprivate.h -./src/H5Z.c -./src/H5Zdeflate.c -./src/H5Zdevelop.h -./src/H5Zfletcher32.c -./src/H5Zmodule.h -./src/H5Znbit.c -./src/H5Zpkg.h -./src/H5Zprivate.h -./src/H5Zpublic.h -./src/H5Zscaleoffset.c -./src/H5Zshuffle.c -./src/H5Zszip.c -./src/H5Ztrans.c -./src/Makefile.am -./src/hdf5.h -./src/libhdf5.settings.in -./src/H5win32defs.h -./src/uthash.h - -./test/AtomicWriterReader.txt -./test/H5srcdir.h -./test/H5srcdir_str.h.in -./test/Makefile.am -./test/POSIX_Order_Write_Test_Report.docx -./test/POSIX_Order_Write_Test_Report.pdf -./test/SWMR_POSIX_Order_UG.txt -./test/SWMR_UseCase_UG.txt -./test/accum.c -./test/accum_swmr_reader.c -./test/aggr.h5 -./test/app_ref.c -./test/atomic_reader.c -./test/atomic_writer.c -./test/bad_compound.h5 -./test/bad_offset.h5 -./test/be_data.h5 -./test/be_extlink1.h5 -./test/be_extlink2.h5 -./test/big.c -./test/bittests.c -./test/btree2.c -./test/btree_idx_1_6.h5 -./test/btree_idx_1_8.h5 -./test/cache.c -./test/cache_api.c -./test/cache_common.c -./test/cache_common.h -./test/cache_image.c -./test/cache_logging.c -./test/cache_tagging.c -./test/chunk_info.c -./test/cmpd_dset.c -./test/cmpd_dtransform.c -./test/cork.c -./test/corrupt_stab_msg.h5 -./test/cross_read.c -./test/cve_2020_10810.h5 -./test/dangle.c -./test/deflate.h5 -./test/del_many_dense_attrs.c -./test/direct_chunk.c -./test/dsets.c -./test/dt_arith.c -./test/dtransform.c -./test/dtypes.c -./test/earray.c -./test/efc.c -./test/enc_dec_plist.c -./test/enc_dec_plist_cross_platform.c -./test/enum.c -./test/err_compat.c -./test/error_test.c -./test/event_set.c -./test/evict_on_close.c -./test/extend.c -./test/external.c -./test/external_common.c -./test/external_common.h -./test/external_env.c -./test/external_fname.h -./test/family_v16-000000.h5 -./test/family_v16-000001.h5 -./test/family_v16-000002.h5 -./test/family_v16-000003.h5 -./test/farray.c -./test/fheap.c -./test/file_image.c -./test/file_image_core_test.h5 -./test/filenotclosed.c -./test/filespace_1_6.h5 -./test/filespace_1_8.h5 -./test/fill18.h5 -./test/fill_old.h5 -./test/fillval.c -./test/filter_error.h5 -./test/filter_fail.c -./test/filter_plugin.c -./test/filter_plugin1_dsets.c -./test/filter_plugin2_dsets.c -./test/filter_plugin3_dsets.c -./test/filter_plugin4_groups.c -./test/flush1.c -./test/flush2.c -./test/flushrefresh.c -./test/freespace.c -./test/fsm_aggr_nopersist.h5 -./test/fsm_aggr_persist.h5 -./test/gen_bad_compound.c -./test/gen_bad_offset.c -./test/gen_bad_ohdr.c -./test/gen_bogus.c -./test/gen_bounds.c -./test/gen_cross.c -./test/gen_deflate.c -./test/gen_file_image.c -./test/gen_filespace.c -./test/gen_filters.c -./test/gen_mergemsg.c -./test/gen_new_array.c -./test/gen_new_fill.c -./test/gen_new_group.c -./test/gen_new_mtime.c -./test/gen_new_super.c -./test/gen_noencoder.c -./test/gen_nullspace.c -./test/gen_old_array.c -./test/gen_old_group.c -./test/gen_old_layout.c -./test/gen_old_mtime.c -./test/gen_plist.c -./test/gen_sizes_lheap.c -./test/gen_specmetaread.c -./test/gen_udlinks.c -./test/genall5.c -./test/genall5.h -./test/getname.c -./test/gheap.c -./test/group_old.h5 -./test/h5fc_ext1_f.h5 -./test/h5fc_ext1_i.h5 -./test/h5fc_ext2_if.h5 -./test/h5fc_ext2_sf.h5 -./test/h5fc_ext3_isf.h5 -./test/h5fc_ext_none.h5 -./test/h5test.c -./test/h5test.h -./test/hdfs.c -./test/hyperslab.c -./test/istore.c -./test/le_data.h5 -./test/le_extlink1.h5 -./test/le_extlink2.h5 -./test/lheap.c -./test/links.c -./test/links_env.c -./test/memleak_H5O_dtype_decode_helper_H5Odtype.h5 -./test/mergemsg.h5 -./test/mf.c -./test/mirror_vfd.c -./test/mount.c -./test/mtime.c -./test/multi_file_v16-r.h5 -./test/multi_file_v16-s.h5 -./test/noencoder.h5 -./test/none.h5 -./test/ntypes.c -./test/null_vol_connector.c -./test/null_vol_connector.h -./test/null_vfd_plugin.c -./test/null_vfd_plugin.h -./test/objcopy.c -./test/objcopy_ref.c -./test/ohdr.c -./test/page_buffer.c -./test/paged_nopersist.h5 -./test/paged_persist.h5 -./test/pool.c -./test/reserved.c -./test/ros3.c -./test/s3comms.c -./test/set_extent.c -# ====distribute this for now. See HDFFV-8236==== -./test/space_overflow.c -# ====end distribute this for now. See HDFFV-8236==== -./test/specmetaread.h5 -./test/stab.c -./test/swmr.c -./test/swmr_addrem_writer.c -./test/swmr_common.c -./test/swmr_common.h -./test/swmr_generator.c -./test/swmr_reader.c -./test/swmr_remove_reader.c -./test/swmr_remove_writer.c -./test/swmr_sparse_reader.c -./test/swmr_sparse_writer.c -./test/swmr_start_write.c -./test/swmr_writer.c -./test/tarray.c -./test/tarrold.h5 -./test/tattr.c -./test/tbad_msg_count.h5 -./test/tbogus.h5 -./test/tcheck_version.c -./test/tchecksum.c -./test/tconfig.c -./test/tcoords.c -./test/test_abort_fail.sh.in -./test/test_check_version.sh.in -./test/test_error.sh.in -./test/test_external_env.sh.in -./test/test_filters_be.h5 -./test/test_filters_le.h5 -./test/test_flush_refresh.sh.in -./test/test_libinfo.sh.in -./test/test_links_env.sh.in -./test/test_mirror.sh.in -./test/test_plugin.sh.in -./test/test_swmr.pwsh.in -./test/test_swmr.sh.in -./test/test_use_cases.sh.in -./test/test_vds_env.sh.in -./test/test_vds_swmr.pwsh.in -./test/test_vds_swmr.sh.in -./test/testframe.c -./test/testhdf5.c -./test/testhdf5.h -./test/testmeta.c -./test/tfile.c -./test/tgenprop.c -./test/th5o.c -./test/th5s.c -./test/th5s.h5 -./test/theap.c -./test/thread_id.c -./test/tid.c -./test/timer.c -./test/titerate.c -./test/tlayouto.h5 -./test/tmeta.c -./test/tmisc.c -./test/tmtimen.h5 -./test/tmtimeo.h5 -./test/trefer.c -./test/trefer_deprec.c -./test/trefer_shutdown.c -./test/trefstr.c -./test/tselect.c -./test/tsizeslheap.h5 -./test/tskiplist.c -./test/tsohm.c -./test/ttime.c -./test/ttsafe.c -./test/ttsafe.h -./test/ttsafe_acreate.c -./test/ttsafe_attr_vlen.c -./test/ttsafe_cancel.c -./test/ttsafe_dcreate.c -./test/ttsafe_error.c -./test/tunicode.c -./test/tvlstr.c -./test/tvltypes.c -./test/twriteorder.c -./test/unlink.c -./test/unregister.c -./test/use.h -./test/use_append_chunk.c -./test/use_append_chunk_mirror.c -./test/use_append_mchunks.c -./test/use_common.c -./test/use_disable_mdc_flushes.c -./test/vds.c -./test/vds_env.c -./test/vds_swmr.h -./test/vds_swmr_gen.c -./test/vds_swmr_reader.c -./test/vds_swmr_writer.c -./test/vfd.c -./test/vfd_plugin.c -./test/vol.c -./test/vol_plugin.c - -./test/testfiles/err_compat_1 -./test/testfiles/err_compat_2 -./test/testfiles/error_test_1 -./test/testfiles/error_test_2 -./test/testfiles/links_env.out -./test/testfiles/plist_files/acpl_32be -./test/testfiles/plist_files/acpl_32le -./test/testfiles/plist_files/acpl_64be -./test/testfiles/plist_files/acpl_64le -./test/testfiles/plist_files/dapl_32be -./test/testfiles/plist_files/dapl_32le -./test/testfiles/plist_files/dapl_64be -./test/testfiles/plist_files/dapl_64le -./test/testfiles/plist_files/dcpl_32be -./test/testfiles/plist_files/dcpl_32le -./test/testfiles/plist_files/dcpl_64be -./test/testfiles/plist_files/dcpl_64le -./test/testfiles/plist_files/def_acpl_32be -./test/testfiles/plist_files/def_acpl_32le -./test/testfiles/plist_files/def_acpl_64be -./test/testfiles/plist_files/def_acpl_64le -./test/testfiles/plist_files/def_dapl_32be -./test/testfiles/plist_files/def_dapl_32le -./test/testfiles/plist_files/def_dapl_64be -./test/testfiles/plist_files/def_dapl_64le -./test/testfiles/plist_files/def_dcpl_32be -./test/testfiles/plist_files/def_dcpl_32le -./test/testfiles/plist_files/def_dcpl_64be -./test/testfiles/plist_files/def_dcpl_64le -./test/testfiles/plist_files/def_dxpl_32be -./test/testfiles/plist_files/def_dxpl_32le -./test/testfiles/plist_files/def_dxpl_64be -./test/testfiles/plist_files/def_dxpl_64le -./test/testfiles/plist_files/def_fapl_32be -./test/testfiles/plist_files/def_fapl_32le -./test/testfiles/plist_files/def_fapl_64be -./test/testfiles/plist_files/def_fapl_64le -./test/testfiles/plist_files/def_fcpl_32be -./test/testfiles/plist_files/def_fcpl_32le -./test/testfiles/plist_files/def_fcpl_64be -./test/testfiles/plist_files/def_fcpl_64le -./test/testfiles/plist_files/def_gcpl_32be -./test/testfiles/plist_files/def_gcpl_32le -./test/testfiles/plist_files/def_gcpl_64be -./test/testfiles/plist_files/def_gcpl_64le -./test/testfiles/plist_files/def_lapl_32be -./test/testfiles/plist_files/def_lapl_32le -./test/testfiles/plist_files/def_lapl_64be -./test/testfiles/plist_files/def_lapl_64le -./test/testfiles/plist_files/def_lcpl_32be -./test/testfiles/plist_files/def_lcpl_32le -./test/testfiles/plist_files/def_lcpl_64be -./test/testfiles/plist_files/def_lcpl_64le -./test/testfiles/plist_files/def_ocpl_32be -./test/testfiles/plist_files/def_ocpl_32le -./test/testfiles/plist_files/def_ocpl_64be -./test/testfiles/plist_files/def_ocpl_64le -./test/testfiles/plist_files/def_ocpypl_32be -./test/testfiles/plist_files/def_ocpypl_32le -./test/testfiles/plist_files/def_ocpypl_64be -./test/testfiles/plist_files/def_ocpypl_64le -./test/testfiles/plist_files/def_strcpl_32be -./test/testfiles/plist_files/def_strcpl_32le -./test/testfiles/plist_files/def_strcpl_64be -./test/testfiles/plist_files/def_strcpl_64le -./test/testfiles/plist_files/dxpl_32be -./test/testfiles/plist_files/dxpl_32le -./test/testfiles/plist_files/dxpl_64be -./test/testfiles/plist_files/dxpl_64le -./test/testfiles/plist_files/fapl_32be -./test/testfiles/plist_files/fapl_32le -./test/testfiles/plist_files/fapl_64be -./test/testfiles/plist_files/fapl_64le -./test/testfiles/plist_files/fcpl_32be -./test/testfiles/plist_files/fcpl_32le -./test/testfiles/plist_files/fcpl_64be -./test/testfiles/plist_files/fcpl_64le -./test/testfiles/plist_files/gcpl_32be -./test/testfiles/plist_files/gcpl_32le -./test/testfiles/plist_files/gcpl_64be -./test/testfiles/plist_files/gcpl_64le -./test/testfiles/plist_files/lapl_32be -./test/testfiles/plist_files/lapl_32le -./test/testfiles/plist_files/lapl_64be -./test/testfiles/plist_files/lapl_64le -./test/testfiles/plist_files/lcpl_32be -./test/testfiles/plist_files/lcpl_32le -./test/testfiles/plist_files/lcpl_64be -./test/testfiles/plist_files/lcpl_64le -./test/testfiles/plist_files/ocpl_32be -./test/testfiles/plist_files/ocpl_32le -./test/testfiles/plist_files/ocpl_64be -./test/testfiles/plist_files/ocpl_64le -./test/testfiles/plist_files/ocpypl_32be -./test/testfiles/plist_files/ocpypl_32le -./test/testfiles/plist_files/ocpypl_64be -./test/testfiles/plist_files/ocpypl_64le -./test/testfiles/plist_files/strcpl_32be -./test/testfiles/plist_files/strcpl_32le -./test/testfiles/plist_files/strcpl_64be -./test/testfiles/plist_files/strcpl_64le - -./testpar/Makefile.am -./testpar/t_bigio.c -./testpar/t_cache.c -./testpar/t_cache_image.c -./testpar/t_chunk_alloc.c -./testpar/t_coll_chunk.c -./testpar/t_coll_md_read.c -./testpar/t_dset.c -./testpar/t_file.c -./testpar/t_file_image.c -./testpar/t_filter_read.c -./testpar/t_filters_parallel.c -./testpar/t_filters_parallel.h -./testpar/t_mdset.c -./testpar/t_mpi.c -./testpar/t_ph5basic.c -./testpar/t_pflush1.c -./testpar/t_pflush2.c -./testpar/t_pread.c -./testpar/t_prop.c -./testpar/t_shapesame.c -./testpar/t_pshutdown.c -./testpar/t_prestart.c -./testpar/t_span_tree.c -./testpar/t_init_term.c -./testpar/t_2Gio.c -./testpar/testpar.h -./testpar/testpflush.sh.in -./testpar/testphdf5.c -./testpar/testphdf5.h - -./tools/Makefile.am -./tools/src/Makefile.am -./tools/test/Makefile.am - -./tools/src/h5dump/Makefile.am -./tools/src/h5dump/h5dump.c -./tools/src/h5dump/h5dump.h -./tools/src/h5dump/h5dump_defines.h -./tools/src/h5dump/h5dump_extern.h -./tools/src/h5dump/h5dump_ddl.c -./tools/src/h5dump/h5dump_ddl.h -./tools/src/h5dump/h5dump_xml.c -./tools/src/h5dump/h5dump_xml.h -./tools/test/h5dump/Makefile.am -./tools/test/h5dump/dynlib_dump.c -./tools/test/h5dump/h5dumpgentest.c -./tools/test/h5dump/h5dump_plugin.sh.in -./tools/test/h5dump/testh5dump.sh.in -./tools/test/h5dump/testh5dumppbits.sh.in -./tools/test/h5dump/testh5dumpxml.sh.in -./tools/test/h5dump/testh5dumpvds.sh.in -./tools/test/h5dump/binread.c - -./tools/src/h5import/Makefile.am -./tools/src/h5import/h5import.h -./tools/src/h5import/h5import.c -./tools/test/h5import/Makefile.am -./tools/test/h5import/h5importtest.c -./tools/test/h5import/h5importtestutil.sh.in - -# testfiles for h5import -./tools/test/h5import/testfiles/binfp64.h5 -./tools/test/h5import/testfiles/binin16.h5 -./tools/test/h5import/testfiles/binin32.h5 -./tools/test/h5import/testfiles/binin8.h5 -./tools/test/h5import/testfiles/binin8w.h5 -./tools/test/h5import/testfiles/binuin16.h5 -./tools/test/h5import/testfiles/binuin32.h5 -./tools/test/h5import/testfiles/tall_fp32.ddl -./tools/test/h5import/testfiles/tall_i32.ddl -./tools/test/h5import/testfiles/tintsattrs_u32.ddl -./tools/test/h5import/testfiles/textpfe.conf -./tools/test/h5import/testfiles/textpfe.h5 -./tools/test/h5import/testfiles/textpfe64.txt -./tools/test/h5import/testfiles/txtfp32.conf -./tools/test/h5import/testfiles/txtfp32.h5 -./tools/test/h5import/testfiles/txtfp32.txt -./tools/test/h5import/testfiles/txtfp64.conf -./tools/test/h5import/testfiles/txtfp64.h5 -./tools/test/h5import/testfiles/txtfp64.txt -./tools/test/h5import/testfiles/txtin16.conf -./tools/test/h5import/testfiles/txtin16.h5 -./tools/test/h5import/testfiles/txtin16.txt -./tools/test/h5import/testfiles/txtin32.conf -./tools/test/h5import/testfiles/txtin32.h5 -./tools/test/h5import/testfiles/txtin32.txt -./tools/test/h5import/testfiles/txtin8.conf -./tools/test/h5import/testfiles/txtin8.h5 -./tools/test/h5import/testfiles/txtin8.txt -./tools/test/h5import/testfiles/txtuin16.conf -./tools/test/h5import/testfiles/txtuin16.h5 -./tools/test/h5import/testfiles/txtuin16.txt -./tools/test/h5import/testfiles/txtuin32.conf -./tools/test/h5import/testfiles/txtuin32.h5 -./tools/test/h5import/testfiles/txtuin32.txt -./tools/test/h5import/testfiles/txtstr.conf -./tools/test/h5import/testfiles/txtstr.h5 -./tools/test/h5import/testfiles/txtstr.txt -./tools/test/h5import/testfiles/dbinfp64.h5.txt -./tools/test/h5import/testfiles/dbinin8.h5.txt -./tools/test/h5import/testfiles/dbinin8w.h5.txt -./tools/test/h5import/testfiles/dbinin16.h5.txt -./tools/test/h5import/testfiles/dbinin32.h5.txt -./tools/test/h5import/testfiles/dbinuin16.h5.txt -./tools/test/h5import/testfiles/dbinuin32.h5.txt -./tools/test/h5import/testfiles/dtxtstr.h5.txt - -# h5diff sources -./tools/src/h5diff/Makefile.am -./tools/src/h5diff/h5diff_common.c -./tools/src/h5diff/h5diff_common.h -./tools/src/h5diff/h5diff_main.c -./tools/src/h5diff/ph5diff_main.c -./tools/test/h5diff/Makefile.am -./tools/test/h5diff/dynlib_diff.c -./tools/test/h5diff/h5diffgentest.c -./tools/test/h5diff/h5diff_plugin.sh.in -./tools/test/h5diff/testh5diff.sh.in -./tools/test/h5diff/testph5diff.sh.in - -# h5format_convert sources -./tools/src/h5format_convert/Makefile.am -./tools/src/h5format_convert/h5format_convert.c -./tools/test/h5format_convert/Makefile.am -./tools/test/h5format_convert/h5fc_chk_idx.c -./tools/test/h5format_convert/h5fc_gentest.c -./tools/test/h5format_convert/testfiles/h5fc_v_n_all.ddl -./tools/test/h5format_convert/testfiles/h5fc_v_bt1.ddl -./tools/test/h5format_convert/testfiles/h5fc_v_err.ddl -./tools/test/h5format_convert/testfiles/h5fc_v_err.ddl.err -./tools/test/h5format_convert/testfiles/h5fc_v_non_chunked.ddl -./tools/test/h5format_convert/testfiles/h5fc_d_file.ddl -./tools/test/h5format_convert/testfiles/h5fc_d_file.ddl.err -./tools/test/h5format_convert/testfiles/h5fc_d_file-d.ddl -./tools/test/h5format_convert/testfiles/h5fc_v_ndata_bt1.ddl -./tools/test/h5format_convert/testfiles/h5fc_dname.ddl -./tools/test/h5format_convert/testfiles/h5fc_dname.err -./tools/test/h5format_convert/testfiles/h5fc_nonexistfile.ddl.err -./tools/test/h5format_convert/testfiles/h5fc_nonexistdset_file.ddl.err -./tools/test/h5format_convert/testfiles/h5fc_help.ddl -./tools/test/h5format_convert/testfiles/h5fc_v_all.ddl -./tools/test/h5format_convert/testfiles/h5fc_nooption.ddl -./tools/test/h5format_convert/testfiles/h5fc_v_n_1d.ddl -./tools/test/h5format_convert/testfiles/h5fc_non_v3.h5 -./tools/test/h5format_convert/testfiles/h5fc_edge_v3.h5 -./tools/test/h5format_convert/testfiles/h5fc_err_level.h5 -./tools/test/h5format_convert/testfiles/h5fc_ext1_f.h5 -./tools/test/h5format_convert/testfiles/h5fc_ext1_i.h5 -./tools/test/h5format_convert/testfiles/h5fc_ext1_s.h5 -./tools/test/h5format_convert/testfiles/h5fc_ext2_if.h5 -./tools/test/h5format_convert/testfiles/h5fc_ext2_is.h5 -./tools/test/h5format_convert/testfiles/h5fc_ext2_sf.h5 -./tools/test/h5format_convert/testfiles/h5fc_ext3_isf.h5 -./tools/test/h5format_convert/testfiles/h5fc_ext_none.h5 -./tools/test/h5format_convert/testfiles/old_h5fc_ext1_f.h5 -./tools/test/h5format_convert/testfiles/old_h5fc_ext1_i.h5 -./tools/test/h5format_convert/testfiles/old_h5fc_ext1_s.h5 -./tools/test/h5format_convert/testfiles/old_h5fc_ext2_if.h5 -./tools/test/h5format_convert/testfiles/old_h5fc_ext2_is.h5 -./tools/test/h5format_convert/testfiles/old_h5fc_ext2_sf.h5 -./tools/test/h5format_convert/testfiles/old_h5fc_ext3_isf.h5 -./tools/test/h5format_convert/testfiles/old_h5fc_ext_none.h5 -./tools/test/h5format_convert/testfiles/h5fc_ext1_f.ddl -./tools/test/h5format_convert/testfiles/h5fc_ext1_i.ddl -./tools/test/h5format_convert/testfiles/h5fc_ext1_s.ddl -./tools/test/h5format_convert/testfiles/h5fc_ext2_if.ddl -./tools/test/h5format_convert/testfiles/h5fc_ext2_is.ddl -./tools/test/h5format_convert/testfiles/h5fc_ext2_sf.ddl -./tools/test/h5format_convert/testfiles/h5fc_ext3_isf.ddl -./tools/test/h5format_convert/testfiles/old_h5fc_ext1_f.ddl -./tools/test/h5format_convert/testfiles/old_h5fc_ext1_i.ddl -./tools/test/h5format_convert/testfiles/old_h5fc_ext1_s.ddl -./tools/test/h5format_convert/testfiles/old_h5fc_ext2_if.ddl -./tools/test/h5format_convert/testfiles/old_h5fc_ext2_is.ddl -./tools/test/h5format_convert/testfiles/old_h5fc_ext2_sf.ddl -./tools/test/h5format_convert/testfiles/old_h5fc_ext3_isf.ddl -./tools/test/h5format_convert/testh5fc.sh.in - -# h5repack sources -./tools/src/h5repack/Makefile.am -./tools/src/h5repack/h5repack.c -./tools/src/h5repack/h5repack.h -./tools/src/h5repack/h5repack_copy.c -./tools/src/h5repack/h5repack_refs.c -./tools/src/h5repack/h5repack_filters.c -./tools/src/h5repack/h5repack_main.c -./tools/src/h5repack/h5repack_opttable.c -./tools/src/h5repack/h5repack_parse.c -./tools/src/h5repack/h5repack_verify.c -./tools/test/h5repack/Makefile.am -./tools/test/h5repack/dynlib_rpk.c -./tools/test/h5repack/dynlib_vrpk.c -./tools/test/h5repack/h5repack.sh.in -./tools/test/h5repack/h5repackgentest.c -./tools/test/h5repack/h5repack_plugin.sh.in -./tools/test/h5repack/h5repacktst.c -./tools/test/h5repack/testh5repack_detect_szip.c - -# h5ls sources -./tools/src/h5ls/Makefile.am -./tools/src/h5ls/h5ls.c -./tools/test/h5ls/Makefile.am -./tools/test/h5ls/dynlib_ls.c -./tools/test/h5ls/h5ls_plugin.sh.in -./tools/test/h5ls/testh5ls.sh.in -./tools/test/h5ls/testh5lsvds.sh.in -./tools/test/h5ls/vds_prefix/tvds-1.ls -./tools/test/h5ls/vds_prefix/tvds-2.ls -./tools/test/h5ls/vds_prefix/tvds-3_1.ls -./tools/test/h5ls/vds_prefix/tvds-3_2.ls -./tools/test/h5ls/vds_prefix/tvds-4.ls -./tools/test/h5ls/vds_prefix/tvds-5.ls - -# h5ls test error files -./tools/test/h5ls/errfiles/nosuchfile.err -./tools/test/h5ls/errfiles/textlinksrc-nodangle-1.err -./tools/test/h5ls/errfiles/tgroup-1.err - -# h5copy sources -./tools/src/h5copy/Makefile.am -./tools/src/h5copy/h5copy.c -./tools/test/h5copy/Makefile.am -./tools/test/h5copy/h5copygentest.c -./tools/test/h5copy/testh5copy.sh.in -./tools/test/h5copy/dynlib_copy.c - - -./tools/lib/Makefile.am -./tools/lib/h5diff.c -./tools/lib/h5diff.h -./tools/lib/h5diff_array.c -./tools/lib/h5diff_attr.c -./tools/lib/h5diff_dset.c -./tools/lib/h5diff_util.c -./tools/lib/h5trav.c -./tools/lib/h5trav.h -./tools/lib/h5tools.c -./tools/lib/h5tools.h -./tools/lib/h5tools_dump.c -./tools/lib/h5tools_dump.h -./tools/lib/h5tools_filters.c -./tools/lib/h5tools_str.c -./tools/lib/h5tools_str.h -./tools/lib/h5tools_utils.c -./tools/lib/h5tools_utils.h -./tools/lib/h5tools_ref.c -./tools/lib/h5tools_ref.h -./tools/lib/h5tools_type.c -./tools/lib/ph5diff.h -./tools/lib/h5tools_error.h -./tools/lib/io_timer.c -./tools/lib/io_timer.h - -./tools/libtest/Makefile.am -./tools/libtest/h5tools_test_utils.c - -./tools/src/misc/Makefile.am -./tools/src/misc/h5clear.c -./tools/src/misc/h5debug.c -./tools/src/misc/h5delete.c -./tools/src/misc/h5mkgrp.c -./tools/src/misc/h5repart.c -./tools/test/misc/Makefile.am -./tools/test/misc/h5repart_gentest.c -./tools/test/misc/repart_test.c -./tools/test/misc/testh5mkgrp.sh.in -./tools/test/misc/testh5repart.sh.in -./tools/test/misc/talign.c -./tools/test/misc/testfiles/h5clear_equal_after_size.ddl -./tools/test/misc/testfiles/h5clear_equal_before_size.ddl -./tools/test/misc/testfiles/h5clear_greater_after_size.ddl -./tools/test/misc/testfiles/h5clear_greater_before_size.ddl -./tools/test/misc/testfiles/h5clear_less_after_size.ddl -./tools/test/misc/testfiles/h5clear_less_before_size.ddl -./tools/test/misc/testfiles/h5clear_missing_file.ddl -./tools/test/misc/testfiles/h5clear_missing_file.err -./tools/test/misc/testfiles/h5clear_noclose_after_size.ddl -./tools/test/misc/testfiles/h5clear_noclose_before_size.ddl -./tools/test/misc/testfiles/h5clear_no_mdc_image.err -./tools/test/misc/testfiles/h5clear_open_fail.err -./tools/test/misc/testfiles/h5clear_status_noclose_after_size.ddl -./tools/test/misc/testfiles/h5clear_usage.ddl -./tools/test/misc/testfiles/h5clear_user_equal_after_size.ddl -./tools/test/misc/testfiles/h5clear_user_equal_before_size.ddl -./tools/test/misc/testfiles/h5clear_user_greater_after_size.ddl -./tools/test/misc/testfiles/h5clear_user_greater_before_size.ddl -./tools/test/misc/testfiles/h5clear_user_less_after_size.ddl -./tools/test/misc/testfiles/h5clear_user_less_before_size.ddl -./tools/test/misc/testfiles/h5clear_fsm_persist_equal.h5 -./tools/test/misc/testfiles/h5clear_fsm_persist_greater.h5 -./tools/test/misc/testfiles/h5clear_fsm_persist_less.h5 -./tools/test/misc/testfiles/h5clear_fsm_persist_noclose.h5 -./tools/test/misc/testfiles/h5clear_fsm_persist_user_equal.h5 -./tools/test/misc/testfiles/h5clear_fsm_persist_user_greater.h5 -./tools/test/misc/testfiles/h5clear_fsm_persist_user_less.h5 -./tools/test/misc/testfiles/h5clear_log_v3.h5 -./tools/test/misc/testfiles/h5clear_mdc_image.h5 -./tools/test/misc/testfiles/h5clear_sec2_v0.h5 -./tools/test/misc/testfiles/h5clear_sec2_v2.h5 -./tools/test/misc/testfiles/h5clear_sec2_v3.h5 -./tools/test/misc/testfiles/h5clear_status_noclose.h5 -./tools/test/misc/testfiles/latest_h5clear_log_v3.h5 -./tools/test/misc/testfiles/latest_h5clear_sec2_v3.h5 -./tools/test/misc/testfiles/mod_h5clear_mdc_image.h5 -./tools/test/misc/testfiles/h5mkgrp_help.txt -./tools/test/misc/testfiles/h5mkgrp_version.txt.in -./tools/test/misc/h5perf_gentest.c -./tools/test/misc/vds/Makefile.am -./tools/test/misc/vds/UC_1.h -./tools/test/misc/vds/UC_1_one_dim_gen.c -./tools/test/misc/vds/UC_2.h -./tools/test/misc/vds/UC_2_two_dims_gen.c -./tools/test/misc/vds/UC_3.h -./tools/test/misc/vds/UC_3_gaps_gen.c -./tools/test/misc/vds/UC_4.h -./tools/test/misc/vds/UC_4_printf_gen.c -./tools/test/misc/vds/UC_5.h -./tools/test/misc/vds/UC_5_stride_gen.c -./tools/test/misc/vds/UC_common.h -./tools/test/misc/h5clear_gentest.c -./tools/test/misc/clear_open_chk.c -./tools/test/misc/testh5clear.sh.in - -# h5stat sources -./tools/src/h5stat/Makefile.am -./tools/src/h5stat/h5stat.c - -# h5stat test files -./tools/test/h5stat/Makefile.am -./tools/test/h5stat/h5stat_gentest.c -./tools/test/h5stat/testh5stat.sh.in -./tools/test/h5stat/testfiles/h5stat_dims1.ddl -./tools/test/h5stat/testfiles/h5stat_dims2.ddl -./tools/test/h5stat/testfiles/h5stat_err_old_fill.h5 -./tools/test/h5stat/testfiles/h5stat_err_old_fill.ddl -./tools/test/h5stat/testfiles/h5stat_err_old_layout.h5 -./tools/test/h5stat/testfiles/h5stat_err_old_layout.ddl -./tools/test/h5stat/testfiles/h5stat_err_refcount.h5 -./tools/test/h5stat/testfiles/h5stat_err_refcount.ddl -./tools/test/h5stat/testfiles/h5stat_filters.ddl -./tools/test/h5stat/testfiles/h5stat_filters-d.ddl -./tools/test/h5stat/testfiles/h5stat_filters-dT.ddl -./tools/test/h5stat/testfiles/h5stat_filters-F.ddl -./tools/test/h5stat/testfiles/h5stat_filters-file.ddl -./tools/test/h5stat/testfiles/h5stat_filters-g.ddl -./tools/test/h5stat/testfiles/h5stat_filters-UD.ddl -./tools/test/h5stat/testfiles/h5stat_filters-UT.ddl -./tools/test/h5stat/testfiles/h5stat_filters.h5 -./tools/test/h5stat/testfiles/h5stat_help1.ddl -./tools/test/h5stat/testfiles/h5stat_help2.ddl -./tools/test/h5stat/testfiles/h5stat_idx.h5 -./tools/test/h5stat/testfiles/h5stat_idx.ddl -./tools/test/h5stat/testfiles/h5stat_links1.ddl -./tools/test/h5stat/testfiles/h5stat_links2.ddl -./tools/test/h5stat/testfiles/h5stat_links3.ddl -./tools/test/h5stat/testfiles/h5stat_links4.ddl -./tools/test/h5stat/testfiles/h5stat_links5.ddl -./tools/test/h5stat/testfiles/h5stat_newgrat.ddl -./tools/test/h5stat/testfiles/h5stat_newgrat-UA.ddl -./tools/test/h5stat/testfiles/h5stat_newgrat-UG.ddl -./tools/test/h5stat/testfiles/h5stat_newgrat.h5 -./tools/test/h5stat/testfiles/h5stat_nofile.ddl -./tools/test/h5stat/testfiles/h5stat_notexist.ddl -./tools/test/h5stat/testfiles/h5stat_numattrs1.ddl -./tools/test/h5stat/testfiles/h5stat_numattrs2.ddl -./tools/test/h5stat/testfiles/h5stat_numattrs3.ddl -./tools/test/h5stat/testfiles/h5stat_numattrs4.ddl -./tools/test/h5stat/testfiles/h5stat_threshold.h5 -./tools/test/h5stat/testfiles/h5stat_tsohm.ddl -./tools/test/h5stat/testfiles/h5stat_tsohm.h5 -./tools/test/h5stat/testfiles/h5stat_err_refcount.err -./tools/test/h5stat/testfiles/h5stat_err_old_layout.err -./tools/test/h5stat/testfiles/h5stat_err_old_fill.err -./tools/test/h5stat/testfiles/h5stat_err1_dims.err -./tools/test/h5stat/testfiles/h5stat_err1_links.err -./tools/test/h5stat/testfiles/h5stat_err1_numattrs.err -./tools/test/h5stat/testfiles/h5stat_err2_numattrs.err -./tools/test/h5stat/testfiles/h5stat_notexist.err -./tools/test/h5stat/testfiles/h5stat_nofile.err - -# h5dump test files -./tools/testfiles/charsets.h5 -./tools/testfiles/charsets.ddl -./tools/testfiles/err_attr_dspace.h5 -./tools/testfiles/err_attr_dspace.ddl -./tools/testfiles/family_file00000.h5 -./tools/testfiles/family_file00001.h5 -./tools/testfiles/family_file00002.h5 -./tools/testfiles/family_file00003.h5 -./tools/testfiles/family_file00004.h5 -./tools/testfiles/family_file00005.h5 -./tools/testfiles/family_file00006.h5 -./tools/testfiles/family_file00007.h5 -./tools/testfiles/family_file00008.h5 -./tools/testfiles/family_file00009.h5 -./tools/testfiles/family_file00010.h5 -./tools/testfiles/family_file00011.h5 -./tools/testfiles/family_file00012.h5 -./tools/testfiles/family_file00013.h5 -./tools/testfiles/family_file00014.h5 -./tools/testfiles/family_file00015.h5 -./tools/testfiles/family_file00016.h5 -./tools/testfiles/family_file00017.h5 -./tools/testfiles/file_space.h5 -./tools/testfiles/file_space.ddl -./tools/testfiles/filter_fail.h5 -./tools/testfiles/filter_fail.ddl -./tools/testfiles/h5dump-help.txt -./tools/testfiles/non_existing.ddl -./tools/testfiles/packedbits.ddl -./tools/testfiles/t128bit_float.h5 -./tools/testfiles/taindices.h5 -./tools/testfiles/tall-1.ddl -./tools/testfiles/tall-2.ddl -./tools/testfiles/tall-2A.ddl -./tools/testfiles/tall-2A0.ddl -./tools/testfiles/tall-2B.ddl -./tools/testfiles/tall-3.ddl -./tools/testfiles/tall-4s.ddl -./tools/testfiles/tall-5s.ddl -./tools/testfiles/tall-6.ddl -./tools/testfiles/tall-6.exp -./tools/testfiles/tall-7.ddl -./tools/testfiles/tall-7N.ddl -./tools/testfiles/tall.h5 -./tools/testfiles/tallfilters.ddl -./tools/testfiles/tarray1.ddl -./tools/testfiles/tarray1.h5 -./tools/testfiles/tarray1_big.ddl -./tools/testfiles/tarray1_big.h5 -./tools/testfiles/tarray2.ddl -./tools/testfiles/tarray2.h5 -./tools/testfiles/tarray3.ddl -./tools/testfiles/tarray3.h5 -./tools/testfiles/tarray4.ddl -./tools/testfiles/tarray4.h5 -./tools/testfiles/tarray5.ddl -./tools/testfiles/tarray5.h5 -./tools/testfiles/tarray6.ddl -./tools/testfiles/tarray6.h5 -./tools/testfiles/tarray7.ddl -./tools/testfiles/tarray7.h5 -./tools/testfiles/tarray8.ddl -./tools/testfiles/tarray8.h5 -./tools/testfiles/tattr.h5 -./tools/testfiles/tattr-1.ddl -./tools/testfiles/tattr-2.ddl -./tools/testfiles/tattr-3.ddl -./tools/testfiles/tattr-4_be.ddl -./tools/testfiles/tattr2.h5 -./tools/testfiles/tattr4_be.h5 -./tools/testfiles/tattrcontents1.ddl -./tools/testfiles/tattrcontents2.ddl -./tools/testfiles/tattrintsize.ddl -./tools/testfiles/tattrintsize.h5 -./tools/testfiles/tattrreg.h5 -./tools/testfiles/tattrreg.ddl -./tools/testfiles/tattrregR.ddl -./tools/testfiles/tbigdims.ddl -./tools/testfiles/tbigdims.h5 -./tools/testfiles/tbinary.h5 -./tools/testfiles/tbin1.ddl -./tools/testfiles/tbin2.ddl -./tools/testfiles/tbin3.ddl -./tools/testfiles/tbin4.ddl -./tools/testfiles/tbinregR.exp -./tools/testfiles/tbinregR.ddl -./tools/testfiles/tbitfields.h5 -./tools/testfiles/tbitnopaque_be.ddl -./tools/testfiles/tbitnopaque_le.ddl -./tools/testfiles/tbitnopaque.h5 -./tools/testfiles/tboot1.ddl -./tools/testfiles/tboot2.ddl -./tools/testfiles/tboot2A.ddl -./tools/testfiles/tboot2B.ddl -./tools/testfiles/tchar.h5 -./tools/testfiles/tchar1.ddl -./tools/testfiles/tchunked.ddl -./tools/testfiles/tcompact.ddl -./tools/testfiles/tcontents.ddl -./tools/testfiles/tcontiguos.ddl -./tools/testfiles/tcmpdattrintsize.ddl -./tools/testfiles/tcmpdintarray.ddl -./tools/testfiles/tcmpdints.ddl -./tools/testfiles/tcmpdintsize.ddl -./tools/testfiles/tcmpdattrintsize.h5 -./tools/testfiles/tcmpdintarray.h5 -./tools/testfiles/tcmpdints.h5 -./tools/testfiles/tcmpdintsize.h5 -./tools/testfiles/tcomp-1.ddl -./tools/testfiles/tcomp-2.ddl -./tools/testfiles/tcomp-3.ddl -./tools/testfiles/tcomp-4.ddl -./tools/testfiles/tcompound.h5 -./tools/testfiles/tcompound2.h5 -./tools/testfiles/tcompound_complex.h5 -./tools/testfiles/tcompound_complex2.h5 -./tools/testfiles/tcompound_complex2.ddl -./tools/testfiles/tdatareg.h5 -./tools/testfiles/tdatareg.ddl -./tools/testfiles/tdataregR.ddl -./tools/testfiles/tdeflate.ddl -./tools/testfiles/tdset-1.ddl -./tools/testfiles/tdset-2.ddl -./tools/testfiles/tdset-3s.ddl -./tools/testfiles/tdset.h5 -./tools/testfiles/tdset2.h5 -./tools/testfiles/tdset_idx.ls -./tools/testfiles/tdset_idx.h5 -./tools/testfiles/tempty.ddl -./tools/testfiles/tempty.h5 -./tools/testfiles/tenum.h5 -./tools/testfiles/texceedsubblock.ddl -./tools/testfiles/texceedsubcount.ddl -./tools/testfiles/texceedsubstart.ddl -./tools/testfiles/texceedsubstride.ddl -./tools/testfiles/texternal.ddl -./tools/testfiles/textlink.h5 -./tools/testfiles/textlink.h5.xml -./tools/testfiles/textlink.ddl -./tools/testfiles/textlinkfar.ddl -./tools/testfiles/textlinkfar.h5 -./tools/testfiles/textlinksrc.ddl -./tools/testfiles/textlinksrc.h5 -./tools/testfiles/textlinktar.h5 -./tools/testfiles/tfamily.ddl -./tools/testfiles/tfamily00000.h5 -./tools/testfiles/tfamily00001.h5 -./tools/testfiles/tfamily00002.h5 -./tools/testfiles/tfamily00003.h5 -./tools/testfiles/tfamily00004.h5 -./tools/testfiles/tfamily00005.h5 -./tools/testfiles/tfamily00006.h5 -./tools/testfiles/tfamily00007.h5 -./tools/testfiles/tfamily00008.h5 -./tools/testfiles/tfamily00009.h5 -./tools/testfiles/tfamily00010.h5 -./tools/testfiles/tfcontents1.h5 -./tools/testfiles/tfcontents2.h5 -./tools/testfiles/tfill.ddl -./tools/testfiles/tfilters.h5 -./tools/testfiles/tfletcher32.ddl -./tools/testfiles/tfloatsattrs.ddl -./tools/testfiles/tfloatsattrs.h5 -./tools/testfiles/tfloatsattrs.wddl -./tools/testfiles/tfvalues.h5 -./tools/testfiles/tgroup-1.ddl -./tools/testfiles/tgroup-2.ddl -./tools/testfiles/tgroup.h5 -./tools/testfiles/tgrp_comments.ls -./tools/testfiles/tgrp_comments.ddl -./tools/testfiles/tgrp_comments.h5 -./tools/testfiles/tgrpnullspace.h5 -./tools/testfiles/tgrpnullspace.ddl -./tools/testfiles/tgrpnullspace.ls -./tools/testfiles/thlink-1.ddl -./tools/testfiles/thlink-2.ddl -./tools/testfiles/thlink-3.ddl -./tools/testfiles/thlink-4.ddl -./tools/testfiles/thlink-5.ddl -./tools/testfiles/thlink.h5 -./tools/testfiles/thyperslab.ddl -./tools/testfiles/thyperslab.h5 -./tools/testfiles/tindicesyes.ddl -./tools/testfiles/tindicesno.ddl -./tools/testfiles/tindicessub2.ddl -./tools/testfiles/tindicessub3.ddl -./tools/testfiles/tindicessub4.ddl -./tools/testfiles/tindicessub1.ddl -./tools/testfiles/tints4dims.ddl -./tools/testfiles/tints4dimsBlock2.ddl -./tools/testfiles/tints4dimsBlockEq.ddl -./tools/testfiles/tints4dimsCount2.ddl -./tools/testfiles/tints4dimsCountEq.ddl -./tools/testfiles/tints4dimsStride2.ddl -./tools/testfiles/tints4dims.h5 -./tools/testfiles/tintsattrs.ddl -./tools/testfiles/tintsattrs.h5 -./tools/testfiles/tintsnodata.ddl -./tools/testfiles/tintsnodata.h5 -./tools/testfiles/tlarge_objname.ddl -./tools/testfiles/tlarge_objname.h5 -./tools/testfiles/tldouble.ddl -./tools/testfiles/tldouble.h5 -./tools/testfiles/tldouble_scalar.ddl -./tools/testfiles/tldouble_scalar.h5 -./tools/testfiles/tlonglinks.ddl -./tools/testfiles/tlonglinks.h5 -./tools/testfiles/tloop-1.ddl -./tools/testfiles/tloop.h5 -./tools/testfiles/tloop2.h5 -./tools/testfiles/tmany.h5 -./tools/testfiles/tmulti-b.h5 -./tools/testfiles/tmulti.ddl -./tools/testfiles/tmulti-g.h5 -./tools/testfiles/tmulti-l.h5 -./tools/testfiles/tmulti-o.h5 -./tools/testfiles/tmulti-r.h5 -./tools/testfiles/tmulti-s.h5 -./tools/testfiles/tmultifile.ls -./tools/testfiles/tmultifile.ddl -./tools/testfiles/tnbit.ddl -./tools/testfiles/tnestcomp-1.ddl -./tools/testfiles/tnestedcomp.h5 -./tools/testfiles/tnestedcmpddt.ddl -./tools/testfiles/tnestedcmpddt.h5 -./tools/testfiles/tnoattrdata.ddl -./tools/testfiles/tnoattrddl.ddl -./tools/testfiles/tnodata.ddl -./tools/testfiles/tnoddl.ddl -./tools/testfiles/tnoddlfile.ddl -./tools/testfiles/tnoddlfile.exp -./tools/testfiles/tno-subset.h5 -./tools/testfiles/tno-subset.ddl -./tools/testfiles/tnullspace.h5 -./tools/testfiles/tnullspace.h5.xml -./tools/testfiles/tnullspace.ddl -./tools/testfiles/tobjref.h5 -./tools/testfiles/topaque.h5 -./tools/testfiles/tordercontents1.ddl -./tools/testfiles/tordercontents2.ddl -./tools/testfiles/torderlinks1.ddl -./tools/testfiles/torderlinks2.ddl -./tools/testfiles/tperror.ddl -./tools/testfiles/tqmarkfile.ddl -./tools/testfiles/trawdatafile.ddl -./tools/testfiles/trawdatafile.exp -./tools/testfiles/trawssetfile.ddl -./tools/testfiles/trawssetfile.exp -./tools/testfiles/treadfilter.ddl -./tools/testfiles/treadintfilter.ddl -./tools/testfiles/treference.ddl -./tools/testfiles/tsaf.ddl -./tools/testfiles/tsaf.h5 -./tools/testfiles/tscalarattrintsize.ddl -./tools/testfiles/tscalarattrintsize.h5 -./tools/testfiles/tscalarintattrsize.ddl -./tools/testfiles/tscalarintattrsize.h5 -./tools/testfiles/tscalarintsize.ddl -./tools/testfiles/tscalarintsize.h5 -./tools/testfiles/tscalarstring.ddl -./tools/testfiles/tscalarstring.h5 -./tools/testfiles/tscaleoffset.ddl -./tools/testfiles/tshuffle.ddl -./tools/testfiles/tslink-1.ddl -./tools/testfiles/tslink-2.ddl -./tools/testfiles/tslink-D.ddl -./tools/testfiles/tslink.h5 -./tools/testfiles/tsoftlinks.h5 -./tools/testfiles/tsplit_file-m.h5 -./tools/testfiles/tsplit_file-r.h5 -./tools/testfiles/tsplit_file.ddl -./tools/testfiles/tstarfile.ddl -./tools/testfiles/tstr.h5 -./tools/testfiles/tstr2.h5 -./tools/testfiles/tstr3.h5 -./tools/testfiles/tstr-1.ddl -./tools/testfiles/tstr-2.ddl -./tools/testfiles/tstr2bin2.exp -./tools/testfiles/tstr2bin6.exp -./tools/testfiles/tstring.ddl -./tools/testfiles/tstring2.ddl -./tools/testfiles/tstringe.ddl -./tools/testfiles/tszip.ddl -./tools/testfiles/tudfilter.ddl -./tools/testfiles/tudfilter.h5 -./tools/testfiles/tudfilter.ls -./tools/testfiles/tudlink.h5 -./tools/testfiles/tudlink.h5.xml -./tools/testfiles/tudlink-1.ddl -./tools/testfiles/tudlink-2.ddl -./tools/testfiles/tuserfilter.ddl -./tools/testfiles/tvldtypes1.ddl -./tools/testfiles/tvldtypes1.h5 -./tools/testfiles/tvldtypes2.ddl -./tools/testfiles/tvldtypes2.h5 -./tools/testfiles/tvldtypes3.ddl -./tools/testfiles/tvldtypes3.h5 -./tools/testfiles/tvldtypes4.ddl -./tools/testfiles/tvldtypes4.h5 -./tools/testfiles/tvldtypes5.ddl -./tools/testfiles/tvldtypes5.h5 -./tools/testfiles/tvlenstr_array.ddl -./tools/testfiles/tvlenstr_array.h5 -./tools/testfiles/tvlstr.h5 -./tools/testfiles/tvlstr.ddl -./tools/testfiles/tvms.ddl -./tools/testfiles/tvms.h5 -./tools/testfiles/twidedisplay.ddl -./tools/testfiles/twithddl.exp -./tools/testfiles/twithddlfile.ddl -./tools/testfiles/twithddlfile.exp -./tools/testfiles/tCVE_2018_11206_fill_old.h5 -./tools/testfiles/tCVE_2018_11206_fill_new.h5 - -# h5dump test error files -./tools/test/h5dump/errfiles/filter_fail.err -./tools/test/h5dump/errfiles/non_existing.err -./tools/test/h5dump/errfiles/tall-1.err -./tools/test/h5dump/errfiles/tall-2A.err -./tools/test/h5dump/errfiles/tall-2A0.err -./tools/test/h5dump/errfiles/tall-2B.err -./tools/test/h5dump/errfiles/tarray1_big.err -./tools/test/h5dump/errfiles/tattr-3.err -./tools/test/h5dump/errfiles/tattrregR.err -./tools/test/h5dump/errfiles/tcomp-3.err -./tools/test/h5dump/errfiles/tdataregR.err -./tools/test/h5dump/errfiles/tdset-2.err -./tools/test/h5dump/errfiles/texceedsubblock.err -./tools/test/h5dump/errfiles/texceedsubcount.err -./tools/test/h5dump/errfiles/texceedsubstart.err -./tools/test/h5dump/errfiles/texceedsubstride.err -./tools/test/h5dump/errfiles/textlink.err -./tools/test/h5dump/errfiles/textlinkfar.err -./tools/test/h5dump/errfiles/textlinksrc.err -./tools/test/h5dump/errfiles/tgroup-2.err -./tools/test/h5dump/errfiles/tnofilename-with-packed-bits.err -./tools/test/h5dump/errfiles/torderlinks1.err -./tools/test/h5dump/errfiles/torderlinks2.err -./tools/test/h5dump/errfiles/tpbitsCharLengthExceeded.err -./tools/test/h5dump/errfiles/tpbitsCharOffsetExceeded.err -./tools/test/h5dump/errfiles/tpbitsIncomplete.err -./tools/test/h5dump/errfiles/tpbitsIntLengthExceeded.err -./tools/test/h5dump/errfiles/tpbitsIntOffsetExceeded.err -./tools/test/h5dump/errfiles/tpbitsLengthExceeded.err -./tools/test/h5dump/errfiles/tpbitsLengthPositive.err -./tools/test/h5dump/errfiles/tpbitsLongLengthExceeded.err -./tools/test/h5dump/errfiles/tpbitsLongOffsetExceeded.err -./tools/test/h5dump/errfiles/tpbitsMaxExceeded.err -./tools/test/h5dump/errfiles/tpbitsOffsetExceeded.err -./tools/test/h5dump/errfiles/tpbitsOffsetNegative.err -./tools/test/h5dump/errfiles/tperror.err -./tools/test/h5dump/errfiles/tqmarkfile.err -./tools/test/h5dump/errfiles/tslink-D.err - -# h5dump packed bits validation -./tools/testfiles/pbits/tnofilename-with-packed-bits.ddl -./tools/testfiles/pbits/tpbitsLengthPositive.ddl -./tools/testfiles/pbits/tpbitsMaxExceeded.ddl -./tools/testfiles/pbits/tpbitsSigned.ddl -./tools/testfiles/pbits/tpbitsSigned2.ddl -./tools/testfiles/pbits/tpbitsSigned4.ddl -./tools/testfiles/pbits/tpbitsSignedWhole.ddl -./tools/testfiles/pbits/tpbitsSignedInt.ddl -./tools/testfiles/pbits/tpbitsSignedInt4.ddl -./tools/testfiles/pbits/tpbitsSignedInt8.ddl -./tools/testfiles/pbits/tpbitsSignedIntWhole.ddl -./tools/testfiles/pbits/tpbitsSignedLong.ddl -./tools/testfiles/pbits/tpbitsSignedLong8.ddl -./tools/testfiles/pbits/tpbitsSignedLong16.ddl -./tools/testfiles/pbits/tpbitsSignedLongWhole.ddl -./tools/testfiles/pbits/tpbitsSignedLongLong.ddl -./tools/testfiles/pbits/tpbitsSignedLongLong16.ddl -./tools/testfiles/pbits/tpbitsSignedLongLong32.ddl -./tools/testfiles/pbits/tpbitsSignedLongLongWhole.ddl -./tools/testfiles/pbits/tpbitsSignedLongLongWhole1.ddl -./tools/testfiles/pbits/tpbitsSignedLongLongWhole63.ddl -./tools/testfiles/pbits/tpbitsOffsetNegative.ddl -./tools/testfiles/pbits/tpbitsUnsigned.ddl -./tools/testfiles/pbits/tpbitsUnsigned2.ddl -./tools/testfiles/pbits/tpbitsUnsigned4.ddl -./tools/testfiles/pbits/tpbitsUnsignedWhole.ddl -./tools/testfiles/pbits/tpbitsUnsignedInt.ddl -./tools/testfiles/pbits/tpbitsUnsignedInt4.ddl -./tools/testfiles/pbits/tpbitsUnsignedInt8.ddl -./tools/testfiles/pbits/tpbitsUnsignedIntWhole.ddl -./tools/testfiles/pbits/tpbitsUnsignedLong.ddl -./tools/testfiles/pbits/tpbitsUnsignedLong8.ddl -./tools/testfiles/pbits/tpbitsUnsignedLong16.ddl -./tools/testfiles/pbits/tpbitsUnsignedLongWhole.ddl -./tools/testfiles/pbits/tpbitsUnsignedLongLong.ddl -./tools/testfiles/pbits/tpbitsUnsignedLongLong16.ddl -./tools/testfiles/pbits/tpbitsUnsignedLongLong32.ddl -./tools/testfiles/pbits/tpbitsUnsignedLongLongWhole.ddl -./tools/testfiles/pbits/tpbitsUnsignedLongLongWhole1.ddl -./tools/testfiles/pbits/tpbitsUnsignedLongLongWhole63.ddl -./tools/testfiles/pbits/tpbitsMax.ddl -./tools/testfiles/pbits/tpbitsArray.ddl -./tools/testfiles/pbits/tpbitsCompound.ddl -./tools/testfiles/pbits/tpbitsIncomplete.ddl -./tools/testfiles/pbits/tpbitsLengthExceeded.ddl -./tools/testfiles/pbits/tpbitsCharLengthExceeded.ddl -./tools/testfiles/pbits/tpbitsIntLengthExceeded.ddl -./tools/testfiles/pbits/tpbitsLongLengthExceeded.ddl -./tools/testfiles/pbits/tpbitsOffsetExceeded.ddl -./tools/testfiles/pbits/tpbitsCharOffsetExceeded.ddl -./tools/testfiles/pbits/tpbitsIntOffsetExceeded.ddl -./tools/testfiles/pbits/tpbitsLongOffsetExceeded.ddl -./tools/testfiles/pbits/tpbitsOverlapped.ddl -./tools/testfiles/packedbits.h5 - -# h5dump vds validation -./tools/testfiles/vds/tvds-1.ddl -./tools/testfiles/vds/tvds-2.ddl -./tools/testfiles/vds/tvds-3_1.ddl -./tools/testfiles/vds/tvds-3_2.ddl -./tools/testfiles/vds/tvds-4.ddl -./tools/testfiles/vds/tvds-5.ddl -./tools/testfiles/vds/tvds_layout-1.ddl -./tools/testfiles/vds/tvds_layout-2.ddl -./tools/testfiles/vds/tvds_layout-3_1.ddl -./tools/testfiles/vds/tvds_layout-3_2.ddl -./tools/testfiles/vds/tvds_layout-4.ddl -./tools/testfiles/vds/tvds_layout-5.ddl -./tools/testfiles/vds/vds-first.ddl -./tools/testfiles/vds/vds-gap1.ddl -./tools/testfiles/vds/vds-gap2.ddl -./tools/testfiles/vds/vds_layout-eiger.ddl -./tools/testfiles/vds/vds_layout-maxmin.ddl -./tools/testfiles/vds/1_a.h5 -./tools/testfiles/vds/1_b.h5 -./tools/testfiles/vds/1_c.h5 -./tools/testfiles/vds/1_d.h5 -./tools/testfiles/vds/1_e.h5 -./tools/testfiles/vds/1_f.h5 -./tools/testfiles/vds/1_vds.h5 -./tools/testfiles/vds/2_a.h5 -./tools/testfiles/vds/2_b.h5 -./tools/testfiles/vds/2_c.h5 -./tools/testfiles/vds/2_d.h5 -./tools/testfiles/vds/2_e.h5 -./tools/testfiles/vds/2_vds.h5 -./tools/testfiles/vds/3_1_vds.h5 -./tools/testfiles/vds/3_2_vds.h5 -./tools/testfiles/vds/4_0.h5 -./tools/testfiles/vds/4_1.h5 -./tools/testfiles/vds/4_2.h5 -./tools/testfiles/vds/4_vds.h5 -./tools/testfiles/vds/5_a.h5 -./tools/testfiles/vds/5_b.h5 -./tools/testfiles/vds/5_c.h5 -./tools/testfiles/vds/5_vds.h5 -./tools/testfiles/vds/a.h5 -./tools/testfiles/vds/b.h5 -./tools/testfiles/vds/c.h5 -./tools/testfiles/vds/d.h5 -./tools/testfiles/vds/vds-percival-unlim-maxmin.h5 -./tools/testfiles/vds/f-0.h5 -./tools/testfiles/vds/f-3.h5 -./tools/testfiles/vds/vds-eiger.h5 - -# h5dump h5import validation -./tools/testfiles/out3.h5import -./tools/testfiles/tordergr.h5 -./tools/testfiles/tordergr3.ddl -./tools/testfiles/tordergr4.ddl -./tools/testfiles/tordergr1.ddl -./tools/testfiles/tordergr5.ddl -./tools/testfiles/tordergr2.ddl -./tools/testfiles/torderattr1.ddl -./tools/testfiles/torderattr2.ddl -./tools/testfiles/torderattr3.ddl -./tools/testfiles/torderattr4.ddl -./tools/testfiles/torderattr.h5 -./tools/testfiles/tfpformat.ddl -./tools/testfiles/tfpformat.h5 - -# h5dump h5repack validation -./tools/testfiles/zerodim.ddl -./tools/testfiles/zerodim.h5 - -# h5dump new reference validation -./tools/testfiles/trefer_attrR.ddl -./tools/testfiles/trefer_compatR.ddl -./tools/testfiles/trefer_extR.ddl -./tools/testfiles/trefer_grpR.ddl -./tools/testfiles/trefer_obj_delR.ddl -./tools/testfiles/trefer_objR.ddl -./tools/testfiles/trefer_paramR.ddl -./tools/testfiles/trefer_reg_1dR.ddl -./tools/testfiles/trefer_regR.ddl -# h5dump and h5diff new reference files -./tools/testfiles/trefer_attr.h5 -./tools/testfiles/trefer_compat.h5 -./tools/testfiles/trefer_ext1.h5 -./tools/testfiles/trefer_ext2.h5 -./tools/testfiles/trefer_grp.h5 -./tools/testfiles/trefer_obj_del.h5 -./tools/testfiles/trefer_obj.h5 -./tools/testfiles/trefer_param.h5 -./tools/testfiles/trefer_reg_1d.h5 -./tools/testfiles/trefer_reg.h5 - -# Expected output from h5ls tests -./tools/testfiles/nosuchfile.ls -./tools/testfiles/help-1.ls -./tools/testfiles/help-2.ls -./tools/testfiles/help-3.ls -./tools/testfiles/tall-1.ls -./tools/testfiles/tall-2.ls -./tools/testfiles/tcomp-1.ls -./tools/testfiles/tdset-1.ls -./tools/testfiles/tgroup-1.ls -./tools/testfiles/tgroup-2.ls -./tools/testfiles/tgroup-3.ls -./tools/testfiles/tgroup.ls -./tools/testfiles/tloop-1.ls -./tools/testfiles/tnestcomp-1.ls -./tools/testfiles/tnestcomp-2.ls -./tools/testfiles/tnestcomp-3.ls -./tools/testfiles/tnestcomp-4.ls -./tools/testfiles/tsaf.ls -./tools/testfiles/tstr-1.ls -./tools/testfiles/tattr2.ls -./tools/testfiles/tattrreg_le.ls -./tools/testfiles/tattrreg_be.ls -./tools/testfiles/tvldtypes1.ls -./tools/testfiles/tvldtypes2le.ls -./tools/testfiles/tvldtypes2be.ls -./tools/testfiles/tdataregle.ls -./tools/testfiles/tdataregbe.ls -./tools/testfiles/tarray1.ls -./tools/testfiles/tempty.ls -./tools/testfiles/thlink-1.ls -./tools/testfiles/tslink-1.ls -./tools/testfiles/textlink-1.ls -./tools/testfiles/textlinksrc-1.ls -./tools/testfiles/textlinksrc-2.ls -./tools/testfiles/textlinksrc-3.ls -./tools/testfiles/textlinksrc-4.ls -./tools/testfiles/textlinksrc-5.ls -./tools/testfiles/textlinksrc-6.ls -./tools/testfiles/textlinksrc-7.ls -./tools/testfiles/textlinksrc-1-old.ls -./tools/testfiles/textlinksrc-2-old.ls -./tools/testfiles/textlinksrc-3-old.ls -./tools/testfiles/textlinksrc-6-old.ls -./tools/testfiles/textlinksrc-7-old.ls -./tools/testfiles/tsoftlinks-1.ls -./tools/testfiles/tsoftlinks-2.ls -./tools/testfiles/tsoftlinks-3.ls -./tools/testfiles/tsoftlinks-4.ls -./tools/testfiles/tsoftlinks-5.ls -./tools/testfiles/textlinksrc-nodangle-1.ls -./tools/testfiles/textlinksrc-nodangle-2.ls -./tools/testfiles/tsoftlinks-nodangle-1.ls -./tools/testfiles/thlinks-nodangle-1.ls -./tools/testfiles/tudlink-1.ls - -# h5ls vds validation -./tools/testfiles/vds/tvds-1.ls -./tools/testfiles/vds/tvds-2.ls -./tools/testfiles/vds/tvds-3_1.ls -./tools/testfiles/vds/tvds-3_2.ls -./tools/testfiles/vds/tvds-4.ls -./tools/testfiles/vds/tvds-5.ls - -#additional test input and output for h5dump XML -./tools/testfiles/tall.h5.xml -./tools/testfiles/tarray1.h5.xml -./tools/testfiles/tarray2.h5.xml -./tools/testfiles/tarray3.h5.xml -./tools/testfiles/tarray6.h5.xml -./tools/testfiles/tarray7.h5.xml -./tools/testfiles/tattr.h5.xml -./tools/testfiles/tbitfields_be.h5.xml -./tools/testfiles/tbitfields_le.h5.xml -./tools/testfiles/tcompound.h5.xml -./tools/testfiles/tcompound2.h5.xml -./tools/testfiles/tcompound_complex.h5.xml -./tools/testfiles/tdatareg.h5.xml -./tools/testfiles/tdset.h5.xml -./tools/testfiles/tdset2.h5.xml -./tools/testfiles/tempty.h5.xml -./tools/testfiles/tenum.h5.xml -./tools/testfiles/test35.nc -./tools/testfiles/test35.nc.xml -./tools/testfiles/tfpformat.h5.xml -./tools/testfiles/tgroup.h5.xml -./tools/testfiles/thlink.h5.xml -./tools/testfiles/tloop.h5.xml -./tools/testfiles/tloop2.h5.xml -./tools/testfiles/tmany.h5.xml -./tools/testfiles/tname-amp.h5 -./tools/testfiles/tname-amp.h5.xml -./tools/testfiles/tname-apos.h5 -./tools/testfiles/tname-apos.h5.xml -./tools/testfiles/tname-gt.h5 -./tools/testfiles/tname-gt.h5.xml -./tools/testfiles/tname-lt.h5 -./tools/testfiles/tname-lt.h5.xml -./tools/testfiles/tname-quot.h5 -./tools/testfiles/tname-quot.h5.xml -./tools/testfiles/tname-sp.h5 -./tools/testfiles/tname-sp.h5.xml -./tools/testfiles/tnamed_dtype_attr.ddl -./tools/testfiles/tnamed_dtype_attr.h5 -./tools/testfiles/tnamed_dtype_attr.h5.xml -./tools/testfiles/tnestedcomp.h5.xml -./tools/testfiles/tnodata.h5 -./tools/testfiles/tnodata.h5.xml -./tools/testfiles/tobjref.h5.xml -./tools/testfiles/topaque.h5.xml -./tools/testfiles/tref-escapes-at.h5 -./tools/testfiles/tref-escapes-at.h5.xml -./tools/testfiles/tref-escapes.h5 -./tools/testfiles/tref-escapes.h5.xml -./tools/testfiles/tref.h5 -./tools/testfiles/tref.h5.xml -./tools/testfiles/tsaf.h5.xml -./tools/testfiles/tslink.h5.xml -./tools/testfiles/tstr.h5.xml -./tools/testfiles/tstr2.h5.xml -./tools/testfiles/tstring-at.h5 -./tools/testfiles/tstring-at.h5.xml -./tools/testfiles/tstring.h5 -./tools/testfiles/tstring.h5.xml -./tools/testfiles/tvldtypes1.h5.xml -./tools/testfiles/tvldtypes2.h5.xml -./tools/testfiles/tvldtypes3.h5.xml -./tools/testfiles/tvldtypes4.h5.xml -./tools/testfiles/tvldtypes5.h5.xml -./tools/testfiles/tvlstr.h5.xml -./tools/testfiles/tempty-dtd.h5.xml -./tools/testfiles/tempty-dtd-uri.h5.xml -./tools/testfiles/tempty-nons.h5.xml -./tools/testfiles/tempty-nons-uri.h5.xml -./tools/testfiles/tempty-ns.h5.xml -./tools/testfiles/tempty-dtd-2.h5.xml -./tools/testfiles/tempty-nons-2.h5.xml -./tools/testfiles/tempty-ns-2.h5.xml -./tools/testfiles/tall-2A.h5.xml -./tools/testfiles/torderattr4.h5.xml -./tools/testfiles/torderattr2.h5.xml -./tools/testfiles/torderattr3.h5.xml -./tools/testfiles/torderattr1.h5.xml - - -#test files for h5diff -./tools/test/h5diff/testfiles/h5diff_10.txt -./tools/test/h5diff/testfiles/h5diff_11.txt -./tools/test/h5diff/testfiles/h5diff_12.txt -./tools/test/h5diff/testfiles/h5diff_13.txt -./tools/test/h5diff/testfiles/h5diff_14.txt -./tools/test/h5diff/testfiles/h5diff_15.txt -./tools/test/h5diff/testfiles/h5diff_16_1.txt -./tools/test/h5diff/testfiles/h5diff_16_2.txt -./tools/test/h5diff/testfiles/h5diff_16_3.txt -./tools/test/h5diff/testfiles/h5diff_17.txt -./tools/test/h5diff/testfiles/h5diff_171.txt -./tools/test/h5diff/testfiles/h5diff_172.txt -./tools/test/h5diff/testfiles/h5diff_18.txt -./tools/test/h5diff/testfiles/h5diff_18_1.txt -./tools/test/h5diff/testfiles/h5diff_19.txt -./tools/test/h5diff/testfiles/h5diff_20.txt -./tools/test/h5diff/testfiles/h5diff_21.txt -./tools/test/h5diff/testfiles/h5diff_22.txt -./tools/test/h5diff/testfiles/h5diff_23.txt -./tools/test/h5diff/testfiles/h5diff_24.txt -./tools/test/h5diff/testfiles/h5diff_25.txt -./tools/test/h5diff/testfiles/h5diff_26.txt -./tools/test/h5diff/testfiles/h5diff_27.txt -./tools/test/h5diff/testfiles/h5diff_28.txt -./tools/test/h5diff/testfiles/h5diff_30.txt -./tools/test/h5diff/testfiles/h5diff_50.txt -./tools/test/h5diff/testfiles/h5diff_51.txt -./tools/test/h5diff/testfiles/h5diff_52.txt -./tools/test/h5diff/testfiles/h5diff_53.txt -./tools/test/h5diff/testfiles/h5diff_54.txt -./tools/test/h5diff/testfiles/h5diff_55.txt -./tools/test/h5diff/testfiles/h5diff_56.txt -./tools/test/h5diff/testfiles/h5diff_57.txt -./tools/test/h5diff/testfiles/h5diff_58.txt -./tools/test/h5diff/testfiles/h5diff_58_ref.txt -./tools/test/h5diff/testfiles/h5diff_59.txt -./tools/test/h5diff/testfiles/h5diff_60.txt -./tools/test/h5diff/testfiles/h5diff_61.txt -./tools/test/h5diff/testfiles/h5diff_62.txt -./tools/test/h5diff/testfiles/h5diff_63.txt -./tools/test/h5diff/testfiles/h5diff_600.txt -./tools/test/h5diff/testfiles/h5diff_601.txt -./tools/test/h5diff/testfiles/h5diff_601_ERR.err -./tools/test/h5diff/testfiles/h5diff_603.txt -./tools/test/h5diff/testfiles/h5diff_604.txt -./tools/test/h5diff/testfiles/h5diff_605.txt -./tools/test/h5diff/testfiles/h5diff_606.txt -./tools/test/h5diff/testfiles/h5diff_607.txt -./tools/test/h5diff/testfiles/h5diff_608.txt -./tools/test/h5diff/testfiles/h5diff_609.txt -./tools/test/h5diff/testfiles/h5diff_610.txt -./tools/test/h5diff/testfiles/h5diff_612.txt -./tools/test/h5diff/testfiles/h5diff_613.txt -./tools/test/h5diff/testfiles/h5diff_614.txt -./tools/test/h5diff/testfiles/h5diff_615.txt -./tools/test/h5diff/testfiles/h5diff_616.txt -./tools/test/h5diff/testfiles/h5diff_617.txt -./tools/test/h5diff/testfiles/h5diff_618.txt -./tools/test/h5diff/testfiles/h5diff_619.txt -./tools/test/h5diff/testfiles/h5diff_621.txt -./tools/test/h5diff/testfiles/h5diff_622.txt -./tools/test/h5diff/testfiles/h5diff_623.txt -./tools/test/h5diff/testfiles/h5diff_624.txt -./tools/test/h5diff/testfiles/h5diff_625.txt -./tools/test/h5diff/testfiles/h5diff_626.txt -./tools/test/h5diff/testfiles/h5diff_627.txt -./tools/test/h5diff/testfiles/h5diff_628.txt -./tools/test/h5diff/testfiles/h5diff_629.txt -./tools/test/h5diff/testfiles/h5diff_630.txt -./tools/test/h5diff/testfiles/h5diff_631.txt -./tools/test/h5diff/testfiles/h5diff_640.txt -./tools/test/h5diff/testfiles/h5diff_641.txt -./tools/test/h5diff/testfiles/h5diff_642.txt -./tools/test/h5diff/testfiles/h5diff_643.txt -./tools/test/h5diff/testfiles/h5diff_644.txt -./tools/test/h5diff/testfiles/h5diff_645.txt -./tools/test/h5diff/testfiles/h5diff_646.txt -./tools/test/h5diff/testfiles/h5diff_70.txt -./tools/test/h5diff/testfiles/h5diff_700.txt -./tools/test/h5diff/testfiles/h5diff_701.txt -./tools/test/h5diff/testfiles/h5diff_702.txt -./tools/test/h5diff/testfiles/h5diff_703.txt -./tools/test/h5diff/testfiles/h5diff_704.txt -./tools/test/h5diff/testfiles/h5diff_705.txt -./tools/test/h5diff/testfiles/h5diff_706.txt -./tools/test/h5diff/testfiles/h5diff_707.txt -./tools/test/h5diff/testfiles/h5diff_708.txt -./tools/test/h5diff/testfiles/h5diff_709.txt -./tools/test/h5diff/testfiles/h5diff_710.txt -./tools/test/h5diff/testfiles/h5diff_80.txt -./tools/test/h5diff/testfiles/h5diff_800.txt -./tools/test/h5diff/testfiles/h5diff_801.txt -./tools/test/h5diff/testfiles/h5diff_830.txt -./tools/test/h5diff/testfiles/h5diff_90.txt -./tools/test/h5diff/testfiles/h5diff_100.txt -./tools/test/h5diff/testfiles/h5diff_101.txt -./tools/test/h5diff/testfiles/h5diff_102.txt -./tools/test/h5diff/testfiles/h5diff_103.txt -./tools/test/h5diff/testfiles/h5diff_104.txt -# w for Windows-specific -./tools/test/h5diff/testfiles/h5diff_101w.txt -./tools/test/h5diff/testfiles/h5diff_102w.txt -./tools/test/h5diff/testfiles/h5diff_103w.txt -./tools/test/h5diff/testfiles/h5diff_104w.txt -./tools/test/h5diff/testfiles/h5diff_200.txt -./tools/test/h5diff/testfiles/h5diff_201.txt -./tools/test/h5diff/testfiles/h5diff_202.txt -./tools/test/h5diff/testfiles/h5diff_203.txt -./tools/test/h5diff/testfiles/h5diff_204.txt -./tools/test/h5diff/testfiles/h5diff_205.txt -./tools/test/h5diff/testfiles/h5diff_206.txt -./tools/test/h5diff/testfiles/h5diff_207.txt -./tools/test/h5diff/testfiles/h5diff_208.txt -./tools/test/h5diff/testfiles/h5diff_220.txt -./tools/test/h5diff/testfiles/h5diff_221.txt -./tools/test/h5diff/testfiles/h5diff_222.txt -./tools/test/h5diff/testfiles/h5diff_223.txt -./tools/test/h5diff/testfiles/h5diff_224.txt -./tools/test/h5diff/testfiles/h5diff_300.txt -./tools/test/h5diff/testfiles/h5diff_400.txt -./tools/test/h5diff/testfiles/h5diff_401.txt -./tools/test/h5diff/testfiles/h5diff_402.txt -./tools/test/h5diff/testfiles/h5diff_403.txt -./tools/test/h5diff/testfiles/h5diff_404.txt -./tools/test/h5diff/testfiles/h5diff_405.txt -./tools/test/h5diff/testfiles/h5diff_406.txt -./tools/test/h5diff/testfiles/h5diff_407.txt -./tools/test/h5diff/testfiles/h5diff_408.txt -./tools/test/h5diff/testfiles/h5diff_409.txt -./tools/test/h5diff/testfiles/h5diff_410.txt -./tools/test/h5diff/testfiles/h5diff_411.txt -./tools/test/h5diff/testfiles/h5diff_412.txt -./tools/test/h5diff/testfiles/h5diff_413.txt -./tools/test/h5diff/testfiles/h5diff_414.txt -./tools/test/h5diff/testfiles/h5diff_415.txt -./tools/test/h5diff/testfiles/h5diff_416.txt -./tools/test/h5diff/testfiles/h5diff_417.txt -./tools/test/h5diff/testfiles/h5diff_418.txt -./tools/test/h5diff/testfiles/h5diff_419.txt -./tools/test/h5diff/testfiles/h5diff_420.txt -./tools/test/h5diff/testfiles/h5diff_421.txt -./tools/test/h5diff/testfiles/h5diff_422.txt -./tools/test/h5diff/testfiles/h5diff_423.txt -./tools/test/h5diff/testfiles/h5diff_424.txt -./tools/test/h5diff/testfiles/h5diff_425.txt -./tools/test/h5diff/testfiles/h5diff_450.txt -./tools/test/h5diff/testfiles/h5diff_451.txt -./tools/test/h5diff/testfiles/h5diff_452.txt -./tools/test/h5diff/testfiles/h5diff_453.txt -./tools/test/h5diff/testfiles/h5diff_454.txt -./tools/test/h5diff/testfiles/dangling_link.err -./tools/test/h5diff/testfiles/h5diff_455.txt -./tools/test/h5diff/testfiles/h5diff_456.txt -./tools/test/h5diff/testfiles/h5diff_457.txt -./tools/test/h5diff/testfiles/h5diff_458.txt -./tools/test/h5diff/testfiles/h5diff_459.txt -./tools/test/h5diff/testfiles/h5diff_465.txt -./tools/test/h5diff/testfiles/h5diff_466.txt -./tools/test/h5diff/testfiles/h5diff_467.txt -./tools/test/h5diff/testfiles/h5diff_468.txt -./tools/test/h5diff/testfiles/h5diff_469.txt -./tools/test/h5diff/testfiles/h5diff_471.txt -./tools/test/h5diff/testfiles/h5diff_472.txt -./tools/test/h5diff/testfiles/h5diff_473.txt -./tools/test/h5diff/testfiles/h5diff_474.txt -./tools/test/h5diff/testfiles/h5diff_475.txt -./tools/test/h5diff/testfiles/h5diff_480.txt -./tools/test/h5diff/testfiles/h5diff_481.txt -./tools/test/h5diff/testfiles/h5diff_482.txt -./tools/test/h5diff/testfiles/h5diff_483.txt -./tools/test/h5diff/testfiles/h5diff_484.txt -./tools/test/h5diff/testfiles/h5diff_485.txt -./tools/test/h5diff/testfiles/h5diff_486.txt -./tools/test/h5diff/testfiles/h5diff_487.txt -./tools/test/h5diff/testfiles/h5diff_500.txt -./tools/test/h5diff/testfiles/h5diff_501.txt -./tools/test/h5diff/testfiles/h5diff_502.txt -./tools/test/h5diff/testfiles/h5diff_503.txt -./tools/test/h5diff/testfiles/h5diff_504.txt -./tools/test/h5diff/testfiles/h5diff_505.txt -./tools/test/h5diff/testfiles/h5diff_506.txt -./tools/test/h5diff/testfiles/h5diff_507.txt -./tools/test/h5diff/testfiles/h5diff_508.txt -./tools/test/h5diff/testfiles/h5diff_509.txt -./tools/test/h5diff/testfiles/h5diff_510.txt -./tools/test/h5diff/testfiles/h5diff_511.txt -./tools/test/h5diff/testfiles/h5diff_512.txt -./tools/test/h5diff/testfiles/h5diff_513.txt -./tools/test/h5diff/testfiles/h5diff_514.txt -./tools/test/h5diff/testfiles/h5diff_515.txt -./tools/test/h5diff/testfiles/h5diff_516.txt -./tools/test/h5diff/testfiles/h5diff_517.txt -./tools/test/h5diff/testfiles/h5diff_518.txt -./tools/test/h5diff/testfiles/h5diff_530.txt -./tools/test/h5diff/testfiles/h5diff_540.txt - -./tools/test/h5diff/testfiles/h5diff_attr1.h5 -./tools/test/h5diff/testfiles/h5diff_attr2.h5 -./tools/test/h5diff/testfiles/h5diff_attr3.h5 -./tools/test/h5diff/testfiles/h5diff_attr_v_level1.h5 -./tools/test/h5diff/testfiles/h5diff_attr_v_level2.h5 -./tools/test/h5diff/testfiles/h5diff_basic1.h5 -./tools/test/h5diff/testfiles/h5diff_basic2.h5 -./tools/test/h5diff/testfiles/h5diff_dset1.h5 -./tools/test/h5diff/testfiles/h5diff_dset2.h5 -./tools/test/h5diff/testfiles/h5diff_dset3.h5 -./tools/test/h5diff/testfiles/h5diff_dtypes.h5 -./tools/test/h5diff/testfiles/h5diff_empty.h5 -./tools/test/h5diff/testfiles/h5diff_eps1.h5 -./tools/test/h5diff/testfiles/h5diff_eps2.h5 -./tools/test/h5diff/testfiles/h5diff_hyper1.h5 -./tools/test/h5diff/testfiles/h5diff_hyper2.h5 -./tools/test/h5diff/testfiles/h5diff_types.h5 -./tools/test/h5diff/testfiles/h5diff_links.h5 -./tools/test/h5diff/testfiles/h5diff_ext2softlink_src.h5 -./tools/test/h5diff/testfiles/h5diff_ext2softlink_trg.h5 -./tools/test/h5diff/testfiles/h5diff_extlink_src.h5 -./tools/test/h5diff/testfiles/h5diff_extlink_trg.h5 -./tools/test/h5diff/testfiles/h5diff_linked_softlink.h5 -./tools/test/h5diff/testfiles/h5diff_softlinks.h5 -./tools/test/h5diff/testfiles/h5diff_danglelinks1.h5 -./tools/test/h5diff/testfiles/h5diff_danglelinks2.h5 -./tools/test/h5diff/testfiles/h5diff_grp_recurse1.h5 -./tools/test/h5diff/testfiles/h5diff_grp_recurse2.h5 -./tools/test/h5diff/testfiles/h5diff_grp_recurse_ext1.h5 -./tools/test/h5diff/testfiles/h5diff_grp_recurse_ext2-1.h5 -./tools/test/h5diff/testfiles/h5diff_grp_recurse_ext2-2.h5 -./tools/test/h5diff/testfiles/h5diff_grp_recurse_ext2-3.h5 -./tools/test/h5diff/testfiles/h5diff_exclude1-1.h5 -./tools/test/h5diff/testfiles/h5diff_exclude1-2.h5 -./tools/test/h5diff/testfiles/h5diff_exclude2-1.h5 -./tools/test/h5diff/testfiles/h5diff_exclude2-2.h5 -./tools/test/h5diff/testfiles/h5diff_exclude3-1.h5 -./tools/test/h5diff/testfiles/h5diff_exclude3-2.h5 -./tools/test/h5diff/testfiles/h5diff_comp_vl_strs.h5 -./tools/test/h5diff/testfiles/h5diff_dset_zero_dim_size1.h5 -./tools/test/h5diff/testfiles/h5diff_dset_zero_dim_size2.h5 -./tools/test/h5diff/testfiles/h5diff_enum_invalid_values.h5 -./tools/test/h5diff/testfiles/h5diff_eps.txt -./tools/test/h5diff/testfiles/compounds_array_vlen1.h5 -./tools/test/h5diff/testfiles/compounds_array_vlen2.h5 -./tools/test/h5diff/testfiles/non_comparables1.h5 -./tools/test/h5diff/testfiles/non_comparables2.h5 -./tools/test/h5diff/testfiles/h5diff_8625.txt -./tools/test/h5diff/testfiles/h5diff_8639.txt -./tools/test/h5diff/testfiles/tudfilter.h5 -./tools/test/h5diff/testfiles/tudfilter2.h5 -./tools/test/h5diff/testfiles/h5diff_ud.txt -./tools/test/h5diff/testfiles/h5diff_udfail.err -./tools/test/h5diff/testfiles/h5diff_udfail.txt -./tools/test/h5diff/testfiles/h5diff_strings1.h5 -./tools/test/h5diff/testfiles/h5diff_strings2.h5 -./tools/test/h5diff/testfiles/h5diff_vlstr.txt -#vds -./tools/test/h5diff/testfiles/h5diff_v1.txt -./tools/test/h5diff/testfiles/h5diff_v2.txt -./tools/test/h5diff/testfiles/h5diff_v3.txt -#reg_ref -./tools/test/h5diff/testfiles/h5diff_reg.txt - -#test files for h5repack -./tools/test/h5repack/testfiles/README -./tools/test/h5repack/testfiles/bounds_latest_latest.h5 -./tools/test/h5repack/testfiles/h5copy_extlinks_src.h5 -./tools/test/h5repack/testfiles/h5repack_aggr.h5 -./tools/test/h5repack/testfiles/h5repack_attr.h5 -./tools/test/h5repack/testfiles/h5repack_attr_refs.h5 -./tools/test/h5repack/testfiles/h5repack_deflate.h5 -./tools/test/h5repack/testfiles/h5repack_early.h5 -./tools/test/h5repack/testfiles/h5repack_ext.bin -./tools/test/h5repack/testfiles/h5repack_ext.h5 -./tools/test/h5repack/testfiles/h5repack_f32le.h5 -./tools/test/h5repack/testfiles/h5repack_f32le_ex-0.dat -./tools/test/h5repack/testfiles/h5repack_f32le_ex.h5 -./tools/test/h5repack/testfiles/h5repack_fill.h5 -./tools/test/h5repack/testfiles/h5repack_filters.h5 -./tools/test/h5repack/testfiles/h5repack_fletcher.h5 -./tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 -./tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 -./tools/test/h5repack/testfiles/h5repack_hlink.h5 -./tools/test/h5repack/testfiles/h5repack_int32le_1d.h5 -./tools/test/h5repack/testfiles/h5repack_int32le_1d_ex-0.dat -./tools/test/h5repack/testfiles/h5repack_int32le_1d_ex-1.dat -./tools/test/h5repack/testfiles/h5repack_int32le_1d_ex.h5 -./tools/test/h5repack/testfiles/h5repack_int32le_2d.h5 -./tools/test/h5repack/testfiles/h5repack_int32le_2d_ex-0.dat -./tools/test/h5repack/testfiles/h5repack_int32le_2d_ex.h5 -./tools/test/h5repack/testfiles/h5repack_int32le_3d.h5 -./tools/test/h5repack/testfiles/h5repack_int32le_3d_ex-0.dat -./tools/test/h5repack/testfiles/h5repack_int32le_3d_ex.h5 -./tools/test/h5repack/testfiles/h5repack.info -./tools/test/h5repack/testfiles/h5repack_layout.h5 -./tools/test/h5repack/testfiles/h5repack_layouto.h5 -./tools/test/h5repack/testfiles/h5repack_layout2.h5 -./tools/test/h5repack/testfiles/h5repack_layout3.h5 -./tools/test/h5repack/testfiles/h5repack_layout.UD.h5 -./tools/test/h5repack/testfiles/h5repack_named_dtypes.h5 -./tools/test/h5repack/testfiles/h5repack_nested_8bit_enum_deflated.h5 -./tools/test/h5repack/testfiles/h5repack_nested_8bit_enum.h5 -./tools/test/h5repack/testfiles/h5repack_nbit.h5 -./tools/test/h5repack/testfiles/h5repack_none.h5 -./tools/test/h5repack/testfiles/h5repack_objs.h5 -./tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 -./tools/test/h5repack/testfiles/h5repack_paged_persist.h5 -./tools/test/h5repack/testfiles/h5repack_refs.h5 -./tools/test/h5repack/testfiles/h5repack_shuffle.h5 -./tools/test/h5repack/testfiles/h5repack_soffset.h5 -./tools/test/h5repack/testfiles/h5repack_szip.h5 -./tools/test/h5repack/testfiles/h5repack_uint8be.h5 -./tools/test/h5repack/testfiles/h5repack_uint8be_ex-0.dat -./tools/test/h5repack/testfiles/h5repack_uint8be_ex-1.dat -./tools/test/h5repack/testfiles/h5repack_uint8be_ex-2.dat -./tools/test/h5repack/testfiles/h5repack_uint8be_ex-3.dat -./tools/test/h5repack/testfiles/h5repack_uint8be_ex.h5 -./tools/test/h5repack/testfiles/ublock.bin -./tools/test/h5repack/testfiles/crtorder.tordergr.h5.ddl -./tools/test/h5repack/testfiles/deflate_limit.h5repack_layout.h5.ddl -./tools/test/h5repack/testfiles/plugin_none.h5repack_layout.UD.h5.tst -./tools/test/h5repack/testfiles/plugin_test.h5repack_layout.h5.tst -./tools/test/h5repack/testfiles/plugin_zero.h5repack_layout.h5.tst -./tools/test/h5repack/testfiles/plugin_version_test.h5repack_layout.h5.tst -./tools/test/h5repack/testfiles/h5repack-help.txt -./tools/test/h5repack/testfiles/h5repack_filters.h5-gzip_verbose_filters.tst -./tools/test/h5repack/testfiles/h5repack_layout.h5-dset2_chunk_20x10-errstk.tst -./tools/test/h5repack/testfiles/h5repack_layout.h5.ddl -./tools/test/h5repack/testfiles/h5repack_layout.UD.h5-plugin_none.ddl -./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_test.ddl -./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl -./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_zero.ddl -./tools/test/h5repack/testfiles/h5repack_CVE-2018-17432.h5 -./tools/test/h5repack/testfiles/h5repack_CVE-2018-14460.h5 -./tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl -./tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl -./tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl -./tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl -./tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl -./tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl -./tools/test/h5repack/testfiles/1_vds.h5-vds_dset_chunk20x10x5-v.ddl -./tools/test/h5repack/testfiles/2_vds.h5-vds_chunk3x6x9-v.ddl -./tools/test/h5repack/testfiles/3_1_vds.h5-vds_chunk2x5x8-v.ddl -./tools/test/h5repack/testfiles/4_vds.h5-vds_conti-v.ddl -./tools/test/h5repack/testfiles/4_vds.h5-vds_compa-v.ddl -./tools/test/h5repack/testfiles/attrregion.tattrreg.h5.ddl -./tools/test/h5repack/testfiles/dataregion.tdatareg.h5.ddl -./tools/test/h5repack/testfiles/textlink-base.textlink.h5.ddl -./tools/test/h5repack/testfiles/textlink-merge.textlink.h5.tst -./tools/test/h5repack/testfiles/textlink-mergeprune.textlink.h5.ddl -./tools/test/h5repack/testfiles/textlink-prune.textlink.h5.ddl -./tools/test/h5repack/testfiles/textlinkfar-base.textlinkfar.h5.ddl -./tools/test/h5repack/testfiles/textlinkfar-merge.textlinkfar.h5.tst -./tools/test/h5repack/testfiles/textlinkfar-mergeprune.textlinkfar.h5.ddl -./tools/test/h5repack/testfiles/textlinkfar-prune.textlinkfar.h5.ddl -./tools/test/h5repack/testfiles/textlinksrc-base.textlinksrc.h5.ddl -./tools/test/h5repack/testfiles/textlinksrc-merge.textlinksrc.h5.tst -./tools/test/h5repack/testfiles/textlinksrc-mergeprune.textlinksrc.h5.ddl -./tools/test/h5repack/testfiles/textlinksrc-prune.textlinksrc.h5.ddl -./tools/test/h5repack/testfiles/textlinktar-base.textlinktar.h5.ddl -./tools/test/h5repack/testfiles/textlinktar-merge.textlinktar.h5.tst -./tools/test/h5repack/testfiles/textlinktar-mergeprune.textlinktar.h5.ddl -./tools/test/h5repack/testfiles/textlinktar-prune.textlinktar.h5.ddl -./tools/test/h5repack/testfiles/tsoftlinks-base.tsoftlinks.h5.ddl -./tools/test/h5repack/testfiles/tsoftlinks-merge.tsoftlinks.h5.tst -./tools/test/h5repack/testfiles/tsoftlinks-mergeprune.tsoftlinks.h5.ddl -./tools/test/h5repack/testfiles/tsoftlinks-prune.tsoftlinks.h5.ddl -./tools/test/h5repack/testfiles/h5copy_extlinks_src-base.h5copy_extlinks_src.h5.ddl -./tools/test/h5repack/testfiles/h5copy_extlinks_src-merge.h5copy_extlinks_src.h5.tst -./tools/test/h5repack/testfiles/h5copy_extlinks_src-mergeprune.h5copy_extlinks_src.h5.ddl -./tools/test/h5repack/testfiles/h5copy_extlinks_src-prune.h5copy_extlinks_src.h5.ddl -./tools/test/h5repack/testfiles/h5copy_extlinks_trg.h5 - -# jam utility and tests -./tools/src/h5jam/Makefile.am -./tools/src/h5jam/h5jam.c -./tools/src/h5jam/h5unjam.c -./tools/test/h5jam/Makefile.am -./tools/test/h5jam/h5jamgentest.c -./tools/test/h5jam/testh5jam.sh.in -./tools/test/h5jam/tellub.c -./tools/test/h5jam/getub.c - -# test files for jam -./tools/test/h5jam/testfiles/h5jam-help.txt -./tools/test/h5jam/testfiles/h5unjam-help.txt -./tools/test/h5jam/testfiles/tall.h5 -./tools/test/h5jam/testfiles/twithub.h5 -./tools/test/h5jam/testfiles/twithub513.h5 -./tools/test/h5jam/testfiles/u10.txt -./tools/test/h5jam/testfiles/u511.txt -./tools/test/h5jam/testfiles/u512.txt -./tools/test/h5jam/testfiles/u513.txt -./tools/test/h5jam/testfiles/h5jam-ub-nohdf5.txt - -# test files for h5copy -./tools/test/h5copy/testfiles/h5copytst.h5 -./tools/test/h5copy/testfiles/h5copytst.out.ls -./tools/test/h5copy/testfiles/h5copy_ref.h5 -./tools/test/h5copy/testfiles/h5copy_ref.out.ls -./tools/test/h5copy/testfiles/h5copy_extlinks_src.h5 -./tools/test/h5copy/testfiles/h5copy_extlinks_trg.h5 -./tools/test/h5copy/testfiles/h5copy_extlinks_src.out.ls -./tools/test/h5copy/testfiles/h5copy_misc1.err -./tools/test/h5copy/testfiles/h5copy_misc1.out -./tools/test/h5copy/testfiles/h5copytst_new.h5 -./tools/test/h5copy/testfiles/h5copytst_new.out.ls -./tools/test/h5copy/testfiles/h5copy_plugin_fail_ERR.out.h5.txt -./tools/test/h5copy/testfiles/h5copy_plugin_test.out.h5.txt -./tools/test/h5copy/testfiles/tudfilter.h5 -./tools/test/h5copy/testfiles/tudfilter2.h5 -./tools/test/h5copy/testfiles/tudfilter.h5.txt -./tools/test/h5copy/testfiles/tudfilter.h5_ERR.txt - -# test files for h5mkgrp -./tools/testfiles/h5mkgrp_nested_p.ls -./tools/testfiles/h5mkgrp_nested_lp.ls -./tools/testfiles/h5mkgrp_nested_mult_p.ls -./tools/testfiles/h5mkgrp_nested_mult_lp.ls -./tools/testfiles/h5mkgrp_several.ls -./tools/testfiles/h5mkgrp_several_v.ls -./tools/testfiles/h5mkgrp_several_p.ls -./tools/testfiles/h5mkgrp_several_l.ls -./tools/testfiles/h5mkgrp_single.ls -./tools/testfiles/h5mkgrp_single_v.ls -./tools/testfiles/h5mkgrp_single_p.ls -./tools/testfiles/h5mkgrp_single_l.ls - -./tools/src/h5perf/Makefile.am -./tools/src/h5perf/perf.c -./tools/src/h5perf/pio_engine.c -./tools/src/h5perf/pio_perf.c -./tools/src/h5perf/pio_perf.h -./tools/src/h5perf/sio_engine.c -./tools/src/h5perf/sio_perf.c -./tools/src/h5perf/sio_perf.h - -./tools/test/perform/Makefile.am -./tools/test/perform/build_h5perf_alone.sh -./tools/test/perform/build_h5perf_serial_alone.sh -./tools/test/perform/chunk.c -./tools/test/perform/chunk_cache.c -./tools/test/perform/direct_write_perf.c -./tools/test/perform/gen_report.pl -./tools/test/perform/iopipe.c -./tools/test/perform/overhead.c -./tools/test/perform/perf_meta.c -./tools/test/perform/pio_standalone.c -./tools/test/perform/pio_standalone.h -./tools/test/perform/sio_standalone.c -./tools/test/perform/sio_standalone.h -./tools/test/perform/zip_perf.c - -# Utils directory -./utils/Makefile.am - -# Mirror VFD utilities -./utils/mirror_vfd/Makefile.am -./utils/mirror_vfd/mirror_remote.c -./utils/mirror_vfd/mirror_remote.h -./utils/mirror_vfd/mirror_server.c -./utils/mirror_vfd/mirror_server_stop.c -./utils/mirror_vfd/mirror_writer.c - -# test utilities -./utils/test/Makefile.am -./utils/test/swmr_check_compat_vfd.c - -# parallel tools (h5dwalk) and tests -./utils/tools/CMakeLists.txt -./utils/tools/Makefile.am -./utils/tools/h5dwalk/CMakeLists.txt -./utils/tools/h5dwalk/Makefile.am -./utils/tools/h5dwalk/h5dwalk.1 -./utils/tools/h5dwalk/h5dwalk.c -./utils/tools/test/CMakeLists.txt -./utils/tools/test/Makefile.am -./utils/tools/test/h5dwalk/CMakeLists.txt -./utils/tools/test/h5dwalk/CMakeTests.cmake -./utils/tools/test/h5dwalk/Makefile.am -./utils/tools/test/h5dwalk/copy_demo_files.sh.in -./utils/tools/test/h5dwalk/help.h5dwalk -./utils/tools/test/h5dwalk/testh5dwalk.sh.in - -# high level libraries -./hl/Makefile.am -./hl/examples/Makefile.am -./hl/examples/ex_ds1.c -./hl/examples/ex_lite1.c -./hl/examples/ex_lite2.c -./hl/examples/ex_lite3.c -./hl/examples/ex_image1.c -./hl/examples/ex_image2.c -./hl/examples/ex_table_01.c -./hl/examples/ex_table_02.c -./hl/examples/ex_table_03.c -./hl/examples/ex_table_04.c -./hl/examples/ex_table_05.c -./hl/examples/ex_table_06.c -./hl/examples/ex_table_07.c -./hl/examples/ex_table_08.c -./hl/examples/ex_table_09.c -./hl/examples/ex_table_10.c -./hl/examples/ex_table_11.c -./hl/examples/ex_table_12.c -./hl/examples/image24pixel.txt -./hl/examples/image8.txt -./hl/examples/pal_rgb.h -./hl/examples/ptExampleFL.c -./hl/examples/run-hl-ex.sh -./hl/examples/run-hlc-ex.sh.in -./hl/src/Makefile.am -./hl/src/H5DO.c -./hl/src/H5DOpublic.h -./hl/src/H5DS.c -./hl/src/H5DSprivate.h -./hl/src/H5DSpublic.h -./hl/src/H5HLprivate2.h -./hl/src/H5IM.c -./hl/src/H5IMprivate.h -./hl/src/H5IMpublic.h -./hl/src/H5LD.c -./hl/src/H5LDprivate.h -./hl/src/H5LDpublic.h -./hl/src/H5LT.c -./hl/src/H5LTanalyze.c -./hl/src/H5LTanalyze.l -./hl/src/H5LTparse.c -./hl/src/H5LTparse.h -./hl/src/H5LTparse.y -./hl/src/H5LTprivate.h -./hl/src/H5LTpublic.h -./hl/src/H5PT.c -./hl/src/H5PTprivate.h -./hl/src/H5PTpublic.h -./hl/src/H5TB.c -./hl/src/H5TBprivate.h -./hl/src/H5TBpublic.h -./hl/src/hdf5_hl.h -./hl/test/H5srcdir_str.h.in -./hl/test/Makefile.am -./hl/test/dsdata.txt -./hl/test/dslat.txt -./hl/test/dslon.txt -./hl/test/dtype_file.txt -./hl/test/dtype_file_readable.txt -./hl/test/earth.pal -./hl/test/gen_test_ds.c -./hl/test/gen_test_ld.c -./hl/test/h5hltest.h -./hl/test/image24pixel.txt -./hl/test/image24plane.txt -./hl/test/image8.txt -./hl/test/pal_rgb.h -./hl/test/sepia.pal -./hl/test/test_ds.c -./hl/test/test_ds_be.h5 -./hl/test/test_ds_be_new_ref.h5 -./hl/test/test_ds_be_new_ref-32bit.h5 -./hl/test/test_ds_le.h5 -./hl/test/test_ds_le_new_ref.h5 -./hl/test/test_dset_append.c -./hl/test/test_file_image.c -./hl/test/test_h5do_compat.c -./hl/test/test_image.c -./hl/test/test_ld.c -./hl/test/test_ld.h5 -./hl/test/test_lite.c -./hl/test/test_packet.c -./hl/test/test_packet_vlen.c -./hl/test/test_table.c -./hl/test/test_table_be.h5 -./hl/test/test_table_cray.h5 -./hl/test/test_table_le.h5 -./hl/test/usa.wri - -# tools -./hl/tools/Makefile.am -./hl/tools/gif2h5/Makefile.am -./hl/tools/gif2h5/decompress.c -./hl/tools/gif2h5/gif.h -./hl/tools/gif2h5/gif2hdf.c -./hl/tools/gif2h5/gif2mem.c -./hl/tools/gif2h5/gifread.c -./hl/tools/gif2h5/hdf2gif.c -./hl/tools/gif2h5/hdfgifwr.c -./hl/tools/gif2h5/writehdf.c -./hl/tools/gif2h5/h52gifgentst.c -./hl/tools/gif2h5/h52giftest.sh.in -./hl/tools/gif2h5/testfiles/README -./hl/tools/gif2h5/testfiles/ex_image2.h5 -./hl/tools/gif2h5/testfiles/image1.gif -./hl/tools/gif2h5/testfiles/h52giftst.h5 -# -./hl/tools/h5watch/Makefile.am -./hl/tools/h5watch/extend_dset.c -./hl/tools/h5watch/h5watch.c -./hl/tools/h5watch/h5watchgentest.c -./hl/tools/h5watch/testh5watch.sh.in - -# expected test output from testing h5watch -# -./hl/tools/testfiles/w-err-std.ddl -./hl/tools/testfiles/w-err-cmpd1.err -./hl/tools/testfiles/w-err-cmpd2.err -./hl/tools/testfiles/w-err-cmpd3.err -./hl/tools/testfiles/w-err-cmpd4.err -./hl/tools/testfiles/w-err-cmpd5.err -./hl/tools/testfiles/w-err-dset1.err -./hl/tools/testfiles/w-err-dset2.err -./hl/tools/testfiles/w-err-dset-nomax.err -./hl/tools/testfiles/w-err-dset-none.err -./hl/tools/testfiles/w-err-file.err -./hl/tools/testfiles/w-err-poll0.ddl -./hl/tools/testfiles/w-err-poll.ddl -./hl/tools/testfiles/w-err-width.ddl -./hl/tools/testfiles/w-ext-cmpd.ddl -./hl/tools/testfiles/w-ext-cmpd-esc.ddl -./hl/tools/testfiles/w-ext-cmpd-esc-f1.ddl -./hl/tools/testfiles/w-ext-cmpd-esc-f3.ddl -./hl/tools/testfiles/w-ext-cmpd-esc-ff2.ddl -./hl/tools/testfiles/w-ext-cmpd-f1.ddl -./hl/tools/testfiles/w-ext-cmpd-f2.ddl -./hl/tools/testfiles/w-ext-cmpd-ff3.ddl -./hl/tools/testfiles/w-ext-cmpd-label.ddl -./hl/tools/testfiles/w-ext-cmpd-two.ddl -./hl/tools/testfiles/w-ext-cmpd-two-f1.ddl -./hl/tools/testfiles/w-ext-cmpd-two-f3.ddl -./hl/tools/testfiles/w-ext-cmpd-two-ff2.ddl -./hl/tools/testfiles/w-ext-early.ddl -./hl/tools/testfiles/w-ext-late.ddl -./hl/tools/testfiles/w-ext-one-d.ddl -./hl/tools/testfiles/w-ext-one.ddl -./hl/tools/testfiles/w-ext-one-simple.ddl -./hl/tools/testfiles/w-ext-two-d.ddl -./hl/tools/testfiles/w-ext-two.ddl -./hl/tools/testfiles/w-ext-two-width.ddl -./hl/tools/testfiles/w-help1.ddl - -# hl fortran -./hl/fortran/Makefile.am -./hl/fortran/examples/Makefile.am -./hl/fortran/examples/run-hlfortran-ex.sh.in -./hl/fortran/src/hdf5_hl_fortrandll.def.in -./hl/fortran/src/H5DSfc.c -./hl/fortran/src/H5DSff.F90 -./hl/fortran/src/H5HL_buildiface.F90 -./hl/fortran/src/H5IMcc.c -./hl/fortran/src/H5IMcc.h -./hl/fortran/src/H5IMfc.c -./hl/fortran/src/H5IMff.F90 -./hl/fortran/src/H5LTf90proto.h -./hl/fortran/src/H5LTfc.c -./hl/fortran/src/H5LTff.F90 -./hl/fortran/src/H5TBfc.c -./hl/fortran/src/H5TBff.F90 -./hl/fortran/src/Makefile.am -./hl/fortran/test/Makefile.am -./hl/fortran/test/tstds.F90 -./hl/fortran/test/tstimage.F90 -./hl/fortran/test/tstlite.F90 -./hl/fortran/test/tsttable.F90 - -# hl c++ -./hl/c++/Makefile.am -./hl/c++/examples/Makefile.am -./hl/c++/examples/ptExampleFL.cpp -./hl/c++/examples/run-hlc++-ex.sh.in -./hl/c++/src/H5PacketTable.h -./hl/c++/src/H5PacketTable.cpp -./hl/c++/src/Makefile.am -./hl/c++/test/ptableTest.h -./hl/c++/test/ptableTest.cpp -./hl/c++/test/Makefile.am - -# java -./java/Makefile.am -./java/CMakeLists.txt - -./java/src/Makefile.am -./java/src/CMakeLists.txt -./java/src/jni/Makefile.am -./java/src/jni/CMakeLists.txt -./java/src/jni/exceptionImp.c -./java/src/jni/exceptionImp.h -./java/src/jni/h5Constants.c -./java/src/jni/nativeData.c -./java/src/jni/nativeData.h -./java/src/jni/h5jni.h -./java/src/jni/h5util.c -./java/src/jni/h5util.h -./java/src/jni/h5Imp.c -./java/src/jni/h5Imp.h -./java/src/jni/h5aImp.c -./java/src/jni/h5aImp.h -./java/src/jni/h5dImp.c -./java/src/jni/h5dImp.h -./java/src/jni/h5eImp.c -./java/src/jni/h5eImp.h -./java/src/jni/h5fImp.c -./java/src/jni/h5fImp.h -./java/src/jni/h5gImp.c -./java/src/jni/h5gImp.h -./java/src/jni/h5iImp.c -./java/src/jni/h5iImp.h -./java/src/jni/h5lImp.c -./java/src/jni/h5lImp.h -./java/src/jni/h5oImp.c -./java/src/jni/h5oImp.h -./java/src/jni/h5pImp.c -./java/src/jni/h5pImp.h -./java/src/jni/h5pACPLImp.c -./java/src/jni/h5pACPLImp.h -./java/src/jni/h5pDAPLImp.c -./java/src/jni/h5pDAPLImp.h -./java/src/jni/h5pDCPLImp.c -./java/src/jni/h5pDCPLImp.h -./java/src/jni/h5pDXPLImp.c -./java/src/jni/h5pDXPLImp.h -./java/src/jni/h5pFAPLImp.c -./java/src/jni/h5pFAPLImp.h -./java/src/jni/h5pFCPLImp.c -./java/src/jni/h5pFCPLImp.h -./java/src/jni/h5pGAPLImp.c -./java/src/jni/h5pGAPLImp.h -./java/src/jni/h5pGCPLImp.c -./java/src/jni/h5pGCPLImp.h -./java/src/jni/h5pLAPLImp.c -./java/src/jni/h5pLAPLImp.h -./java/src/jni/h5pLCPLImp.c -./java/src/jni/h5pLCPLImp.h -./java/src/jni/h5pOCPLImp.c -./java/src/jni/h5pOCPLImp.h -./java/src/jni/h5pOCpyPLImp.c -./java/src/jni/h5pOCpyPLImp.h -./java/src/jni/h5pStrCPLImp.c -./java/src/jni/h5pStrCPLImp.h -./java/src/jni/h5plImp.c -./java/src/jni/h5plImp.h -./java/src/jni/h5rImp.c -./java/src/jni/h5rImp.h -./java/src/jni/h5sImp.c -./java/src/jni/h5sImp.h -./java/src/jni/h5tImp.c -./java/src/jni/h5tImp.h -./java/src/jni/h5vlImp.c -./java/src/jni/h5vlImp.h -./java/src/jni/h5zImp.c -./java/src/jni/h5zImp.h - -./java/src/hdf/CMakeLists.txt -./java/src/hdf/overview.html -./java/src/hdf/hdf5lib/CMakeLists.txt - -./java/src/hdf/hdf5lib/callbacks/Callbacks.java -./java/src/hdf/hdf5lib/callbacks/H5A_iterate_cb.java -./java/src/hdf/hdf5lib/callbacks/H5A_iterate_t.java -./java/src/hdf/hdf5lib/callbacks/H5D_iterate_cb.java -./java/src/hdf/hdf5lib/callbacks/H5D_iterate_t.java -./java/src/hdf/hdf5lib/callbacks/H5D_append_cb.java -./java/src/hdf/hdf5lib/callbacks/H5D_append_t.java -./java/src/hdf/hdf5lib/callbacks/H5E_walk_cb.java -./java/src/hdf/hdf5lib/callbacks/H5E_walk_t.java -./java/src/hdf/hdf5lib/callbacks/H5L_iterate_t.java -./java/src/hdf/hdf5lib/callbacks/H5L_iterate_opdata_t.java -./java/src/hdf/hdf5lib/callbacks/H5O_iterate_t.java -./java/src/hdf/hdf5lib/callbacks/H5O_iterate_opdata_t.java -./java/src/hdf/hdf5lib/callbacks/H5P_cls_close_func_cb.java -./java/src/hdf/hdf5lib/callbacks/H5P_cls_close_func_t.java -./java/src/hdf/hdf5lib/callbacks/H5P_cls_copy_func_cb.java -./java/src/hdf/hdf5lib/callbacks/H5P_cls_copy_func_t.java -./java/src/hdf/hdf5lib/callbacks/H5P_cls_create_func_cb.java -./java/src/hdf/hdf5lib/callbacks/H5P_cls_create_func_t.java -./java/src/hdf/hdf5lib/callbacks/H5P_prp_close_func_cb.java -./java/src/hdf/hdf5lib/callbacks/H5P_prp_compare_func_cb.java -./java/src/hdf/hdf5lib/callbacks/H5P_prp_copy_func_cb.java -./java/src/hdf/hdf5lib/callbacks/H5P_prp_create_func_cb.java -./java/src/hdf/hdf5lib/callbacks/H5P_prp_delete_func_cb.java -./java/src/hdf/hdf5lib/callbacks/H5P_prp_get_func_cb.java -./java/src/hdf/hdf5lib/callbacks/H5P_prp_set_func_cb.java -./java/src/hdf/hdf5lib/callbacks/H5P_iterate_cb.java -./java/src/hdf/hdf5lib/callbacks/H5P_iterate_t.java -./java/src/hdf/hdf5lib/callbacks/package-info.java - -./java/src/hdf/hdf5lib/exceptions/HDF5AttributeException.java -./java/src/hdf/hdf5lib/exceptions/HDF5BtreeException.java -./java/src/hdf/hdf5lib/exceptions/HDF5DataFiltersException.java -./java/src/hdf/hdf5lib/exceptions/HDF5DataStorageException.java -./java/src/hdf/hdf5lib/exceptions/HDF5DatasetInterfaceException.java -./java/src/hdf/hdf5lib/exceptions/HDF5DataspaceInterfaceException.java -./java/src/hdf/hdf5lib/exceptions/HDF5DatatypeInterfaceException.java -./java/src/hdf/hdf5lib/exceptions/HDF5Exception.java -./java/src/hdf/hdf5lib/exceptions/HDF5ExternalFileListException.java -./java/src/hdf/hdf5lib/exceptions/HDF5FileInterfaceException.java -./java/src/hdf/hdf5lib/exceptions/HDF5FunctionArgumentException.java -./java/src/hdf/hdf5lib/exceptions/HDF5FunctionEntryExitException.java -./java/src/hdf/hdf5lib/exceptions/HDF5HeapException.java -./java/src/hdf/hdf5lib/exceptions/HDF5IdException.java -./java/src/hdf/hdf5lib/exceptions/HDF5InternalErrorException.java -./java/src/hdf/hdf5lib/exceptions/HDF5JavaException.java -./java/src/hdf/hdf5lib/exceptions/HDF5LibraryException.java -./java/src/hdf/hdf5lib/exceptions/HDF5LowLevelIOException.java -./java/src/hdf/hdf5lib/exceptions/HDF5MetaDataCacheException.java -./java/src/hdf/hdf5lib/exceptions/HDF5ObjectHeaderException.java -./java/src/hdf/hdf5lib/exceptions/HDF5PropertyListInterfaceException.java -./java/src/hdf/hdf5lib/exceptions/HDF5ReferenceException.java -./java/src/hdf/hdf5lib/exceptions/HDF5ResourceUnavailableException.java -./java/src/hdf/hdf5lib/exceptions/HDF5SymbolTableException.java -./java/src/hdf/hdf5lib/exceptions/package-info.java - -./java/src/hdf/hdf5lib/structs/H5_ih_info_t.java -./java/src/hdf/hdf5lib/structs/H5A_info_t.java -./java/src/hdf/hdf5lib/structs/H5AC_cache_config_t.java -./java/src/hdf/hdf5lib/structs/H5E_error2_t.java -./java/src/hdf/hdf5lib/structs/H5F_info2_t.java -./java/src/hdf/hdf5lib/structs/H5FD_hdfs_fapl_t.java -./java/src/hdf/hdf5lib/structs/H5FD_ros3_fapl_t.java -./java/src/hdf/hdf5lib/structs/H5G_info_t.java -./java/src/hdf/hdf5lib/structs/H5L_info_t.java -./java/src/hdf/hdf5lib/structs/H5O_hdr_info_t.java -./java/src/hdf/hdf5lib/structs/H5O_info_t.java -./java/src/hdf/hdf5lib/structs/H5O_native_info_t.java -./java/src/hdf/hdf5lib/structs/H5O_token_t.java -./java/src/hdf/hdf5lib/structs/package-info.java - -./java/src/hdf/hdf5lib/H5.java -./java/src/hdf/hdf5lib/HDF5Constants.java -./java/src/hdf/hdf5lib/HDF5GroupInfo.java -./java/src/hdf/hdf5lib/HDFArray.java -./java/src/hdf/hdf5lib/HDFNativeData.java -./java/src/hdf/hdf5lib/package-info.java - -./java/examples/Makefile.am -./java/examples/CMakeLists.txt - -./java/examples/intro/Makefile.am -./java/examples/intro/CMakeLists.txt -./java/examples/intro/JavaIntroExample.sh.in -./java/examples/intro/H5_CreateAttribute.java -./java/examples/intro/H5_CreateDataset.java -./java/examples/intro/H5_CreateFile.java -./java/examples/intro/H5_CreateGroup.java -./java/examples/intro/H5_CreateGroupAbsoluteRelative.java -./java/examples/intro/H5_CreateGroupDataset.java -./java/examples/intro/H5_ReadWrite.java - -./java/examples/groups/Makefile.am -./java/examples/groups/CMakeLists.txt -./java/examples/groups/JavaGroupExample.sh.in -./java/examples/groups/H5Ex_G_Create.java -./java/examples/groups/H5Ex_G_Iterate.java -./java/examples/groups/H5Ex_G_Compact.java -./java/examples/groups/H5Ex_G_Corder.java -./java/examples/groups/H5Ex_G_Intermediate.java -./java/examples/groups/H5Ex_G_Phase.java -./java/examples/groups/H5Ex_G_Traverse.java -./java/examples/groups/H5Ex_G_Visit.java -./java/examples/groups/h5ex_g_iterate.h5 -./java/examples/groups/h5ex_g_visit.h5 - -./java/examples/datasets/Makefile.am -./java/examples/datasets/CMakeLists.txt -./java/examples/datasets/JavaDatasetExample.sh.in -./java/examples/datasets/H5Ex_D_Alloc.java -./java/examples/datasets/H5Ex_D_Checksum.java -./java/examples/datasets/H5Ex_D_Chunk.java -./java/examples/datasets/H5Ex_D_Compact.java -./java/examples/datasets/H5Ex_D_External.java -./java/examples/datasets/H5Ex_D_FillValue.java -./java/examples/datasets/H5Ex_D_Gzip.java -./java/examples/datasets/H5Ex_D_Hyperslab.java -./java/examples/datasets/H5Ex_D_ReadWrite.java -./java/examples/datasets/H5Ex_D_Shuffle.java -./java/examples/datasets/H5Ex_D_Szip.java -./java/examples/datasets/H5Ex_D_UnlimitedAdd.java -./java/examples/datasets/H5Ex_D_UnlimitedGzip.java -./java/examples/datasets/H5Ex_D_UnlimitedMod.java -./java/examples/datasets/H5Ex_D_Nbit.java -./java/examples/datasets/H5Ex_D_Transform.java -./java/examples/datasets/H5Ex_D_Sofloat.java -./java/examples/datasets/H5Ex_D_Soint.java - -./java/examples/datatypes/Makefile.am -./java/examples/datatypes/CMakeLists.txt -./java/examples/datatypes/JavaDatatypeExample.sh.in -./java/examples/datatypes/H5Ex_T_Array.java -./java/examples/datatypes/H5Ex_T_ArrayAttribute.java -./java/examples/datatypes/H5Ex_T_Bit.java -./java/examples/datatypes/H5Ex_T_BitAttribute.java -./java/examples/datatypes/H5Ex_T_Commit.java -./java/examples/datatypes/H5Ex_T_Compound.java -./java/examples/datatypes/H5Ex_T_CompoundAttribute.java -./java/examples/datatypes/H5Ex_T_Float.java -./java/examples/datatypes/H5Ex_T_FloatAttribute.java -./java/examples/datatypes/H5Ex_T_Integer.java -./java/examples/datatypes/H5Ex_T_IntegerAttribute.java -./java/examples/datatypes/H5Ex_T_ObjectReference.java -./java/examples/datatypes/H5Ex_T_ObjectReferenceAttribute.java -./java/examples/datatypes/H5Ex_T_Opaque.java -./java/examples/datatypes/H5Ex_T_OpaqueAttribute.java -./java/examples/datatypes/H5Ex_T_String.java -./java/examples/datatypes/H5Ex_T_StringAttribute.java -./java/examples/datatypes/H5Ex_T_VLString.java - -./java/examples/testfiles/examples.intro.H5_CreateAttribute.txt -./java/examples/testfiles/examples.intro.H5_CreateDataset.txt -./java/examples/testfiles/examples.intro.H5_CreateFile.txt -./java/examples/testfiles/examples.intro.H5_CreateGroup.txt -./java/examples/testfiles/examples.intro.H5_CreateGroupAbsoluteRelative.txt -./java/examples/testfiles/examples.intro.H5_CreateGroupDataset.txt -./java/examples/testfiles/examples.intro.H5_ReadWrite.txt -./java/examples/testfiles/examples.groups.H5Ex_G_Create.txt -./java/examples/testfiles/examples.groups.H5Ex_G_Iterate.txt -./java/examples/testfiles/examples.groups.H5Ex_G_Compact.txt -./java/examples/testfiles/examples.groups.H5Ex_G_Corder.txt -./java/examples/testfiles/examples.groups.H5Ex_G_Intermediate.txt -./java/examples/testfiles/examples.groups.H5Ex_G_Phase.txt -./java/examples/testfiles/examples.groups.H5Ex_G_Visit.txt -./java/examples/testfiles/examples.datasets.H5Ex_D_Alloc.txt -./java/examples/testfiles/examples.datasets.H5Ex_D_Checksum.txt -./java/examples/testfiles/examples.datasets.H5Ex_D_Chunk.txt -./java/examples/testfiles/examples.datasets.H5Ex_D_Compact.txt -./java/examples/testfiles/examples.datasets.H5Ex_D_External.txt -./java/examples/testfiles/examples.datasets.H5Ex_D_FillValue.txt -./java/examples/testfiles/examples.datasets.H5Ex_D_Gzip.txt -./java/examples/testfiles/examples.datasets.H5Ex_D_Hyperslab.txt -./java/examples/testfiles/examples.datasets.H5Ex_D_ReadWrite.txt -./java/examples/testfiles/examples.datasets.H5Ex_D_Shuffle.txt -./java/examples/testfiles/examples.datasets.H5Ex_D_Szip.txt -./java/examples/testfiles/examples.datasets.H5Ex_D_UnlimitedAdd.txt -./java/examples/testfiles/examples.datasets.H5Ex_D_UnlimitedGzip.txt -./java/examples/testfiles/examples.datasets.H5Ex_D_UnlimitedMod.txt -./java/examples/testfiles/examples.datasets.H5Ex_D_Nbit.txt -./java/examples/testfiles/examples.datasets.H5Ex_D_Transform.txt -./java/examples/testfiles/examples.datasets.H5Ex_D_Sofloat.txt -./java/examples/testfiles/examples.datasets.H5Ex_D_Soint.txt -./java/examples/testfiles/examples.datatypes.H5Ex_T_Array.txt -./java/examples/testfiles/examples.datatypes.H5Ex_T_ArrayAttribute.txt -./java/examples/testfiles/examples.datatypes.H5Ex_T_Bit.txt -./java/examples/testfiles/examples.datatypes.H5Ex_T_BitAttribute.txt -./java/examples/testfiles/examples.datatypes.H5Ex_T_Commit.txt -./java/examples/testfiles/examples.datatypes.H5Ex_T_Compound.txt -./java/examples/testfiles/examples.datatypes.H5Ex_T_CompoundAttribute.txt -./java/examples/testfiles/examples.datatypes.H5Ex_T_Float.txt -./java/examples/testfiles/examples.datatypes.H5Ex_T_FloatAttribute.txt -./java/examples/testfiles/examples.datatypes.H5Ex_T_Integer.txt -./java/examples/testfiles/examples.datatypes.H5Ex_T_IntegerAttribute.txt -./java/examples/testfiles/examples.datatypes.H5Ex_T_ObjectReference.txt -./java/examples/testfiles/examples.datatypes.H5Ex_T_ObjectReferenceAttribute.txt -./java/examples/testfiles/examples.datatypes.H5Ex_T_Opaque.txt -./java/examples/testfiles/examples.datatypes.H5Ex_T_OpaqueAttribute.txt -./java/examples/testfiles/examples.datatypes.H5Ex_T_String.txt -./java/examples/testfiles/examples.datatypes.H5Ex_T_StringAttribute.txt -./java/examples/testfiles/examples.datatypes.H5Ex_T_VLString.txt - -./java/test/Makefile.am -./java/test/CMakeLists.txt -./java/test/junit.sh.in -./java/test/testfiles/JUnit-TestH5.txt -./java/test/testfiles/JUnit-TestH5A.txt -./java/test/testfiles/JUnit-TestH5Arw.txt -./java/test/testfiles/JUnit-TestH5Dparams.txt -./java/test/testfiles/JUnit-TestH5D.txt -./java/test/testfiles/JUnit-TestH5Dplist.txt -./java/test/testfiles/JUnit-TestH5Drw.txt -./java/test/testfiles/JUnit-TestH5E.txt -./java/test/testfiles/JUnit-TestH5Edefault.txt -./java/test/testfiles/JUnit-TestH5Eparams.txt -./java/test/testfiles/JUnit-TestH5Eregister.txt -./java/test/testfiles/JUnit-TestH5Fparams.txt -./java/test/testfiles/JUnit-TestH5Fbasic.txt -./java/test/testfiles/JUnit-TestH5F.txt -./java/test/testfiles/JUnit-TestH5Fswmr.txt -./java/test/testfiles/JUnit-TestH5Gbasic.txt -./java/test/testfiles/JUnit-TestH5G.txt -./java/test/testfiles/JUnit-TestH5Giterate.txt -./java/test/testfiles/JUnit-TestH5Lparams.txt -./java/test/testfiles/JUnit-TestH5Lbasic.txt -./java/test/testfiles/JUnit-TestH5Lcreate.txt -./java/test/testfiles/JUnit-TestH5Oparams.txt -./java/test/testfiles/JUnit-TestH5Obasic.txt -./java/test/testfiles/JUnit-TestH5Ocreate.txt -./java/test/testfiles/JUnit-TestH5Ocopy.txt -./java/test/testfiles/JUnit-TestH5OcopyOld.txt -./java/test/testfiles/JUnit-TestH5P.txt -./java/test/testfiles/JUnit-TestH5PData.txt -./java/test/testfiles/JUnit-TestH5Pfapl.txt -./java/test/testfiles/JUnit-TestH5Pfaplhdfs.txt -./java/test/testfiles/JUnit-TestH5Pfapls3.txt -./java/test/testfiles/JUnit-TestH5Plist.txt -./java/test/testfiles/JUnit-TestH5Pvirtual.txt -./java/test/testfiles/JUnit-TestH5PL.txt -./java/test/testfiles/JUnit-TestH5R.txt -./java/test/testfiles/JUnit-TestH5Rref.txt -./java/test/testfiles/JUnit-TestH5Sbasic.txt -./java/test/testfiles/JUnit-TestH5S.txt -./java/test/testfiles/JUnit-TestH5Tparams.txt -./java/test/testfiles/JUnit-TestH5Tbasic.txt -./java/test/testfiles/JUnit-TestH5T.txt -./java/test/testfiles/JUnit-TestH5VL.txt -./java/test/testfiles/JUnit-TestH5Z.txt -./java/test/h5ex_g_iterate.orig -./java/test/TestH5.java -./java/test/TestH5A.java -./java/test/TestH5Arw.java -./java/test/TestH5Dparams.java -./java/test/TestH5D.java -./java/test/TestH5Dplist.java -./java/test/TestH5Drw.java -./java/test/TestH5E.java -./java/test/TestH5Edefault.java -./java/test/TestH5Eparams.java -./java/test/TestH5Eregister.java -./java/test/TestH5Fparams.java -./java/test/TestH5Fbasic.java -./java/test/TestH5F.java -./java/test/TestH5Fswmr.java -./java/test/TestH5Gbasic.java -./java/test/TestH5G.java -./java/test/TestH5Giterate.java -./java/test/TestH5Lparams.java -./java/test/TestH5Lbasic.java -./java/test/TestH5Lcreate.java -./java/test/TestH5Oparams.java -./java/test/TestH5Obasic.java -./java/test/TestH5Ocreate.java -./java/test/TestH5Ocopy.java -./java/test/TestH5OcopyOld.java -./java/test/TestH5P.java -./java/test/TestH5PData.java -./java/test/TestH5Pfapl.java -./java/test/TestH5Pfaplhdfs.java -./java/test/TestH5Pfapls3.java -./java/test/TestH5Plist.java -./java/test/TestH5Pvirtual.java -./java/test/TestH5PL.java -./java/test/TestH5R.java -./java/test/TestH5Rref.java -./java/test/TestH5Sbasic.java -./java/test/TestH5S.java -./java/test/TestH5Tparams.java -./java/test/TestH5Tbasic.java -./java/test/TestH5T.java -./java/test/TestH5VL.java -./java/test/TestH5Z.java -./java/test/TestAll.java - -./java/lib/hamcrest-core.jar -./java/lib/junit.jar -./java/lib/simplelogger.properties -./java/lib/slf4j-api-1.7.33.jar -./java/lib/ext/slf4j-nop-1.7.33.jar -./java/lib/ext/slf4j-simple-1.7.33.jar - -# CMake-specific Files -./config/toolchain/build32.cmake -./config/toolchain/clang.cmake -./config/toolchain/crayle.cmake -./config/toolchain/GCC.cmake -./config/toolchain/intel.cmake -./config/toolchain/mingw64.cmake -./config/toolchain/PGI.cmake - -./config/cmake/cacheinit.cmake -./config/cmake/CMakeFindJavaCommon.cmake -./config/cmake/ConversionTests.c -./config/cmake/ConfigureChecks.cmake -./config/cmake/CPack.Info.plist.in -./config/cmake/CTestCustom.cmake -./config/cmake/fileCompareTest.cmake -./config/cmake/FindHDFS.cmake -./config/cmake/H5pubconf.h.in -./config/cmake/hdf5-config.cmake.in -./config/cmake/hdf5-config-version.cmake.in -./config/cmake/HDFCompilerFlags.cmake -./config/cmake/HDFCXXCompilerFlags.cmake -./config/cmake/HDFFortranCompilerFlags.cmake -./config/cmake/HDF5Macros.cmake -./config/cmake/HDF5PluginMacros.cmake -./config/cmake/HDF5PluginCache.cmake -./config/cmake/HDF5UseFortran.cmake -./config/cmake/javaTargets.cmake.in -./config/cmake/jrunTest.cmake -./config/cmake/libh5cc.in -./config/cmake/libhdf5.settings.cmake.in -./config/cmake/mccacheinit.cmake -./config/cmake/patch.xml -./config/cmake/PkgInfo.in -./config/cmake/README.md.cmake.in -./config/cmake/UseJava.cmake -./config/cmake/UseJavaClassFilelist.cmake -./config/cmake/UseJavaSymlinks.cmake -./config/cmake/userblockTest.cmake -./config/cmake/vfdTest.cmake -./config/cmake/volTest.cmake -./config/cmake/wait_H5Tinit.cmake - -./config/cmake_ext_mod/ConfigureChecks.cmake -./config/cmake_ext_mod/CTestCustom.cmake -./config/cmake_ext_mod/FindSZIP.cmake -./config/cmake_ext_mod/GetTimeOfDayTest.cpp -./config/cmake_ext_mod/grepTest.cmake -./config/cmake_ext_mod/hdf.bmp -./config/cmake_ext_mod/hdf.icns -./config/cmake_ext_mod/hdf.ico -./config/cmake_ext_mod/HDFLibMacros.cmake -./config/cmake_ext_mod/HDFMacros.cmake -./config/cmake_ext_mod/HDFTests.c -./config/cmake_ext_mod/HDFUseFortran.cmake -./config/cmake_ext_mod/NSIS.InstallOptions.ini.in -./config/cmake_ext_mod/NSIS.template.in -./config/cmake_ext_mod/runTest.cmake -./config/cmake_ext_mod/version.plist.in - -# CMake-specific User Files -./config/cmake/UserMacros/Windows_MT.cmake - -# CMake-specific Examples Files -./config/cmake/HDF5_Examples.cmake.in -./release_docs/USING_CMake_Examples.txt - -./CMakeLists.txt -./CMakeFilters.cmake -./CMakeInstallation.cmake -./CMakePlugins.cmake -./CTestConfig.cmake -./UserMacros.cmake -./c++/CMakeLists.txt -./c++/examples/CMakeLists.txt -./c++/examples/CMakeTests.cmake -./c++/src/CMakeLists.txt -./c++/test/CMakeLists.txt -./c++/test/CMakeTests.cmake -./c++/test/CMakeVFDTests.cmake -./examples/CMakeLists.txt -./examples/CMakeTests.cmake -./examples/run-all-ex.sh -./examples/run-c-ex.sh.in -./fortran/CMakeLists.txt -./fortran/examples/CMakeLists.txt -./fortran/examples/CMakeTests.cmake -./fortran/src/CMakeLists.txt -./fortran/test/CMakeLists.txt -./fortran/test/CMakeTests.cmake -./fortran/testpar/CMakeLists.txt -./fortran/testpar/CMakeTests.cmake -./hl/CMakeLists.txt -./hl/c++/CMakeLists.txt -./hl/c++/examples/CMakeLists.txt -./hl/c++/examples/CMakeTests.cmake -./hl/c++/src/CMakeLists.txt -./hl/c++/test/CMakeLists.txt -./hl/c++/test/CMakeTests.cmake -./hl/examples/CMakeLists.txt -./hl/examples/CMakeTests.cmake -./hl/fortran/CMakeLists.txt -./hl/fortran/examples/CMakeLists.txt -./hl/fortran/examples/CMakeTests.cmake -./hl/fortran/src/CMakeLists.txt -./hl/fortran/test/CMakeLists.txt -./hl/fortran/test/CMakeTests.cmake -./hl/src/CMakeLists.txt -./hl/test/CMakeLists.txt -./hl/test/CMakeTests.cmake -./hl/tools/CMakeLists.txt -./hl/tools/gif2h5/CMakeLists.txt -./hl/tools/gif2h5/CMakeTests.cmake -./hl/tools/h5watch/CMakeLists.txt -./hl/tools/h5watch/CMakeTests.cmake -./src/CMakeLists.txt -./test/CMakeLists.txt -./test/CMakePassthroughVOLTests.cmake -./test/CMakeTests.cmake -./test/CMakeVFDTests.cmake -./test/flushrefreshTest.cmake -./test/ShellTests.cmake -./testpar/CMakeLists.txt -./testpar/CMakeTests.cmake -./testpar/CMakeVFDTests.cmake -./tools/CMakeLists.txt -./tools/lib/CMakeLists.txt -./tools/libtest/CMakeLists.txt -./tools/libtest/CMakeTests.cmake -./tools/src/CMakeLists.txt -./tools/test/CMakeLists.txt -./tools/src/h5copy/CMakeLists.txt -./tools/test/h5copy/CMakeLists.txt -./tools/test/h5copy/CMakeTests.cmake -./tools/src/h5diff/CMakeLists.txt -./tools/test/h5diff/CMakeLists.txt -./tools/test/h5diff/CMakeTests.cmake -./tools/src/h5dump/CMakeLists.txt -./tools/test/h5dump/CMakeLists.txt -./tools/test/h5dump/CMakeTests.cmake -./tools/test/h5dump/CMakeTestsPBITS.cmake -./tools/test/h5dump/CMakeTestsXML.cmake -./tools/test/h5dump/CMakeTestsVDS.cmake -./tools/test/h5dump/CMakeVFDTests.cmake -./tools/src/h5format_convert/CMakeLists.txt -./tools/test/h5format_convert/CMakeLists.txt -./tools/test/h5format_convert/CMakeTests.cmake -./tools/src/h5import/CMakeLists.txt -./tools/test/h5import/CMakeLists.txt -./tools/test/h5import/CMakeTests.cmake -./tools/src/h5jam/CMakeLists.txt -./tools/test/h5jam/CMakeLists.txt -./tools/test/h5jam/CMakeTests.cmake -./tools/src/h5ls/CMakeLists.txt -./tools/test/h5ls/CMakeLists.txt -./tools/test/h5ls/CMakeTests.cmake -./tools/test/h5ls/CMakeTestsVDS.cmake -./tools/src/h5repack/CMakeLists.txt -./tools/test/h5repack/CMakeLists.txt -./tools/test/h5repack/CMakeTests.cmake -./tools/test/h5repack/CMakeVFDTests.cmake -./tools/src/h5stat/CMakeLists.txt -./tools/test/h5stat/CMakeLists.txt -./tools/test/h5stat/CMakeTests.cmake -./tools/src/misc/CMakeLists.txt -./tools/src/h5perf/CMakeLists.txt -./tools/test/misc/CMakeLists.txt -./tools/test/misc/CMakeTestsClear.cmake -./tools/test/misc/CMakeTestsMkgrp.cmake -./tools/test/misc/CMakeTestsRepart.cmake -./tools/test/misc/vds/CMakeLists.txt -./tools/test/perform/CMakeLists.txt -./tools/test/perform/CMakeTests.cmake - -./utils/CMakeLists.txt -./utils/mirror_vfd/CMakeLists.txt -./utils/test/CMakeLists.txt - -# CMake-specific User Scripts -./config/cmake/CTestScript.cmake -./config/cmake/HDF5_Examples_options.cmake -./config/cmake/scripts/CTestScript.cmake -./config/cmake/scripts/HDF5config.cmake -./config/cmake/scripts/HDF5options.cmake - -# CMake-specific Sanitizer Scripts -./config/sanitizer/code-coverage.cmake -./config/sanitizer/formatting.cmake -./config/sanitizer/sanitizers.cmake -./config/sanitizer/tools.cmake -./config/sanitizer/LICENSE -./config/sanitizer/README.md - -# CMake-specific HPC Scripts -./config/cmake/scripts/HPC/sbatch-HDF5options.cmake -./config/cmake/scripts/HPC/bsub-HDF5options.cmake -./config/cmake/scripts/HPC/qsub-HDF5options.cmake -./config/cmake/scripts/HPC/raybsub-HDF5options.cmake - -# Files generated by autogen -./aclocal.m4 -./bin/compile -./bin/config.guess -./bin/config.sub -./bin/depcomp -./bin/install-sh -./bin/ltmain.sh -./bin/missing -./bin/test-driver -./c++/examples/Makefile.in -./c++/Makefile.in -./configure -./bin/Makefile.in -./c++/src/Makefile.in -./c++/test/Makefile.in -./examples/Makefile.in -./fortran/examples/Makefile.in -./fortran/Makefile.in -./fortran/src/Makefile.in -./fortran/test/Makefile.in -./fortran/testpar/Makefile.in -./hl/c++/examples/Makefile.in -./hl/c++/Makefile.in -./hl/c++/src/Makefile.in -./hl/c++/test/Makefile.in -./hl/examples/Makefile.in -./hl/fortran/examples/Makefile.in -./hl/fortran/Makefile.in -./hl/fortran/src/Makefile.in -./hl/fortran/test/Makefile.in -./hl/Makefile.in -./hl/src/Makefile.in -./hl/test/Makefile.in -./hl/tools/gif2h5/Makefile.in -./hl/tools/h5watch/Makefile.in -./hl/tools/Makefile.in -./java/examples/intro/Makefile.in -./java/examples/datasets/Makefile.in -./java/examples/datatypes/Makefile.in -./java/examples/Makefile.in -./java/examples/groups/Makefile.in -./java/Makefile.in -./java/test/Makefile.in -./java/src/Makefile.in -./java/src/jni/Makefile.in -./m4/libtool.m4 -./m4/lt~obsolete.m4 -./m4/ltoptions.m4 -./m4/ltsugar.m4 -./m4/ltversion.m4 -./Makefile.in -./src/H5config.h.in -./src/H5Edefin.h -./src/H5Einit.h -./src/H5Epubgen.h -./src/H5Eterm.h -./src/H5overflow.h -./src/H5version.h -./src/Makefile.in -./test/Makefile.in -./testpar/Makefile.in -./tools/Makefile.in -./tools/lib/Makefile.in -./tools/libtest/Makefile.in -./tools/src/Makefile.in -./tools/src/h5copy/Makefile.in -./tools/src/h5diff/Makefile.in -./tools/src/h5dump/Makefile.in -./tools/src/h5format_convert/Makefile.in -./tools/src/h5import/Makefile.in -./tools/src/h5jam/Makefile.in -./tools/src/h5ls/Makefile.in -./tools/src/h5perf/Makefile.in -./tools/src/h5repack/Makefile.in -./tools/src/h5stat/Makefile.in -./tools/src/misc/Makefile.in -./tools/test/Makefile.in -./tools/test/h5copy/Makefile.in -./tools/test/h5diff/Makefile.in -./tools/test/h5dump/Makefile.in -./tools/test/h5format_convert/Makefile.in -./tools/test/h5import/Makefile.in -./tools/test/h5jam/Makefile.in -./tools/test/h5ls/Makefile.in -./tools/test/h5repack/Makefile.in -./tools/test/h5stat/Makefile.in -./tools/test/misc/Makefile.in -./tools/test/misc/vds/Makefile.in -./tools/test/perform/Makefile.in -./utils/Makefile.in -./utils/mirror_vfd/Makefile.in -./utils/test/Makefile.in -./utils/tools/Makefile.in -./utils/tools/h5dwalk/Makefile.in -./utils/tools/test/Makefile.in -./utils/tools/test/h5dwalk/Makefile.in diff --git a/bin/bbrelease b/bin/bbrelease index a45be2e..d056f6d 100755 --- a/bin/bbrelease +++ b/bin/bbrelease @@ -14,32 +14,13 @@ # Make a release of hdf5. # -# Programmer: Robb Matzke -# Creation date: on or before 1998-01-29. +# NOTE: +# This script differs from bin/release in that this has an added +# --revision option to create private releases with the code revision +# hash in the version strings. # -# Modifications -# Robb Matzke, 1999-07-16 -# The SunOS 5.6 sed *must* have slashes as delimiters. I changed things like -# `sed s+/CVS++' to `sed 's/\/CVS//' -# -# Albert Cheng, 1999-10-26 -# Moved the MANIFEST checking to a separate command file so that -# it can be invoked individually. -# -# Albert Cheng, 2004-08-14 -# Added the --private option. -# -# James Laird, 2005-09-07 -# Added the md5 method. -# -# Larry Knox, 2016-08-30 -# Added the --revision option to create private releases with the -# code revision hash in the version strings. Currently the version -# of this script with the --revision option is named bbrelease. It -# can probably be merged into the original release script in the -# future. -# Commands to get the revision hash have now been converted to git -# to match the source repository change. +# This script can probably be merged into the original release script in +# the future. # Function definitions # @@ -47,26 +28,25 @@ USAGE() { cat << EOF -Usage: $0 -d [--docver BRANCHNAME] [-h] [--nocheck] [--private] ... - -d DIR The name of the directory where the release(s) should be +Usage: $0 -d [--docver BRANCHNAME] [-h] [--private] [--revision] ... + -d DIR The name of the directory where the release(s) should be placed. --docver BRANCHNAME This is added for 1.8 and beyond to get the correct version of documentation files from the hdf5docs repository. BRANCHNAME for v1.8 should be hdf5_1_8. -h print the help page. - --nocheck Ignore errors in MANIFEST file. - --private Make a private release with today's date in version information. - --revision Make a private release with the code revision number in version information. - + --private Make a private release with today's date in version information. + --revision Make a private release with the code revision number in version information. + This must be run at the top level of the source directory. The other command-line options are the names of the programs to use for compressing the resulting tar archive (if none are given then "tar" is assumed): - tar -- use tar and don't do any compressing. - gzip -- use gzip with "-9" and append ".gz" to the output name. + tar -- use tar and don't do any compressing. + gzip -- use gzip with "-9" and append ".gz" to the output name. bzip2 -- use bzip2 with "-9" and append ".bz2" to the output name. - zip -- convert all text files to DOS style and form a zip file for Windows use. + zip -- convert all text files to DOS style and form a zip file for Windows use. doc -- produce the latest doc tree in addition to the archive. An md5 checksum is produced for each archive created and stored in the md5 file. @@ -97,15 +77,10 @@ EOF # Function name: tar2zip # Convert the release tarball to a Windows zipball. # -# Programmer: Albert Cheng -# Creation date: 2014-04-23 -# -# Modifications -# # Steps: # 1. untar the tarball in a temporary directory; # Note: do this in a temporary directory to avoid changing -# the original source directory which maybe around. +# the original source directory which may be around. # 2. convert all its text files to DOS (LF-CR) style; # 3. form a zip file which is usable by Windows users. # @@ -119,8 +94,8 @@ EOF tar2zip() { if [ $# -ne 3 ]; then - echo "usage: tar2zip " - return 1 + echo "usage: tar2zip " + return 1 fi ztmpdir=/tmp/tmpdir$$ mkdir -p $ztmpdir @@ -132,23 +107,23 @@ tar2zip() (cd $ztmpdir; tar xf -) < $tarfile # sanity check if [ ! -d $ztmpdir/$version ]; then - echo "untar did not create $ztmpdir/$version source dir" - # cleanup - rm -rf $ztmpdir - return 1 + echo "untar did not create $ztmpdir/$version source dir" + # cleanup + rm -rf $ztmpdir + return 1 fi # step 2: convert text files # There maybe a simpler way to do this. # options used in unix2dos: - # -k Keep the date stamp + # -k Keep the date stamp # -q quiet mode # grep redirect output to /dev/null because -q or -s are not portable. find $ztmpdir/$version | \ - while read inf; do \ - if file $inf | grep "$inf\: .*text" > /dev/null 2>&1 ; then \ - unix2dos -q -k $inf; \ - fi\ - done + while read inf; do \ + if file $inf | grep "$inf\: .*text" > /dev/null 2>&1 ; then \ + unix2dos -q -k $inf; \ + fi\ + done # step 3: make zipball # -9 maximum compression # -y Store symbolic links as such in the zip archive @@ -163,14 +138,6 @@ tar2zip() # This command must be run at the top level of the hdf5 source directory. # Verify this requirement. -# Since we are running bbrelease to create an HDF5 source tarfile for buildbot -# testing with source that is not for release, there is not a file named -# "configure" but there will be one named "configure.ac". The "configure" -# file will be created when autogen.sh runs. There probably will always -# be a bin/release file, but just in case it is removed, we can check for -# this script, bbrelease, in the bin directory. The bin/release script should -# continue to check for "configure" because it should be present in release -# source. if [ ! \( -f configure.ac -a -f bin/bbrelease \) ]; then echo "$0 must be run at the top level of the hdf5 source directory" exit 1 @@ -182,12 +149,11 @@ VERS=`perl bin/h5vers` VERS_OLD= test "$VERS" || exit 1 verbose=yes -check=yes release_date=`date +%F` today=`date +%Y%m%d` pmode='no' revmode='no' -tmpdir="../#release_tmp.$$" # tmp work directory +tmpdir="../#release_tmp.$$" # tmp work directory DOC_URL=https://git@bitbucket.hdfgroup.org/scm/hdffv/hdf5doc.git CPPLUS_RM_NAME=cpplus_RM @@ -195,11 +161,11 @@ CPPLUS_RM_NAME=cpplus_RM RESTORE_VERSION() { if [ X-${VERS_OLD} != X- ]; then - echo restoring version information back to $VERS_OLD - rm -f config/lt_vers.am - cp $tmpdir/lt_vers.am config/lt_vers.am - bin/h5vers -s $VERS_OLD - VERS_OLD= + echo restoring version information back to $VERS_OLD + rm -f config/lt_vers.am + cp $tmpdir/lt_vers.am config/lt_vers.am + bin/h5vers -s $VERS_OLD + VERS_OLD= fi } @@ -209,20 +175,17 @@ while [ -n "$1" ]; do arg=$1 shift case "$arg" in - -d) - DEST=$1 - shift - ;; - --nocheck) - check=no - ;; - -h) - USAGE - exit 0 - ;; - --private) - pmode=yes - ;; + -d) + DEST=$1 + shift + ;; + -h) + USAGE + exit 0 + ;; + --private) + pmode=yes + ;; --revision) revmode=yes ;; @@ -230,14 +193,14 @@ while [ -n "$1" ]; do DOCVERSION=$1 shift ;; - -*) - echo "Unknown switch: $arg" 1>&2 - USAGE - exit 1 - ;; - *) - methods="$methods $arg" - ;; + -*) + echo "Unknown switch: $arg" 1>&2 + USAGE + exit 1 + ;; + *) + methods="$methods $arg" + ;; esac done @@ -278,7 +241,7 @@ if [ X$revmode = Xyes ]; then cp config/lt_vers.am $tmpdir branch=`git branch | grep '*' | awk '{print $NF}'` revision=`git rev-parse --short HEAD` -# Set version information to m.n.r-r$revision. + # Set version information to m.n.r-r$revision. # (h5vers does not correctly handle just m.n.r-$today.) VERS=`echo $VERS | sed -e s/-.*//`-$revision echo Private release of $VERS @@ -299,23 +262,10 @@ if [ ! -d $DEST ]; then exit 1 fi -# Check the validity of the MANIFEST file. -bin/chkmanifest || fail=yes -if [ "X$fail" = "Xyes" ]; then - if [ $check = yes ]; then - exit 1 - else - echo "Continuing anyway..." - fi -fi - -# Create a manifest that contains only files for distribution. -MANIFEST=$tmpdir/H5_MANIFEST -grep '^\.' MANIFEST | grep -v _DO_NOT_DISTRIBUTE_ >$MANIFEST - -# Prepare the source tree for a release. -#ln -s `pwd` $tmpdir/$HDF5_VERS || exit 1 +# Create a symlink to the source so files in the tarball have the prefix +# we want (gnu's --transform isn't portable) ln -s `pwd` $tmpdir/$HDF5_IN_VERS || exit 1 + # Save a backup copy of Makefile if exists. test -f Makefile && mv Makefile $tmpdir/Makefile.x cp -p Makefile.dist Makefile @@ -330,64 +280,60 @@ for f in README.md release_docs/RELEASE.txt; do chmod 644 $f done -# trunk is different than branches. +# develop is different than branches. if [ "${DOCVERSION}" ]; then DOC_URL=https://git@bitbucket.hdfgroup.org/scm/hdffv/hdf5doc.git -b ${DOCVERSION} fi # Create the tar file test "$verbose" && echo " Running tar..." 1>&2 -( \ - cd $tmpdir; \ - tar cf $HDF5_VERS.tar $HDF5_IN_VERS/Makefile \ - `sed 's/^\.\//'$HDF5_IN_VERS'\//' $MANIFEST` || exit 1 \ -) +(cd "$tmpdir" && exec tar -ch --exclude-vcs -f "$HDF5_VERS.tar" "./$HDF5_IN_VERS" || exit 1 ) # Compress MD5file=$HDF5_VERS.md5 cp /dev/null $DEST/$MD5file for comp in $methods; do case $comp in - tar) - cp -p $tmpdir/$HDF5_VERS.tar $DEST/$HDF5_VERS.tar - (cd $DEST; md5sum $HDF5_VERS.tar >> $MD5file) - ;; - gzip) - test "$verbose" && echo " Running gzip..." 1>&2 - gzip -9 <$tmpdir/$HDF5_VERS.tar >$DEST/$HDF5_VERS.tar.gz - (cd $DEST; md5sum $HDF5_VERS.tar.gz >> $MD5file) - ;; - bzip2) - test "$verbose" && echo " Running bzip2..." 1>&2 - bzip2 -9 <$tmpdir/$HDF5_VERS.tar >$DEST/$HDF5_VERS.tar.bz2 - (cd $DEST; md5sum $HDF5_VERS.tar.bz2 >> $MD5file) - ;; - zip) - test "$verbose" && echo " Creating zip ball..." 1>&2 - tar2zip $HDF5_IN_VERS $tmpdir/$HDF5_VERS.tar $DEST/$HDF5_VERS.zip 1>&2 - (cd $DEST; md5sum $HDF5_VERS.zip >> $MD5file) - ;; - doc) + tar) + cp -p $tmpdir/$HDF5_VERS.tar $DEST/$HDF5_VERS.tar + (cd $DEST; md5sum $HDF5_VERS.tar >> $MD5file) + ;; + gzip) + test "$verbose" && echo " Running gzip..." 1>&2 + gzip -9 <$tmpdir/$HDF5_VERS.tar >$DEST/$HDF5_VERS.tar.gz + (cd $DEST; md5sum $HDF5_VERS.tar.gz >> $MD5file) + ;; + bzip2) + test "$verbose" && echo " Running bzip2..." 1>&2 + bzip2 -9 <$tmpdir/$HDF5_VERS.tar >$DEST/$HDF5_VERS.tar.bz2 + (cd $DEST; md5sum $HDF5_VERS.tar.bz2 >> $MD5file) + ;; + zip) + test "$verbose" && echo " Creating zip ball..." 1>&2 + tar2zip $HDF5_IN_VERS $tmpdir/$HDF5_VERS.tar $DEST/$HDF5_VERS.zip 1>&2 + (cd $DEST; md5sum $HDF5_VERS.zip >> $MD5file) + ;; + doc) if [ "${DOCVERSION}" = "" ]; then DOCVERSION=master fi - test "$verbose" && echo " Creating docs..." 1>&2 - # Check out docs from git repo - (cd $tmpdir; git clone $DOC_URL > /dev/null) || exit 1 - # Create doxygen C++ RM - (cd c++/src && doxygen cpp_doc_config > /dev/null ) || exit 1 - # Replace version of C++ RM with just-created version + test "$verbose" && echo " Creating docs..." 1>&2 + # Check out docs from git repo + (cd $tmpdir; git clone $DOC_URL > /dev/null) || exit 1 + # Create doxygen C++ RM + (cd c++/src && doxygen cpp_doc_config > /dev/null ) || exit 1 + # Replace version of C++ RM with just-created version rm -rf $tmpdir/${DOCVERSION}/html/$CPPLUS_RM_NAME mv c++/src/$CPPLUS_RM_NAME $tmpdir/${DOCVERSION}/html/$CPPLUS_RM_NAME # Compress the docs and move them to the release area mv $tmpdir/$DOCVERSION $tmpdir/${HDF5_VERS}_docs - (cd $tmpdir && tar cf ${HDF5_VERS}_docs.tar ${HDF5_VERS}_docs) - mv $tmpdir/${HDF5_VERS}_docs.tar $DEST - ;; - *) - echo "***Error*** Unknown method $comp" - exit 1 - ;; + (cd $tmpdir && tar cf ${HDF5_VERS}_docs.tar ${HDF5_VERS}_docs) + mv $tmpdir/${HDF5_VERS}_docs.tar $DEST + ;; + *) + echo "***Error*** Unknown method $comp" + exit 1 + ;; esac done @@ -408,4 +354,6 @@ fi # Remove temporary things rm -rf $tmpdir +echo "DONE" + exit 0 diff --git a/bin/chkmanifest b/bin/chkmanifest deleted file mode 100755 index 08ca1fa..0000000 --- a/bin/chkmanifest +++ /dev/null @@ -1,154 +0,0 @@ -#!/bin/sh -# -# Copyright by The HDF Group. -# Copyright by the Board of Trustees of the University of Illinois. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. -# - -# Check that all the files in MANIFEST exist and (if this is a -# GIT checkout) that all the GIT-managed files appear in the -# MANIFEST. -# - -verbose=yes -MANIFEST=/tmp/HD_MANIFEST.$$ -AUTOGEN=./autogen.sh -AUTOGEN_LOG=/tmp/autogen.log.$$ - -# Main -test "$verbose" && echo " Checking MANIFEST..." 1>&2 -# clean up $MANIFEST file when exits -trap "rm -f $MANIFEST" 0 - -# Only split lines on newline, not whitespace -set -f -IFS=' -' - -# First make sure i am in the directory in which there is an MANIFEST file -# and then do the checking from there. Will try the following, -# current directory, parent directory, the directory this command resides. -if [ -f MANIFEST ]; then - continue -elif [ -f ../MANIFEST ]; then - cd .. -else - commanddir=`dirname $0` - if [ -d "$commanddir" -a -f $commanddir/MANIFEST ]; then - cd $commanddir - continue - else - echo MANIFEST file not found. Abort. - exit 1 - fi -fi - -# Do an autogen if generated files (e.g., configure) is not present -if [ ! -f configure ]; then - echo " running $AUTOGEN" - $AUTOGEN > $AUTOGEN_LOG 2>&1 - if [ $? -ne 0 ]; then - echo $AUTOGEN encountered error. Abort. - echo output from $AUTOGEN: - cat $AUTOGEN_LOG - exit 1 - fi - rm $AUTOGEN_LOG -fi - -# Check for duplicate entries. This can be done at any time, but it may as -# well be sooner so that if something else fails the presence of duplicates -# will already be known. -errcode=0 -DUPLICATES=`perl -ne 's/#.*//; next if /^\s*$/; if ($uniq{$_}++) { print $_; }' MANIFEST` -if [ "$DUPLICATES" ]; then - cat 1>&2 <$MANIFEST - -for file in `cat $MANIFEST`; do - if [ ! -f $file ]; then - echo "- $file" - fail=yes - fi -done - -# Get the list of files under version control and check that they are -# present. -# -# First get a list of all the pending files with git status and -# check those. -git_stat=`git status -s` -for file in $git_stat; do - - # Newly added files are not listed by git ls-files, which - # we check later. - - # The line listing new files starts with 'A'. - letter=`echo $file | head -c 1` - if [ "$letter" = "A" ]; then - # Convert the git status columns to './' so it matches - # the manifest file name. - # - # There is a space between the status columns and file name, hence - # the '3'. - path=`echo $file | sed 's/^.\{3\}/\.\//g'` - # Ignore directories - if [ ! -d $path ]; then - if (grep ^$path$ $MANIFEST >/dev/null); then - : - else - echo "- $path" - fail=yes - fi - fi - fi -done - -# Next check git ls-files, which gets a list of all files that are -# checked in. -git_ls=`git ls-files` -for file in $git_ls; do - path="./${file}" - # Ignore directories - if [ ! -d $path ]; then - if (grep ^$path$ $MANIFEST >/dev/null); then - : - else - echo "+ $path" - fail=yes - fi - fi -done - -# Finish up -if [ "X$fail" = "Xyes" ]; then - cat 1>&2 <&2 -exit 0 diff --git a/bin/locate_sw b/bin/locate_sw deleted file mode 100755 index 1cf84e2..0000000 --- a/bin/locate_sw +++ /dev/null @@ -1,238 +0,0 @@ -#!/bin/sh -# -# Copyright by The HDF Group. -# Copyright by the Board of Trustees of the University of Illinois. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. -# - -# Try to locate the software as named in argument. -# This is a sequential search of all possible locations of the software. -# Usage: locate_sw -# It prints a string showing the paths leading to the include, lib and bin -# directory of the software, separated by colons. E.g., if the software is -# located in /usr/sdt/*, it prints -# /usr/sdt/include:/usr/sdt/lib:/usr/sdt/bin -# Any component that is not found will be returned as an empty string. E.g., -# if somehow the header files of the software are not found, it prints -# :/usr/sdt/lib;/usr/sdt/bin - -# Function definitions -USAGE() -{ - echo "Usage: locate_sw " - echo " where can be hdf4, hdf5, zlib" - echo " It prints the paths leading the header files (include)," - echo " library (lib), and tools (bin). E.g.," - echo " /usr/sdt/include:/usr/sdt/lib:/usr/sdt/bin" - echo " Any component that is not found will be returned as an empty string. E.g.," - echo " if somehow the header files of the software are not found, it prints" - echo " :/usr/sdt/lib;/usr/sdt/bin" - echo "Exit code: 0 if software located; otherwise non-zero" -} - -# locate hdf4 software -locate_hdf4() -{ -# this default is the best guess of locating hdf4 software -swpaths_defaults="/usr/ncsa /usr/sdt /usr/local" -swpaths= - -case "$OSname" in - SunOS) - case "$OSrelease" in - 5.7) - swpaths="/afs/ncsa/packages/hdf/SunOS_5.7" - ;; - *) - # use default - ;; - esac - ;; - HP-UX) - case "$OSrelease" in - B.11.00) - swpaths="/afs/ncsa/packages/hdf/HPUX_11.00" - ;; - *) - # use default - ;; - esac - ;; - Linux) - swpaths="/afs/ncsa/packages/hdf/Linux" - ;; - OSF1) - swpaths="/afs/ncsa/packages/hdf/OSF1_V4.0" - ;; - *) - # just use the defaults - ;; -esac - -# Check if the hdf4 software is actually available. -# Accept the directory only if needed .h, .a and tools are found -# in the same place. That way, they are more likely to be of the -# same version. -# -swpaths="$swpaths $swpaths_defaults" -for sw in $swpaths; do - if [ -r $sw/include/hdf.h -a -r $sw/lib/libdf.a -a -r $sw/bin/hdp ]; then - SW_inc=$sw/include - SW_lib=$sw/lib - SW_bin=$sw/bin - SW_Location=$sw - break - fi -done -} - -# locate hdf5 software -locate_hdf5() -{ -# this default is the best guess of locating hdf5 software -swpaths_defaults="/usr/ncsa /usr/sdt /usr/local" -swpaths= - -case "$OSname" in - SunOS) - case "$OSrelease" in - 5.7) - swpaths="/afs/ncsa/packages/hdf5/SunOS_5.7" - ;; - *) - # use default - ;; - esac - ;; - HP-UX) - case "$OSrelease" in - B.11.00) - swpaths="/afs/ncsa/packages/hdf5/HPUX_11.00" - ;; - *) - # use default - ;; - esac - ;; - Linux) - swpaths="/afs/ncsa/packages/hdf5/Linux" - ;; - FreeBSD) - swpaths="/afs/ncsa/packages/hdf5/FreeBSD" - ;; - OSF1) - swpaths="/afs/ncsa/packages/hdf5/OSF1_V4.0" - ;; - *) - # just use the defaults - ;; -esac - -# Check if the hdf5 software is actually available. -# Accept the directory only if needed .h, .a and tools are found -# in the same place. That way, they are more likely to be of the -# same version. -# -swpaths="$swpaths $swpaths_defaults" -for sw in $swpaths; do - if [ -r $sw/include/hdf5.h -a -r $sw/lib/libhdf5.a -a -r $sw/bin/h5dump ]; then - SW_inc=$sw/include - SW_lib=$sw/lib - SW_bin=$sw/bin - SW_Location=$sw - break - fi -done -} - -# locate zlib software -locate_zlib() -{ -# this default is the best guess of locating zlib software -swpaths_defaults="/usr /usr/local /usr/ncsa /usr/sdt" -swpaths= - - -# Check if the zlib software is actually available. -# Accept the directory only if needed .h, .a and tools are found -# in the same place. That way, they are more likely to be of the -# same version. -# Don't know something specific to check the bin directory. Maybe gzip? -# Just make sure it exists. -# -swpaths="$swpaths $swpaths_defaults" -for sw in $swpaths; do - if [ -r $sw/include/zlib.h -a \ - \( -r $sw/lib/libz.a -o -r $sw/lib/libz.so \) -a -d $cw/bin ]; then - SW_inc=$sw/include - SW_lib=$sw/lib - SW_bin=$sw/bin - SW_Location=$sw - break - fi -done - -# if none found, try HDF4 software which contains a version of zlib. -if [ x-$SW_Location = x- ]; then - locate_hdf4 -fi - -} - -# Main -# -# Options -# -if [ $# -lt 1 ]; then - USAGE - exit 1 -fi - -if [ "$1" = -h ]; then - USAGE - exit 0 -fi - -SW=$1 -shift - -# locations of the software seeked. -SW_inc= # include place -SW_lib= # library place -SW_bin= # binary place -SW_Location= # parent directory of all the above - -OSname=`uname -s` -OSrelease=`uname -r` - -case $SW in -hdf4|hdf) - locate_hdf4 - ;; -hdf5) - locate_hdf5 - ;; -zlib) - locate_zlib - ;; -*) - echo "unknown software ($SW)" - USAGE - exit 1 - ;; -esac - -# show the results located, separated by commas. -if [ -n "${SW_inc}" -a -n "${SW_lib}" -a -n "${SW_bin}" ]; then - echo ${SW_inc},${SW_lib},${SW_bin} - exit 0 -else - exit 1 -fi diff --git a/bin/release b/bin/release index e71a38e..84555b6 100755 --- a/bin/release +++ b/bin/release @@ -13,24 +13,6 @@ # # Make a release of hdf5. -# -# Programmer: Robb Matzke -# Creation date: on or before 1998-01-29. -# -# Modifications -# Robb Matzke, 1999-07-16 -# The SunOS 5.6 sed *must* have slashes as delimiters. I changed things like -# `sed s+/CVS++' to `sed 's/\/CVS//' -# -# Albert Cheng, 1999-10-26 -# Moved the MANIFEST checking to a separate command file so that -# it can be invoked individually. -# -# Albert Cheng, 2004-08-14 -# Added the --private option. -# -# James Laird, 2005-09-07 -# Added the md5 method. # Function definitions # @@ -38,14 +20,13 @@ USAGE() { cat << EOF -Usage: $0 -d [--docver BRANCHNAME] [-h] [--nocheck] [--private] ... - -d DIR The name of the directory where the release(es) should be +Usage: $0 -d [--docver BRANCHNAME] [-h] [--private] ... + -d DIR The name of the directory where the release(s) should be placed. --docver BRANCHNAME This is added for 1.8 and beyond to get the correct version of documentation files from the hdf5docs repository. BRANCHNAME for v1.8 should be hdf5_1_8. -h print the help page. - --nocheck Ignore errors in MANIFEST file. --private Make a private release with today's date in version information. This must be run at the top level of the source directory. @@ -100,11 +81,6 @@ EOF # Function name: tar2zip # Convert the release tarball to a Windows zipball. # -# Programmer: Albert Cheng -# Creation date: 2014-04-23 -# -# Modifications -# # Steps: # 1. untar the tarball in a temporary directory; # Note: do this in a temporary directory to avoid changing @@ -167,11 +143,6 @@ tar2zip() # Function name: tar2cmakezip # Convert the release tarball to a Windows zipball with files to run CMake build. # -# Programmer: Larry Knox -# Creation date: 2017-02-20 -# -# Modifications -# # Steps: # 1. untar the tarball in a temporary directory; # Note: do this in a temporary directory to avoid changing @@ -269,10 +240,6 @@ tar2cmakezip() # Function name: tar2cmaketgz # Convert the release tarball to a gzipped tar file with files to run CMake build. # -# Programmer: Larry Knox -# Creation date: 2017-02-20 -# -# Modifications # # Steps: # 1. untar the tarball in a temporary directory; @@ -347,11 +314,6 @@ tar2cmaketgz() # and HDF5options.cmake files for parallel or serial only builds where build # tests are run on compute nodes using batch scripts. # -# Programmer: Larry Knox -# Creation date: 2019-01-28 -# -# Modifications -# # Steps: # 1. untar the tarball in a temporary directory; # Note: do this in a temporary directory to avoid changing @@ -443,7 +405,6 @@ VERS=`perl bin/h5vers` VERS_OLD= test "$VERS" || exit 1 verbose=yes -check=yes release_date=`date +%F` today=`date +%Y%m%d` pmode='no' @@ -482,9 +443,6 @@ while [ -n "$1" ]; do DEST=$1 shift ;; - --nocheck) - check=no - ;; -h) USAGE exit 0 @@ -546,28 +504,10 @@ if [ ! -d $DEST ]; then exit 1 fi -# Check the validity of the MANIFEST file. -bin/chkmanifest || fail=yes -if [ "X$fail" = "Xyes" ]; then - if [ $check = yes ]; then - echo "" - echo "Note! If you are running bin/release in a development branch" - echo "later than v 1.8 the MANIFEST check is expected to fail when" - echo "autogen.sh has not been run successfully. Either run autogen.sh " - echo "with /usr/hdf/bin/AUTOTOOLS at the beginning of PATH or add the" - echo "--nocheck argument to the bin/release command." - exit 1 - else - echo "Continuing anyway..." - fi -fi - -# Create a manifest that contains only files for distribution. -MANIFEST=$tmpdir/H5_MANIFEST -grep '^\.' MANIFEST | grep -v _DO_NOT_DISTRIBUTE_ >$MANIFEST - -# Prepare the source tree for a release. +# Create a symlink to the source so files in the tarball have the prefix +# we want (gnu's --transform isn't portable) ln -s `pwd` $tmpdir/$HDF5_VERS || exit 1 + # Save a backup copy of Makefile if exists. test -f Makefile && mv Makefile $tmpdir/Makefile.x cp -p Makefile.dist Makefile @@ -582,18 +522,14 @@ for f in README.md release_docs/RELEASE.txt; do chmod 644 $f done -# trunk is different than branches. +# develop is different than branches. if [ "${DOCVERSION}" ]; then DOC_URL="$DOC_URL -b ${DOCVERSION}" fi # Create the tar file test "$verbose" && echo " Running tar..." 1>&2 -( \ - cd $tmpdir; \ - tar cf $HDF5_VERS.tar $HDF5_VERS/Makefile \ - `sed 's/^\.\//hdf5-'$VERS'\//' $MANIFEST` || exit 1 \ -) +(cd "$tmpdir" && exec tar -ch --exclude-vcs -f "$HDF5_VERS.tar" "./$HDF5_VERS" || exit 1 ) # Compress MD5file=$HDF5_VERS.md5 @@ -680,4 +616,6 @@ fi # Remove temporary things rm -rf $tmpdir +echo "DONE" + exit 0 diff --git a/bin/runtest b/bin/runtest deleted file mode 100755 index 5e05abb..0000000 --- a/bin/runtest +++ /dev/null @@ -1,966 +0,0 @@ -#! /bin/sh -# -# Copyright by The HDF Group. -# Copyright by the Board of Trustees of the University of Illinois. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. -# - -# run the hdf5/bin/snapshot -# Usage: -# runtest run the test for the local host -# runtest run the test for -# runtest -all run the test for all predefined hosts -# -# Assumptions in knowing where to find the right scripts to execute. -# 1. assume we are at the top level of the hdf5 source. So, bin/* are -# where the script files are. -# 2. after the cvs update is completed, we can go to the snapshot area -# hdf5 source tree and use the bin/* there. -# 3. Cannot use the snapshot area scripts from the beginning because -# for one, the current directory is renamed as previous right after -# a snapshot release; and for another, some scripts may be changed -# by the cvs update while it is being used. - -# local setup -DEBUGMODE="" -test -n "$DEBUGMODE" && echo "******** DEBUGMODE is $DEBUGMODE ************" -WHEREAMI='pwd' -CMD= - -# the name of this program -PROGNAME="bin/runtest $DEBUGMODE" - -# Setup -HOSTNAME=`hostname | cut -f1 -d.` # no domain part -TODAY=`date +%m%d%a` -WEEKDAY=`date +%a` -H5VER= # default to current CVS version -H5VERSION= # default to current CVS version -n_test=0 # Number of tests ran -n_pass=0 # Number of tests passed -n_fail=0 # Number of tests failed -n_skip=0 # Number of tests skipped - -# Default to do checkout (only once) and test, no release. -# Will run test only if there is significant differences from previous version. -# If srcdir is not used, don't launched multiple tests -SNAPSHOT="${DEBUGMODE:+echo }bin/snapshot" -SRCDIR="srcdir" -# Default standard Snaptest commands -SNAPCMD="$SRCDIR test clean" -# Default Standard snaptest command options -STANDARD_OPT="" -ENABLE_PARALLEL="--enable-parallel" -CONFIGNAME=$HOSTNAME # Name used in the SNAPTESTCFG file - -# test host default as local host. -TESTHOST="" - -################################# -# Function definitions -################################# - -# Print messages to stdout -# Use this to show output heading to stdout -PRINT() -{ - echo "$*" -} - -# Show seconds since midnight. -# This is used to calculate seconds elapsed -SecOfDay() -{ - set `date '+%H %M %S'` - t_sec=`expr $1 \* 3600 + $2 \* 60 + $3` - echo $t_sec -} - -# Calculated the elapsed time (in seconds) between the first -# and second time. If second time is smaller than the first, -# we assume the clock has passed midnight and calculate appropriately. -ElapsedTime() -{ - if [ $2 -lt $1 ]; then - t_sec=`expr 3600 \* 24 - $1 + $2` - else - t_sec=`expr $2 - $1` - fi - echo `expr $t_sec / 60`m `expr $t_sec % 60`s -} - -# Report errors -# $1--an error message to be printed -REPORT_ERR() -{ - ERRMSG=$1 - # print it with a banner shifted right a bit - PRINT " *************************************" - PRINT " `date`" - PRINT " $ERRMSG" - PRINT " *************************************" - # report it in the FAILED-LOG file too - PRINT "$ERRMSG" >> $FAILEDLOG -} - -# -# Report results of the last test done -REPORT_RESULT() -{ - if [ $retcode -eq 0 ]; then - if [ $skiptest = yes ]; then - n_skip=`expr $n_skip + 1` - PRINT "SKIPPED ${HOSTNAME}: $TEST_TYPE" | tee -a $SKIPPEDLOG - else - n_pass=`expr $n_pass + 1` - PRINT "PASSED ${HOSTNAME}: $TEST_TYPE" | tee -a $PASSEDLOG - fi - else - # test failed. - n_fail=`expr $n_fail + 1` - REPORT_ERR "****FAILED ${HOSTNAME}: $TEST_TYPE****" - fi -} - -# Print a blank line -PRINT_BLANK() -{ - PRINT -} - -# Print test trailer -PRINT_TEST_TRAILER() -{ - PRINT "*** finished $TEST_TYPE tests for $HOSTNAME ***" - date; EndTime=`SecOfDay` - PRINT Total time = `ElapsedTime $StartTime $EndTime` - PRINT_BLANK -} - -# Print trailer summary -PRINT_TRAILER() -{ - PRINT "*** finished tests in $HOSTNAME ***" - date; TotalEndTime=`SecOfDay` - PRINT "${HOSTNAME}: Ran $n_test($n_pass/$n_fail/$n_skip) $runtest_type, Grand total test time = " \ - "`ElapsedTime $TotalStartTime $TotalEndTime`" | tee -a $TIMELOG - PRINT_BLANK -} - -# Figure out which remote command to use to reach a host. -# Try ssh first, then rsh since fewer machines support rsh exec. -# $1--hostname to reach. -CHECK_RSH() -{ - # Figure out how to use ping command in this host. - # Some hosts use "ping host count", some use "ping -c count host". - # Test "ping -c 3 -w 5" since it has timeout feature. - # Test "ping -c ..." style before "ping host 3" because some machines - # that recognize -c treat 'ping localhost 3' as to ping host '3'. - if [ -z "$PING" ]; then - if ping -c 3 -w 5 localhost >/dev/null 2>&1; then - PING='ping -c 3 -w 5' - PINGCOUNT= - elif ping -c 3 localhost >/dev/null 2>&1; then - PING='ping -c 3' - PINGCOUNT= - elif ping localhost 3 >/dev/null 2>&1; then - PING=ping - PINGCOUNT=3 - else # don't know how to use ping. - PING=no_ping - PINGCOUNT= - fi - fi - # - host=$1 - # Try remote command with host if it responds to ping. - # Still try it if we don't know how to do ping. - if [ no_ping = "$PING" ] || $PING $host $PINGCOUNT >/dev/null 2>&1; then - if ssh $host -n hostname >/dev/null 2>&1; then - RSH=ssh - elif rsh $host -n hostname >/dev/null 2>&1; then - RSH=rsh - else - PRINT cannot remote command with $host - RSH="NoRemoteCommand" - fi - else - RSH="NotReachable" - fi -} - - -# Wait for a file for at most number of minutes -# $1--the file -# $2--number of minutes -# WAIT_STATUS set to: -# -1 if errors encountered -# 0 if file found within time limit -# 1 if file not found within time limit -WAITFOR() -{ - wait_file=$1 - nminutes=$2 - if [ -z "$wait_file" -o ! "$nminutes" -ge 0 ] - then - PRINT "errors in argument of WAITFOR(): wait_file($1) or nminutes($2)" - WAIT_STATUS=-1 - return - fi - while [ ! -f $wait_file ]; do - if [ $nminutes -gt 0 ]; then - PRINT "Wait For $wait_file to appear" - sleep 60 #sleep 1 minute - else - WAIT_STATUS=1 - return - fi - nminutes=`expr $nminutes - 1` - done - WAIT_STATUS=0 - return -} - - -# Wait till a file disappears for at most number of minutes. -# Useful to wait till a lock is removed by another process. -# $1--the file -# $2--number of minutes -# WAIT_STATUS set to: -# -1 if errors encountered -# 0 if file disappears within time limit -# 1 if file has not disappeared within time limit -WAITTILL() -{ - wait_file=$1 - nminutes=$2 - if [ -z "$wait_file" -o ! "$nminutes" -ge 0 ] - then - PRINT "errors in argument of WAITTILL(): wait_file($1) or nminutes($2)" - WAIT_STATUS=-1 - return - fi - while [ -f $wait_file ]; do - if [ $nminutes -gt 0 ]; then - PRINT "Wait till $wait_file has disappeared" - sleep 60 #sleep 1 minute - else - WAIT_STATUS=1 - return - fi - nminutes=`expr $nminutes - 1` - done - WAIT_STATUS=0 - return -} - - -# Run one snapshot test -# $*--Types of test being run -RUNSNAPTEST() -{ - SNAPCMD_OPT="$STANDARD_OPT" # snapshot test option - SRCDIRNAME=${HOSTNAME} - # restore CC, PATH in case they were changed in the last test. - CC="$CC_SAVED" - PATH=$PATH_SAVED - export PATH # DEC OSF1 needs to export PATH explicitly - TEST_TYPE=$* - retcode=0 - skiptest=no - date - PRINT "*** starting $TEST_TYPE tests in $HOSTNAME ***" - PRINT "Uname -a: `uname -a`" - - # Parse the test type and set options accordingly. - # See comments of SNAPTEST_CONFIG_PARSE(). - while [ $# -gt 0 ]; do - case $1 in - -n32) # want -n32 option - SRCDIRNAME=${SRCDIRNAME}-n32 - CC="cc -n32" - export CC - ;; - -64) # want -64 option - SRCDIRNAME=${SRCDIRNAME}-64 - CC="cc -64" - export CC - ;; - parallel) # want parallel test - SNAPCMD_OPT="$SNAPCMD_OPT $ENABLE_PARALLEL" - SRCDIRNAME=${SRCDIRNAME}-pp - ;; - standard) # standard test - ;; - --*) - # option for configure - SNAPCMD_OPT="$SNAPCMD_OPT $1" - ;; - op-configure) - # option for configure - SNAPCMD_OPT="$SNAPCMD_OPT $1 $2" - shift - ;; - op-snapshot) - # option for snapshot - shift - SNAPCMD_OPT="$SNAPCMD_OPT $1" - ;; - setenv) - # pass them along to snapshot set environment variable - shift - SNAPCMD_OPT="$SNAPCMD_OPT setenv $1 $2" - shift - ;; - setenvN) - # set environment variable with $1 values - # e.g., setenvN 3 x a b c is same as setenv x="a b c". - # pass them along to snapshot set environment variable - shift - envN=$1 - shift - envname=$1 - SNAPCMD_OPT="$SNAPCMD_OPT setenvN $envN $envname" - envalue= - while test $envN -gt 0; do - shift - envalue="$envalue $1" - envN=`expr $envN - 1` - done - SNAPCMD_OPT="$SNAPCMD_OPT $envalue" - ;; - skip) - # skip this test - skiptest=yes - ;; - srcdirname) - # Use this before using parallel and -n32 since this overrides - # the others. - shift - SRCDIRNAME=$1 - ;; - deploy) - # deploy the built binary. - shift - SNAPCMD_OPT="$SNAPCMD_OPT deploy $1" - ;; - deploydir) - # default directory for deployment. - shift - SNAPCMD_OPT="$SNAPCMD_OPT deploydir $1" - ;; - *) # unknown test - PRINT "$0: unknown type of test ($1)" - retcode=1 - ;; - esac - shift - done - - if [ $retcode -ne 0 -o $skiptest = yes ]; then - errcode=$retcode - return $retcode - fi - - # Track down the zlib software - ans=`$SNAPYARD/current/bin/locate_sw zlib` - if [ $? = 0 ]; then - Z_INC=`echo $ans | cut -f1 -d,` - Z_LIB=`echo $ans | cut -f2 -d,` - SNAPCMD_OPT="$SNAPCMD_OPT zlib $Z_INC,$Z_LIB" - else - # cannot locate zlib software. - # continue the test, maybe configure can find it. - : - fi - - if [ -n "${SRCDIRNAME}" ]; then - SNAPCMD_OPT="$SNAPCMD_OPT srcdirname ${SRCDIRNAME}" - fi - - # Setup log file name to save test output - THIS_MINUTE=`date +%H%M` - LOGFILE=${LOGBASENAME}/${SRCDIRNAME}_${TODAY}_${THIS_MINUTE} - PRINT "Running snapshot with output saved in" - PRINT " $LOGFILE" - (date; PRINT Hostname=$HOSTNAME) >> $LOGFILE - - ( - cd $SNAPYARD/current - $SNAPSHOT $SNAPCMD $SNAPCMD_OPT - ) >> $LOGFILE 2>&1 - retcode=$? - [ $retcode -ne 0 ] && errcode=$retcode - - date >> $LOGFILE - if [ $retcode -ne 0 ]; then - # Dump the first 10 lines and the last 30 lines of the LOGFILE. - ( ntail=30 - echo ========================= - echo "Dumping logfile of ${HOSTNAME}: $TEST_TYPE" - echo "Last $ntail lines of $LOGFILE" - echo ========================= - tail -$ntail $LOGFILE - echo ========================= - echo Dumping done - echo ========================= - echo "" - ) >> $FAILEDDETAIL - fi -} - -TIMELIMIT_PARSE() -{ - # Function returns timeparam for timekeeper via standard out - - # any debug statements should be 'echo "Debug string" >&2' or timekeeper - # will declare timeparam to be non-numeric and ignore it. - while read x y ; do - # Scan for entry for this weekday. - xd=`echo $x | cut -f1 -d/` - if [ "$xd" = ${WEEKDAY} ]; then - # strip away the weekday/ part. - timeparam=`echo $x | cut -f2 -d/` - break - fi - case "$x" in - '' | '#'*) - # blank or comment lines. Continue. - ;; - ???/*) - # Ignore any entry not of this weekday. - ;; - *) - timeparam="$x" - ;; - esac - done - echo $timeparam - return -} - -# configuration parsing. -# Taking configuration from input. -# This should be invoke with configure file as stdin. -# Syntax of the configure file: -# All lines started with the # are comment lines and are ignored. -# Blank lines are ignored too. -# Each config line starts with a "Scope" followed by test types. -# -# Scope can be: -# standard ... # what the standard test types are. -# : Do for -# all: Do for all hosts. -# /... Use this scope if the matches. -# can be {Mon,Tue,Wed,Thu,Fri,Sat,Sun} -# If no : input for a , the standard test is used. -# -# Test types: -# standard tests defined in standard scope. -# -n32 -n32 mode. Apply to 64/32 bit OS such as IRIX64. -# parallel parallel mode. -# op-configure