diff options
| -rw-r--r-- | src/H5AC.c | 68 | ||||
| -rw-r--r-- | src/H5ACprivate.h | 1 | ||||
| -rw-r--r-- | src/H5Cprivate.h | 1 | ||||
| -rw-r--r-- | src/H5Cquery.c | 28 | ||||
| -rw-r--r-- | src/H5FScache.c | 60 | ||||
| -rw-r--r-- | src/H5FSsection.c | 333 | ||||
| -rw-r--r-- | src/H5Fint.c | 6 | ||||
| -rw-r--r-- | src/H5Fpkg.h | 11 | ||||
| -rw-r--r-- | src/H5Fprivate.h | 9 | ||||
| -rw-r--r-- | src/H5Fquery.c | 36 | ||||
| -rw-r--r-- | src/H5Fsuper.c | 127 | ||||
| -rw-r--r-- | src/H5MF.c | 592 | ||||
| -rw-r--r-- | src/H5MFaggr.c | 16 | ||||
| -rw-r--r-- | src/H5MFdbg.c | 8 | ||||
| -rw-r--r-- | src/H5Ocache_image.c | 33 | ||||
| -rw-r--r-- | src/H5Ofsinfo.c | 85 | ||||
| -rw-r--r-- | src/H5Oprivate.h | 4 | ||||
| -rw-r--r-- | src/H5Rint.c | 2 | ||||
| -rw-r--r-- | test/cache_image.c | 29 | ||||
| -rw-r--r-- | test/cache_tagging.c | 23 | ||||
| -rw-r--r-- | test/fheap.c | 57 | ||||
| -rw-r--r-- | test/mf.c | 59 | ||||
| -rw-r--r-- | test/titerate.c | 2 | ||||
| -rw-r--r-- | test/tselect.c | 1 | ||||
| -rw-r--r-- | test/tsohm.c | 78 | ||||
| -rw-r--r-- | tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl | 6 | ||||
| -rw-r--r-- | tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl | 4 | ||||
| -rw-r--r-- | tools/test/perform/pio_standalone.c | 118 | ||||
| -rw-r--r-- | tools/test/perform/sio_standalone.c | 119 |
29 files changed, 676 insertions, 1240 deletions
@@ -698,46 +698,6 @@ done: /*------------------------------------------------------------------------- - * Function: H5AC_force_cache_image_load() - * - * Purpose: On rare occasions, it is necessary to run - * H5MF_tidy_self_referential_fsm_hack() prior to the first - * metadata cache access. This is a problem as if there is a - * cache image at the end of the file, that routine will - * discard it. - * - * We solve this issue by calling this function, which will - * load the cache image and then call - * H5MF_tidy_self_referential_fsm_hack() to discard it. - * - * Return: SUCCEED on success, and FAIL on failure. - * - * Programmer: John Mainzer - * 1/11/17 - * - *------------------------------------------------------------------------- - */ -herr_t -H5AC_force_cache_image_load(H5F_t *f) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - /* Sanity checks */ - HDassert(f); - HDassert(f->shared); - HDassert(f->shared->cache); - - if(H5C_force_cache_image_load(f) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, FAIL, "Can't load cache image") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC_force_cache_image_load() */ - - -/*------------------------------------------------------------------------- * Function: H5AC_get_entry_status * * Purpose: Given a file address, determine whether the metadata @@ -1744,6 +1704,33 @@ done: /*------------------------------------------------------------------------- + * Function: H5AC_get_cache_flush_in_progess + * + * Purpose: Wrapper function for H5C_get_cache_flush_in_progress(). + * + * Return: SUCCEED on success, and FAIL on failure. + * + * Programmer: John Mainzer + * 3/11/05 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_get_cache_flush_in_progress(H5AC_t *cache_ptr, hbool_t *flush_in_progress_ptr) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + if(H5C_get_cache_flush_in_progress((H5C_t *)cache_ptr, flush_in_progress_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_get_cache_flush_in_progress() failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_get_cache_flush_in_progress() */ + + +/*------------------------------------------------------------------------- * Function: H5AC_get_cache_hit_rate * * Purpose: Wrapper function for H5C_get_cache_hit_rate(). @@ -2682,4 +2669,3 @@ H5AC_get_mdc_image_info(H5AC_t *cache_ptr, haddr_t *image_addr, hsize_t *image_l done: FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_get_mdc_image_info() */ - diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index e1fdedf..691f7c6 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -411,6 +411,7 @@ H5_DLL herr_t H5AC_get_cache_auto_resize_config(const H5AC_t * cache_ptr, H5AC_cache_config_t *config_ptr); H5_DLL herr_t H5AC_get_cache_size(H5AC_t *cache_ptr, size_t *max_size_ptr, size_t *min_clean_size_ptr, size_t *cur_size_ptr, uint32_t *cur_num_entries_ptr); +H5_DLL herr_t H5AC_get_cache_flush_in_progress(H5AC_t *cache_ptr, hbool_t *flush_in_progress_ptr); H5_DLL herr_t H5AC_get_cache_hit_rate(H5AC_t *cache_ptr, double *hit_rate_ptr); H5_DLL herr_t H5AC_reset_cache_hit_rate_stats(H5AC_t *cache_ptr); H5_DLL herr_t H5AC_set_cache_auto_resize_config(H5AC_t *cache_ptr, diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index d4ed6fc..bd200e0 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -2258,6 +2258,7 @@ H5_DLL herr_t H5C_get_cache_image_config(const H5C_t * cache_ptr, H5_DLL herr_t H5C_get_cache_size(H5C_t *cache_ptr, size_t *max_size_ptr, size_t *min_clean_size_ptr, size_t *cur_size_ptr, uint32_t *cur_num_entries_ptr); +H5_DLL herr_t H5C_get_cache_flush_in_progress(H5C_t *cache_ptr, hbool_t *flush_in_progress_ptr); H5_DLL herr_t H5C_get_cache_hit_rate(H5C_t *cache_ptr, double *hit_rate_ptr); H5_DLL herr_t H5C_get_entry_status(const H5F_t *f, haddr_t addr, size_t *size_ptr, hbool_t *in_cache_ptr, hbool_t *is_dirty_ptr, diff --git a/src/H5Cquery.c b/src/H5Cquery.c index 51ce1c2..a1267c5 100644 --- a/src/H5Cquery.c +++ b/src/H5Cquery.c @@ -156,6 +156,34 @@ done: /*------------------------------------------------------------------------- + * Function: H5C_get_cache_flush_in_progress + * + * Purpose: Return flush_in_progress in *flush_in_progress_ptr + * If the parameter is NULL, skip that value. + * + * Return: SUCCEED on success, and FAIL on failure. + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_get_cache_flush_in_progress(H5C_t * cache_ptr, hbool_t *flush_in_progress_ptr) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + if((cache_ptr == NULL) || (cache_ptr->magic != H5C__H5C_T_MAGIC)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Bad cache_ptr on entry.") + + if(flush_in_progress_ptr != NULL) + *flush_in_progress_ptr = cache_ptr->flush_in_progress; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_get_cache_flush_in_progress() */ + + +/*------------------------------------------------------------------------- * Function: H5C_get_cache_hit_rate * * Purpose: Compute and return the current cache hit rate in diff --git a/src/H5FScache.c b/src/H5FScache.c index 7525a9a..b520458 100644 --- a/src/H5FScache.c +++ b/src/H5FScache.c @@ -517,14 +517,36 @@ H5FS__cache_hdr_pre_serialize(H5F_t *f, void *_thing, if(!H5F_addr_defined(fspace->sect_addr)) { /* case 1 */ haddr_t tag = HADDR_UNDEF; + haddr_t sect_addr; + hsize_t saved_sect_size, new_sect_size; /* allocate file space for the section info, and insert it * into the metadata cache. */ - if(HADDR_UNDEF == (fspace->sect_addr = H5MF_alloc((H5F_t *)f, H5FD_MEM_FSPACE_SINFO, fspace->sect_size))) + saved_sect_size = fspace->sect_size; + if(HADDR_UNDEF == (sect_addr = H5MF_alloc((H5F_t *)f, H5FD_MEM_FSPACE_SINFO, fspace->sect_size))) HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "file allocation failed for free space sections") - fspace->alloc_sect_size = (size_t)fspace->sect_size; + /* fspace->sect_size may change in size after H5MF_alloc(). + * If increased in size, free the previous allocation and + * allocate again with the bigger fspace->sect_size. + */ + if(fspace->sect_size > saved_sect_size) { + + new_sect_size = fspace->sect_size; + + if(H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, sect_addr, saved_sect_size) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to free free space sections") + + if(HADDR_UNDEF == (sect_addr = H5MF_alloc((H5F_t *)f, H5FD_MEM_FSPACE_SINFO, new_sect_size))) + HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "file allocation failed for free space sections") + fspace->sect_size = new_sect_size; + fspace->alloc_sect_size = new_sect_size; + } else { + fspace->alloc_sect_size = saved_sect_size; + fspace->sect_size = saved_sect_size; + } + fspace->sect_addr = sect_addr; /* Get the tag for this free space manager and use it to insert the entry */ if(H5AC_get_tag((const void *)fspace, &tag) < 0) @@ -593,7 +615,6 @@ H5FS__cache_hdr_pre_serialize(H5F_t *f, void *_thing, * a nonsense section info address. */ if(!H5F_POINT_OF_NO_RETURN(f)) { - HDassert(fspace->serial_sect_count > 0); HDassert(fspace->sect_size > 0); HDassert(fspace->alloc_sect_size == (size_t)fspace->sect_size); } /* end if */ @@ -703,9 +724,8 @@ H5FS__cache_hdr_serialize(const H5F_t *f, void *_image, size_t len, if(!H5F_POINT_OF_NO_RETURN(f)) HDassert((! H5F_addr_defined(fspace->sect_addr)) || - ((fspace->serial_sect_count > 0) && - (fspace->sect_size > 0) && - (fspace->alloc_sect_size == (size_t)fspace->sect_size))); + ((fspace->sect_size > 0) && + (fspace->alloc_sect_size == (size_t)fspace->sect_size))); /* Magic number */ H5MM_memcpy(image, H5FS_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC); @@ -968,6 +988,7 @@ H5FS__cache_sinfo_deserialize(const void *_image, size_t len, void *_udata, haddr_t fs_addr; /* Free space header address */ size_t old_sect_size; /* Old section size */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + const uint8_t *chksum_image; /* Points to chksum location */ uint32_t stored_chksum; /* Stored metadata checksum */ void * ret_value = NULL; /* Return value */ @@ -1064,10 +1085,14 @@ H5FS__cache_sinfo_deserialize(const void *_image, size_t len, void *_udata, if(H5FS_sect_add(udata->f, fspace, new_sect, H5FS_ADD_DESERIALIZING, udata) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, NULL, "can't add section to free space manager") } /* end for */ + + if(fspace->tot_sect_count == old_tot_sect_count) + break; + } while(image < (((const uint8_t *)_image + old_sect_size) - H5FS_SIZEOF_CHKSUM)); /* Sanity check */ - HDassert((size_t)(image - (const uint8_t *)_image) == (old_sect_size - H5FS_SIZEOF_CHKSUM)); + HDassert((size_t)(image - (const uint8_t *)_image) <= (old_sect_size - H5FS_SIZEOF_CHKSUM)); HDassert(old_sect_size == fspace->sect_size); HDassert(old_tot_sect_count == fspace->tot_sect_count); HDassert(old_serial_sect_count == fspace->serial_sect_count); @@ -1077,11 +1102,14 @@ H5FS__cache_sinfo_deserialize(const void *_image, size_t len, void *_udata, /* checksum verification already done in verify_chksum cb */ + /* There may be empty space between entries and chksum */ + chksum_image = (const uint8_t *)(_image) + old_sect_size - H5FS_SIZEOF_CHKSUM; /* Metadata checksum */ - UINT32DECODE(image, stored_chksum); + UINT32DECODE(chksum_image, stored_chksum); /* Sanity check */ - HDassert((size_t)(image - (const uint8_t *)_image) == old_sect_size); + HDassert((image == chksum_image) || + ((size_t)((image - (const uint8_t *)_image) + (chksum_image - image)) == old_sect_size)); /* Set return value */ ret_value = sinfo; @@ -1177,10 +1205,6 @@ H5FS__cache_sinfo_pre_serialize(H5F_t *f, void *_thing, haddr_t addr, HDassert(new_len); HDassert(flags); - /* we shouldn't be called if the section info is empty, unless we hit the point of no return. */ - if(!H5F_POINT_OF_NO_RETURN(f)) - HDassert(fspace->serial_sect_count > 0); - sinfo_addr = addr; /* this will change if we relocate the section data */ /* Check for section info at temporary address */ @@ -1246,6 +1270,7 @@ H5FS__cache_sinfo_serialize(const H5F_t *f, void *_image, size_t len, H5FS_t *fspace; /* Free space header */ H5FS_iter_ud_t udata; /* User data for callbacks */ uint8_t *image = (uint8_t *)_image; /* Pointer into raw data buffer */ + uint8_t *chksum_image = NULL; /* Points to chksum location */ uint32_t metadata_chksum; /* Computed metadata checksum value */ unsigned bin; /* Current bin we are on */ herr_t ret_value = SUCCEED; /* Return value */ @@ -1290,13 +1315,16 @@ H5FS__cache_sinfo_serialize(const H5F_t *f, void *_image, size_t len, /* Compute checksum */ - metadata_chksum = H5_checksum_metadata(_image, (size_t)(image - (uint8_t *)_image), 0); + /* There may be empty space between entries and chksum */ + chksum_image = (uint8_t *)(_image) + len - H5FS_SIZEOF_CHKSUM; + metadata_chksum = H5_checksum_metadata(_image, (size_t)(chksum_image - (uint8_t *)_image), 0); /* Metadata checksum */ - UINT32ENCODE(image, metadata_chksum); + UINT32ENCODE(chksum_image, metadata_chksum); /* Sanity check */ - HDassert((size_t)(image - (uint8_t *)_image) == sinfo->fspace->sect_size); + HDassert((chksum_image == image) || + ((size_t)((image - (uint8_t *)_image) + (chksum_image - image)) == sinfo->fspace->sect_size)); HDassert(sinfo->fspace->sect_size <= sinfo->fspace->alloc_sect_size); done: diff --git a/src/H5FSsection.c b/src/H5FSsection.c index 27486f1..d15e299 100644 --- a/src/H5FSsection.c +++ b/src/H5FSsection.c @@ -307,6 +307,20 @@ HDfprintf(stderr, "%s: fspace->alloc_sect_size = %Hu, fspace->sect_size = %Hu\n" * Purpose: Release the section info, either giving ownership back to * the cache or letting the free space header keep it. * + * Add the fix in this routine to resolve the potential infinite loop + * problem when allocating file space for the meta data of the + * self-referential free-space managers at file closing. + * + * On file close or flushing, when the section info is modified + * and protected/unprotected, does not allow the section info size + * to shrink: + * --if the current allocated section info size in fspace->sect_size is + * larger than the previous section info size in fpsace->alloc_sect_size, + * release the section info + * --Otherwise, set the fspace->sect_size to be the same as + * fpsace->alloc_sect_size. This means fspace->sect_size may be larger + * than what is actually needed. + * * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol @@ -357,6 +371,11 @@ HDfprintf(stderr, "%s: fspace->alloc_sect_size = %Hu, fspace->sect_size = %Hu\n" /* Check if section info lock count dropped to zero */ if(fspace->sinfo_lock_count == 0) { hbool_t release_sinfo_space = FALSE; /* Flag to indicate section info space in file should be released */ + hbool_t flush_in_progress = FALSE; /* Is flushing in progress */ + + /* Check whether cache is flush_in_progress */ + if(H5AC_get_cache_flush_in_progress(f->shared->cache, &flush_in_progress) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't get flush_in_progress") /* Check if we actually protected the section info */ if(fspace->sinfo_protected) { @@ -370,9 +389,17 @@ HDfprintf(stderr, "%s: fspace->alloc_sect_size = %Hu, fspace->sect_size = %Hu\n" /* Note that we've modified the section info */ cache_flags |= H5AC__DIRTIED_FLAG; + /* On file close or flushing, does not allow section info to shrink in size */ + if(f->closing || flush_in_progress) { + if(fspace->sect_size > fspace->alloc_sect_size) + cache_flags |= H5AC__DELETED_FLAG | H5AC__TAKE_OWNERSHIP_FLAG; + else + fspace->sect_size = fspace->alloc_sect_size; + /* Check if the section info size in the file has changed */ - if(fspace->sect_size != fspace->alloc_sect_size) + } else if(fspace->sect_size != fspace->alloc_sect_size) cache_flags |= H5AC__DELETED_FLAG | H5AC__TAKE_OWNERSHIP_FLAG; + } /* end if */ /* Sanity check */ @@ -410,11 +437,20 @@ HDfprintf(stderr, "%s: Relinquishing section info ownership\n", FUNC); /* Check if the section info was modified */ if(fspace->sinfo_modified) { /* Check if we need to release section info in the file */ - if(H5F_addr_defined(fspace->sect_addr)) + if(H5F_addr_defined(fspace->sect_addr)) { /* Set flag to release section info space in file */ - release_sinfo_space = TRUE; - else + /* On file close or flushing, only need to release section info with size + bigger than previous section */ + if(f->closing || flush_in_progress) { + if(fspace->sect_size > fspace->alloc_sect_size) + release_sinfo_space = TRUE; + else + fspace->sect_size = fspace->alloc_sect_size; + } else + release_sinfo_space = TRUE; + } else HDassert(fspace->alloc_sect_size == 0); + } /* end if */ else { /* Sanity checks... */ @@ -2414,84 +2450,31 @@ done: /*------------------------------------------------------------------------- * Function: H5FS_vfd_alloc_hdr_and_section_info_if_needed * - * Purpose: This function is part of a hack to patch over a design - * flaw in the free space managers for file space allocation. - * Specifically, if a free space manager allocates space for - * its own section info, it is possible for it to - * go into an infinite loop as it: - * - * 1) computes the size of the section info - * - * 2) allocates file space for the section info - * - * 3) notices that the size of the section info - * has changed - * - * 4) deallocates the section info file space and - * returns to 1) above. - * - * Similarly, if it allocates space for its own header, it - * can go into an infinite loop as it: - * - * 1) allocates space for the header - * - * 2) notices that the free space manager is empty - * and thus should not have file space allocated - * to its header - * - * 3) frees the space allocated to the header - * - * 4) notices that the free space manager is not - * empty and thus must have file space allocated - * to it, and thus returns to 1) above. - * - * In a nutshell, the solution applied in this hack is to - * deallocate file space for the free space manager(s) that - * handle FSM header and/or section info file space allocations, - * wait until all other allocation/deallocation requests have - * been handled, and then test to see if the free space manager(s) - * in question are empty. If they are, do nothing. If they - * are not, allocate space for them at end of file bypassing the - * usual file space allocation calls, and thus avoiding the - * potential infinite loops. - * - * The purpose of this function is to allocate file space for + * Purpose: The purpose of this function is to allocate file space for * the header and section info of the target free space manager - * directly from the VFD if needed. In this case the function - * also re-inserts the header and section info in the metadata - * cache with this allocation. - * - * When paged allocation is not enabled, allocation of space - * for the free space manager header and section info is - * straight forward -- we simply allocate the space directly - * from file driver. - * - * Note that if f->shared->alignment > 1, and EOA is not a - * multiple of the alignment, it is possible that performing - * these allocations may generate a fragment of file space in - * addition to the space allocated for the section info. This - * excess space is dropped on the floor. As shall be seen, - * it will usually be reclaimed later. - * - * When page allocation is enabled, things are more difficult, - * as there is the possibility that page buffering will be - * enabled when the free space managers are read. To allow - * for this, we must ensure that space allocated for the - * free space manager header and section info is either larger - * than a page, or resides completely within a page. - * - * Do this by allocating space for the free space header and - * section info starting at page boundaries, and extending - * allocation to the next page boundary. This of course wastes - * space, but see below. - * - * On the first free space allocation / deallocation after the - * next file open, we will read the self referential free space - * managers, float them and reduce the EOA to its value prior - * to allocation of file space for the self referential free - * space managers on the preceeding file close. This EOA value - * is stored in the free space manager superblock extension - * message. + * if they are not allocated yet. + * + * The previous hack in this routine to avoid the potential infinite + * loops by allocating file space directly from the end of the file + * is removed. The allocation can now be done via the usual + * file space allocation call H5MF_alloc(). + * + * The design flaw is addressed by not allowing the size + * of section info to shrink. This means, when trying to allocate + * section info size X via H5MF_alloc() and the section info size + * after H5MF_alloc() changes to Y: + * --if Y is larger than X, free the just allocated file space X + * via H5MF_xfree() and set fspace->sect_size to Y. + * This routine will be called again later from + * H5MF_settle_meta_data_fsm() to allocate section info with the + * larger fpsace->sect_size Y. + * --if Y is smaller than X, no further allocation is needed and + * fspace->sect_size and fspace->alloc_sect_size are set to X. + * This means fspace->sect_size may be larger than what is + * actually needed. + * + * This routine also re-inserts the header and section info in the + * metadata cache with this allocation. * * Return: Success: non-negative * Failure: negative @@ -2508,8 +2491,6 @@ H5FS_vfd_alloc_hdr_and_section_info_if_needed(H5F_t *f, H5FS_t *fspace, hsize_t hdr_alloc_size; hsize_t sinfo_alloc_size; haddr_t sect_addr = HADDR_UNDEF; /* address of sinfo */ - haddr_t eoa_frag_addr = HADDR_UNDEF; /* Address of fragment at EOA */ - hsize_t eoa_frag_size = 0; /* Size of fragment at EOA */ haddr_t eoa = HADDR_UNDEF; /* Initial EOA for the file */ herr_t ret_value = SUCCEED; /* Return value */ @@ -2525,12 +2506,6 @@ H5FS_vfd_alloc_hdr_and_section_info_if_needed(H5F_t *f, H5FS_t *fspace, /* the section info should be unlocked */ HDassert(fspace->sinfo_lock_count == 0); - /* no space should be allocated */ - HDassert(*fs_addr_ptr == HADDR_UNDEF); - HDassert(fspace->addr == HADDR_UNDEF); - HDassert(fspace->sect_addr == HADDR_UNDEF); - HDassert(fspace->alloc_sect_size == 0); - /* persistent free space managers must be enabled */ HDassert(f->shared->fs_persist); @@ -2542,125 +2517,109 @@ H5FS_vfd_alloc_hdr_and_section_info_if_needed(H5F_t *f, H5FS_t *fspace, HDassert((f->shared->fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR) || (f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE)); - if(fspace->serial_sect_count > 0) { + if(fspace->serial_sect_count > 0 && fspace->sinfo) { /* the section info is floating, so space->sinfo should be defined */ - HDassert(fspace->sinfo); - - /* start by allocating file space for the header */ - - /* Get the EOA for the file -- need for sanity check below */ - if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_FSPACE_HDR))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "Unable to get eoa") - /* check for overlap into temporary allocation space */ - if(H5F_IS_TMP_ADDR(f, (eoa + fspace->sect_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, FAIL, "hdr file space alloc will overlap into 'temporary' file space") + if(!H5F_addr_defined(fspace->addr)) { - hdr_alloc_size = H5FS_HEADER_SIZE(f); - - /* if page allocation is enabled, extend the hdr_alloc_size to the - * next page boundary. - */ - if(H5F_PAGED_AGGR(f)) { - HDassert(0 == (eoa % f->shared->fs_page_size)); + /* start by allocating file space for the header */ - hdr_alloc_size = ((hdr_alloc_size / f->shared->fs_page_size) + 1) * f->shared->fs_page_size; + /* Get the EOA for the file -- need for sanity check below */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_FSPACE_HDR))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "Unable to get eoa") - HDassert(hdr_alloc_size >= H5FS_HEADER_SIZE(f)); - HDassert((hdr_alloc_size % f->shared->fs_page_size) == 0); - } /* end if */ + /* check for overlap into temporary allocation space */ + if(H5F_IS_TMP_ADDR(f, (eoa + fspace->sect_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, FAIL, "hdr file space alloc will overlap into 'temporary' file space") - /* allocate space for the hdr */ - if(HADDR_UNDEF == (fspace->addr = H5FD_alloc(f->shared->lf, H5FD_MEM_FSPACE_HDR, - f, hdr_alloc_size, &eoa_frag_addr, &eoa_frag_size))) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space for hdr") + hdr_alloc_size = H5FS_HEADER_SIZE(f); - /* if the file alignment is 1, there should be no - * eoa fragment. Otherwise, drop any fragment on the floor. - */ - HDassert((eoa_frag_size == 0) || (f->shared->alignment != 1)); + if(H5F_PAGED_AGGR(f)) + HDassert(0 == (eoa % f->shared->fs_page_size)); - /* Cache the new free space header (pinned) */ - if(H5AC_insert_entry(f, H5AC_FSPACE_HDR, fspace->addr, fspace, H5AC__PIN_ENTRY_FLAG) < 0) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space header to cache") + /* Allocate space for the free space header */ + if(HADDR_UNDEF == (fspace->addr = H5MF_alloc(f, H5FD_MEM_FSPACE_HDR, hdr_alloc_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "file allocation failed for free space header") - *fs_addr_ptr = fspace->addr; + /* Cache the new free space header (pinned) */ + if(H5AC_insert_entry(f, H5AC_FSPACE_HDR, fspace->addr, fspace, H5AC__PIN_ENTRY_FLAG) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space header to cache") - /* now allocate file space for the section info */ + *fs_addr_ptr = fspace->addr; + } - /* Get the EOA for the file -- need for sanity check below */ - if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_FSPACE_SINFO))) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTGET, FAIL, "Unable to get eoa") + if(!H5F_addr_defined(fspace->sect_addr)) { - /* check for overlap into temporary allocation space */ - if(H5F_IS_TMP_ADDR(f, (eoa + fspace->sect_size))) - HGOTO_ERROR(H5E_FSPACE, H5E_BADRANGE, FAIL, "sinfo file space alloc will overlap into 'temporary' file space") + /* now allocate file space for the section info */ - sinfo_alloc_size = fspace->sect_size; + /* Get the EOA for the file -- need for sanity check below */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_FSPACE_SINFO))) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTGET, FAIL, "Unable to get eoa") - /* if paged allocation is enabled, extend the sinfo_alloc_size to the - * next page boundary. - */ - if(H5F_PAGED_AGGR(f)) { - HDassert(0 == (eoa % f->shared->fs_page_size)); + /* check for overlap into temporary allocation space */ + if(H5F_IS_TMP_ADDR(f, (eoa + fspace->sect_size))) + HGOTO_ERROR(H5E_FSPACE, H5E_BADRANGE, FAIL, "sinfo file space alloc will overlap into 'temporary' file space") - sinfo_alloc_size = ((sinfo_alloc_size / f->shared->fs_page_size) + 1) * f->shared->fs_page_size; + sinfo_alloc_size = fspace->sect_size; - HDassert(sinfo_alloc_size >= fspace->sect_size); - HDassert((sinfo_alloc_size % f->shared->fs_page_size) == 0); - } /* end if */ + if(H5F_PAGED_AGGR(f)) + HDassert(0 == (eoa % f->shared->fs_page_size)); - /* allocate space for the section info */ - if(HADDR_UNDEF == (sect_addr = H5FD_alloc(f->shared->lf, H5FD_MEM_FSPACE_SINFO, - f, sinfo_alloc_size, &eoa_frag_addr, &eoa_frag_size))) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space") + /* allocate space for the section info */ + if(HADDR_UNDEF == (sect_addr = H5MF_alloc(f, H5FD_MEM_FSPACE_SINFO, sinfo_alloc_size))) + HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "file allocation failed for section info") - /* if the file alignment is 1, there should be no - * eoa fragment. Otherwise, drop the fragment on the floor. - */ - HDassert((eoa_frag_size == 0) || (f->shared->alignment != 1)); + /* update fspace->alloc_sect_size and fspace->sect_addr to reflect + * the allocation + */ + if(fspace->sect_size > sinfo_alloc_size) { + hsize_t saved_sect_size = fspace->sect_size; - /* update fspace->alloc_sect_size and fspace->sect_addr to reflect - * the allocation - */ - fspace->alloc_sect_size = fspace->sect_size; - fspace->sect_addr = sect_addr; - - /* insert the new section info into the metadata cache. */ - - /* Question: Do we need to worry about this insertion causing an - * eviction from the metadata cache? Think on this. If so, add a - * flag to H5AC_insert() to force it to skip the call to make space in - * cache. - * - * On reflection, no. - * - * On a regular file close, any eviction will not change the - * the contents of the free space manager(s), as all entries - * should have correct file space allocated by the time this - * function is called. - * - * In the cache image case, the selection of entries for inclusion - * in the cache image will not take place until after this call. - * (Recall that this call is made during the metadata fsm settle - * routine, which is called during the serialization routine in - * the cache image case. Entries are not selected for inclusion - * in the image until after the cache is serialized.) - * - * JRM -- 11/4/16 - */ - if(H5AC_insert_entry(f, H5AC_FSPACE_SINFO, sect_addr, fspace->sinfo, H5AC__NO_FLAGS_SET) < 0) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space sinfo to cache") + if(H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, sect_addr, sinfo_alloc_size) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to free free space sections") + fspace->sect_size = saved_sect_size; + } else { + fspace->alloc_sect_size = sinfo_alloc_size; + fspace->sect_size = sinfo_alloc_size; + fspace->sect_addr = sect_addr; + + /* insert the new section info into the metadata cache. */ + + /* Question: Do we need to worry about this insertion causing an + * eviction from the metadata cache? Think on this. If so, add a + * flag to H5AC_insert() to force it to skip the call to make space in + * cache. + * + * On reflection, no. + * + * On a regular file close, any eviction will not change the + * the contents of the free space manager(s), as all entries + * should have correct file space allocated by the time this + * function is called. + * + * In the cache image case, the selection of entries for inclusion + * in the cache image will not take place until after this call. + * (Recall that this call is made during the metadata fsm settle + * routine, which is called during the serialization routine in + * the cache image case. Entries are not selected for inclusion + * in the image until after the cache is serialized.) + * + * JRM -- 11/4/16 + */ + if(H5AC_insert_entry(f, H5AC_FSPACE_SINFO, sect_addr, fspace->sinfo, H5AC__NO_FLAGS_SET) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space sinfo to cache") - /* We have changed the sinfo address -- Mark free space header dirty */ - if(H5AC_mark_entry_dirty(fspace) < 0) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty") + /* We have changed the sinfo address -- Mark free space header dirty */ + if(H5AC_mark_entry_dirty(fspace) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty") - /* since space has been allocated for the section info and the sinfo - * has been inserted into the cache, relinquish ownership (i.e. float) - * the section info. - */ - fspace->sinfo = NULL; + /* since space has been allocated for the section info and the sinfo + * has been inserted into the cache, relinquish ownership (i.e. float) + * the section info. + */ + fspace->sinfo = NULL; + } + } } /* end if */ done: diff --git a/src/H5Fint.c b/src/H5Fint.c index 030fda8..4021e41 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -934,9 +934,9 @@ H5F__new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_ f->shared->fs_addr[u] = HADDR_UNDEF; f->shared->fs_man[u] = NULL; } - f->shared->first_alloc_dealloc = FALSE; - f->shared->eoa_pre_fsm_fsalloc = HADDR_UNDEF; - f->shared->eoa_post_fsm_fsalloc = HADDR_UNDEF; + /* This will be stored as eoa_pre_fsm_fsalloc in the fsinfo message */ + /* This is done to be backward compatible with 1.10 library that has the FSM hack */ + f->shared->eoa_fsm_fsalloc = HADDR_UNDEF; f->shared->eoa_post_mdci_fsalloc = HADDR_UNDEF; /* Initialization for handling file space (for paged aggregation) */ diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 6c1156f..5f25ddc 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -328,14 +328,9 @@ struct H5F_file_t { H5F_fs_state_t fs_state[H5F_MEM_PAGE_NTYPES]; /* State of free space manager for each type */ haddr_t fs_addr[H5F_MEM_PAGE_NTYPES]; /* Address of free space manager info for each type */ H5FS_t *fs_man[H5F_MEM_PAGE_NTYPES]; /* Free space manager for each file space type */ - hbool_t first_alloc_dealloc; /* TRUE iff free space managers */ - /* are persistent and have not */ - /* been used accessed for either */ - /* allocation or deallocation */ - /* since file open. */ - haddr_t eoa_pre_fsm_fsalloc; /* eoa pre file space allocation */ - /* for self referential FSMs */ - haddr_t eoa_post_fsm_fsalloc; /* eoa post file space allocation */ + hbool_t null_fsm_addr; /* Used by h5clear tool to tell the library */ + /* to drop free-space to the floor */ + haddr_t eoa_fsm_fsalloc; /* eoa after file space allocation */ /* for self referential FSMs */ haddr_t eoa_post_mdci_fsalloc; /* eoa past file space allocation */ /* for metadata cache image, or */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index e9b6c32..a8ac158 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -330,8 +330,7 @@ typedef struct H5F_t H5F_t; #define H5F_THRESHOLD(F) ((F)->shared->threshold) #define H5F_PGEND_META_THRES(F) ((F)->shared->fs.pgend_meta_thres) #define H5F_POINT_OF_NO_RETURN(F) ((F)->shared->fs.point_of_no_return) -#define H5F_FIRST_ALLOC_DEALLOC(F) ((F)->shared->first_alloc_dealloc) -#define H5F_EOA_PRE_FSM_FSALLOC(F) ((F)->shared->eoa_pre_fsm_fsalloc) +#define H5F_NULL_FSM_ADDR(F) ((F)->shared->null_fsm_addr) #define H5F_GET_MIN_DSET_OHDR(F) ((F)->shared->crt_dset_min_ohdr_flag) #define H5F_SET_MIN_DSET_OHDR(F, V) ((F)->shared->crt_dset_min_ohdr_flag = (V)) #else /* H5F_MODULE */ @@ -389,8 +388,7 @@ typedef struct H5F_t H5F_t; #define H5F_THRESHOLD(F) (H5F_get_threshold(F)) #define H5F_PGEND_META_THRES(F) (H5F_get_pgend_meta_thres(F)) #define H5F_POINT_OF_NO_RETURN(F) (H5F_get_point_of_no_return(F)) -#define H5F_FIRST_ALLOC_DEALLOC(F) (H5F_get_first_alloc_dealloc(F)) -#define H5F_EOA_PRE_FSM_FSALLOC(F) (H5F_get_eoa_pre_fsm_fsalloc(F)) +#define H5F_NULL_FSM_ADDR(F) (H5F_get_null_fsm_addr(F)) #define H5F_GET_MIN_DSET_OHDR(F) (H5F_get_min_dset_ohdr(F)) #define H5F_SET_MIN_DSET_OHDR(F, V) (H5F_set_min_dset_ohdr((F), (V))) #endif /* H5F_MODULE */ @@ -744,8 +742,7 @@ H5_DLL herr_t H5F_get_obj_count(const H5F_t *f, unsigned types, hbool_t app_ref, H5_DLL herr_t H5F_get_obj_ids(const H5F_t *f, unsigned types, size_t max_objs, hid_t *oid_list, hbool_t app_ref, size_t *obj_id_count_ptr); H5_DLL hsize_t H5F_get_pgend_meta_thres(const H5F_t *f); H5_DLL hbool_t H5F_get_point_of_no_return(const H5F_t *f); -H5_DLL hbool_t H5F_get_first_alloc_dealloc(const H5F_t *f); -H5_DLL haddr_t H5F_get_eoa_pre_fsm_fsalloc(const H5F_t *f); +H5_DLL hbool_t H5F_get_null_fsm_addr(const H5F_t *f); H5_DLL hbool_t H5F_get_min_dset_ohdr(const H5F_t *f); H5_DLL herr_t H5F_set_min_dset_ohdr(H5F_t *f, hbool_t minimize); diff --git a/src/H5Fquery.c b/src/H5Fquery.c index 103ad3b..c168535 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -1190,40 +1190,17 @@ H5F_get_point_of_no_return(const H5F_t *f) FUNC_LEAVE_NOAPI(f->shared->point_of_no_return) } /* end H5F_get_point_of_no_return() */ - /*------------------------------------------------------------------------- - * Function: H5F_get_first_alloc_dealloc + * Function: H5F_get_null_fsm_addr * - * Purpose: Retrieve the 'first alloc / dealloc' value for the file. + * Purpose: Retrieve the 'null_fsm_addr' value for the file. * - * Return: Success: Non-negative, the 'first_alloc_dealloc' + * Return: Success: Non-negative, the 'null_fsm_addr' * Failure: (can't happen) *------------------------------------------------------------------------- */ hbool_t -H5F_get_first_alloc_dealloc(const H5F_t *f) -{ - /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ - FUNC_ENTER_NOAPI_NOINIT_NOERR - - HDassert(f); - HDassert(f->shared); - - FUNC_LEAVE_NOAPI(f->shared->first_alloc_dealloc) -} /* end H5F_get_first_alloc_dealloc() */ - - -/*------------------------------------------------------------------------- - * Function: H5F_get_eoa_pre_fsm_fsalloc - * - * Purpose: Retrieve the 'EOA pre-FSM fsalloc' value for the file. - * - * Return: Success: Non-negative, the 'EOA pre-FSM fsalloc' - * Failure: (can't happen) - *------------------------------------------------------------------------- - */ -haddr_t -H5F_get_eoa_pre_fsm_fsalloc(const H5F_t *f) +H5F_get_null_fsm_addr(const H5F_t *f) { /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -1231,6 +1208,5 @@ H5F_get_eoa_pre_fsm_fsalloc(const H5F_t *f) HDassert(f); HDassert(f->shared); - FUNC_LEAVE_NOAPI(f->shared->eoa_pre_fsm_fsalloc) -} /* end H5F_get_eoa_pre_fsm_fsalloc() */ - + FUNC_LEAVE_NOAPI(f->shared->null_fsm_addr) +} /* end H5F_get_null_fsm_addr() */ diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c index 631937b..4fcc164 100644 --- a/src/H5Fsuper.c +++ b/src/H5Fsuper.c @@ -81,15 +81,6 @@ static const unsigned HDF5_superblock_ver_bounds[] = { HDF5_SUPERBLOCK_VERSION_LATEST /* H5F_LIBVER_LATEST */ }; -/* Format version bounds for fsinfo message */ -/* This message exists starting library release v1.10 */ -static const unsigned H5O_fsinfo_ver_bounds[] = { - H5O_INVALID_VERSION, /* H5F_LIBVER_EARLIEST */ - H5O_INVALID_VERSION, /* H5F_LIBVER_V18 */ - H5O_FSINFO_VERSION_1, /* H5F_LIBVER_V110 */ - H5O_FSINFO_VERSION_LATEST /* H5F_LIBVER_LATEST */ -}; - /*------------------------------------------------------------------------- * Function: H5F__super_ext_create @@ -778,28 +769,26 @@ H5F__super_read(H5F_t *f, H5P_genplist_t *fa_plist, hbool_t initial_read) /* If message is NOT marked "unknown"--set up file space info */ if(!(flags & H5O_MSG_FLAG_WAS_UNKNOWN)) { H5O_fsinfo_t fsinfo; /* File space info message from superblock extension */ - hbool_t null_fsm_addr = FALSE; /* Whether to drop free-space to the floor */ + /* f->shared->null_fsm_addr: Whether to drop free-space to the floor */ /* The h5clear tool uses this property to tell the library * to drop free-space to the floor */ if(H5P_exist_plist(fa_plist, H5F_ACS_NULL_FSM_ADDR_NAME) > 0) - if(H5P_get(fa_plist, H5F_ACS_NULL_FSM_ADDR_NAME, &null_fsm_addr) < 0) + if(H5P_get(fa_plist, H5F_ACS_NULL_FSM_ADDR_NAME, &f->shared->null_fsm_addr) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get clearance for persisting fsm addr") /* Retrieve the 'file space info' structure */ if(NULL == H5O_msg_read(&ext_loc, H5O_FSINFO_ID, &fsinfo)) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get free-space manager info message") - /* Update changed values */ - /* Version bounds check */ - if(H5O_fsinfo_ver_bounds[H5F_HIGH_BOUND(f)] == H5O_INVALID_VERSION || - fsinfo.version > H5O_fsinfo_ver_bounds[H5F_HIGH_BOUND(f)]) + if(H5O_fsinfo_check_version(H5F_HIGH_BOUND(f), &fsinfo) < 0) HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, FAIL, "File space info message's version out of bounds") + + /* Update changed values */ if(f->shared->fs_version != fsinfo.version) f->shared->fs_version = fsinfo.version; - if(f->shared->fs_strategy != fsinfo.strategy) { f->shared->fs_strategy = fsinfo.strategy; @@ -835,35 +824,42 @@ H5F__super_read(H5F_t *f, H5P_genplist_t *fa_plist, hbool_t initial_read) /* Initialize page end meta threshold */ f->shared->pgend_meta_thres = fsinfo.pgend_meta_thres; - if(f->shared->eoa_pre_fsm_fsalloc != fsinfo.eoa_pre_fsm_fsalloc) - f->shared->eoa_pre_fsm_fsalloc = fsinfo.eoa_pre_fsm_fsalloc; + if(f->shared->eoa_fsm_fsalloc != fsinfo.eoa_pre_fsm_fsalloc) + f->shared->eoa_fsm_fsalloc = fsinfo.eoa_pre_fsm_fsalloc; - /* f->shared->eoa_pre_fsm_fsalloc must always be HADDR_UNDEF - * in the absence of persistent free space managers. - */ - /* If the following two conditions are true: + /* + * If the following two conditions are true: * (1) skipping EOF check (skip_eof_check) - * (2) dropping free-space to the floor (null_fsm_addr) - * skip the asserts as "eoa_pre_fsm_fsalloc" may be undefined + * (2) dropping free-space to the floor (f->shared->null_fsm_addr) + * skip the asserts as "eoa_fsm_fsalloc" may be undefined * for a crashed file with persistent free space managers. - * #1 and #2 are enabled when the tool h5clear --increment + * The above two conditions are enabled when the tool h5clear --increment * option is used. */ - if(!skip_eof_check && !null_fsm_addr) { - HDassert((!f->shared->fs_persist) || (f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF)); - HDassert(!f->shared->first_alloc_dealloc); - } - - /* As "eoa_pre_fsm_fsalloc" may be undefined for a crashed file - * with persistent free space managers, therefore, set - * "first_alloc_dealloc" when the condition - * "dropping free-space to the floor is true. - * This will ensure that no action is done to settle things on file - * close via H5MF_settle_meta_data_fsm() and H5MF_settle_raw_data_fsm(). + if(!skip_eof_check && !f->shared->null_fsm_addr) + HDassert((!f->shared->fs_persist) || (f->shared->eoa_fsm_fsalloc != HADDR_UNDEF)); + + /* + * A crashed file with persistent free-space managers may have + * undefined f->shared->eoa_fsm_fsalloc. + * eoa_fsm_fsalloc is the final eoa which is saved in the free-space + * info message's eoa_pre_fsm_fsalloc field for backward compatibility. + * If the tool h5clear sets to dropping free-space to the floor + * as indicated by f->shared->null_fsm_addr, we are not going to + * perform actions to settle things on file close in the routines + * H5MF_settle_meta_data_fsm() and H5MF_settle_raw_data_fsm(). + * + * We remove the following check: + * if((f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF || null_fsm_addr) && + * (H5F_INTENT(f) & H5F_ACC_RDWR)) + * f->shared->first_alloc_dealloc = TRUE; + * + * Because: + * --there is no more f->shared->eoa_pre_fsm_fsalloc and + * f->shared->first_alloc_dealloc + * --the check for null_fsm_addr is directly done in H5MF_settle_meta_data_fsm() + * and H5MF_settle_raw_data_fsm() */ - if((f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF || null_fsm_addr) && - (H5F_INTENT(f) & H5F_ACC_RDWR)) - f->shared->first_alloc_dealloc = TRUE; f->shared->fs_addr[0] = HADDR_UNDEF; for(u = 1; u < NELMTS(f->shared->fs_addr); u++) @@ -871,17 +867,17 @@ H5F__super_read(H5F_t *f, H5P_genplist_t *fa_plist, hbool_t initial_read) /* If the following two conditions are true: * (1) file is persisting free-space - * (2) dropping free-space to the floor (null_fsm_addr) + * (2) dropping free-space to the floor (f->shared->null_fsm_addr) * nullify the addresses of the FSMs */ - if(f->shared->fs_persist && null_fsm_addr) + if(f->shared->fs_persist && f->shared->null_fsm_addr) for(u = 0; u < NELMTS(fsinfo.fs_addr); u++) f->shared->fs_addr[u] = fsinfo.fs_addr[u] = HADDR_UNDEF; /* For fsinfo.mapped: remove the FSINFO message from the superblock extension and write a new message to the extension */ - /* For null_fsm_addr: just update FSINFO message in the superblock extension */ - if(((fsinfo.mapped || null_fsm_addr) && (rw_flags & H5AC__READ_ONLY_FLAG) == 0)) { + /* For f->shared->null_fsm_addr: just update FSINFO message in the superblock extension */ + if(((fsinfo.mapped || f->shared->null_fsm_addr) && (rw_flags & H5AC__READ_ONLY_FLAG) == 0)) { /* Do the same kluge until we know for sure. VC */ #if 1 /* bug fix test code -- tidy this up if all goes well */ /* JRM */ @@ -895,7 +891,7 @@ H5F__super_read(H5F_t *f, H5P_genplist_t *fa_plist, hbool_t initial_read) f->shared->sblock = sblock; #endif /* JRM */ - if(null_fsm_addr) { + if(f->shared->null_fsm_addr) { if(H5F__super_ext_write_msg(f, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing fsinfo message to superblock extension") } @@ -1394,7 +1390,7 @@ H5F__super_init(H5F_t *f) fsinfo.mapped = FALSE; /* Set the version for the fsinfo message */ - if(H5F__fsinfo_set_version(f, &fsinfo) < 0) + if(H5O_fsinfo_set_version(H5F_LOW_BOUND(f), H5F_HIGH_BOUND(f), &fsinfo) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't set version of fsinfo") f->shared->fs_version = fsinfo.version; @@ -1824,44 +1820,3 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5F__super_ext_remove_msg() */ - -/*------------------------------------------------------------------------- - * Function: H5F__fsinfo_set_version - * - * Purpose: Set the version to encode the fsinfo message with. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Vailin Choi; June 2019 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5F__fsinfo_set_version(const H5F_t *f, H5O_fsinfo_t *fsinfo) -{ - unsigned version; /* Message version */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity check */ - HDassert(f); - HDassert(fsinfo); - - version = H5O_FSINFO_VERSION_1; - - /* Upgrade to the version indicated by the file's low bound if higher */ - if(H5O_fsinfo_ver_bounds[H5F_LOW_BOUND(f)] != H5O_INVALID_VERSION) - version = MAX(version, H5O_fsinfo_ver_bounds[H5F_LOW_BOUND(f)]); - - /* Version bounds check */ - if(H5O_fsinfo_ver_bounds[H5F_HIGH_BOUND(f)] == H5O_INVALID_VERSION || - version > H5O_fsinfo_ver_bounds[H5F_HIGH_BOUND(f)]) - HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, FAIL, "File space info message's version out of bounds") - - /* Set the message version */ - fsinfo->version = version; - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5F__fsinfo_set_version() */ @@ -50,6 +50,11 @@ #define H5MF_FSPACE_SHRINK 80 /* Percent of "normal" size to shrink serialized free space size */ #define H5MF_FSPACE_EXPAND 120 /* Percent of "normal" size to expand serialized free space size */ +#define H5MF_CHECK_FSM(FSM, CF) \ + HDassert(*CF == FALSE); \ + if(!H5F_addr_defined(FSM->addr) || !H5F_addr_defined(FSM->sect_addr)) \ + *CF = TRUE; + /******************/ /* Local Typedefs */ /******************/ @@ -90,6 +95,8 @@ static herr_t H5MF__close_shrink_eoa(H5F_t *f); static herr_t H5MF__get_free_sects(H5F_t *f, H5FS_t *fspace, H5MF_sect_iter_ud_t *sect_udata, size_t *nums); static hbool_t H5MF__fsm_type_is_self_referential(H5F_t *f, H5F_mem_page_t fsm_type); static hbool_t H5MF__fsm_is_self_referential(H5F_t *f, H5FS_t *fspace); +static herr_t H5MF__continue_alloc_fsm(H5F_t *f, H5FS_t *sm_hdr_fspace, H5FS_t *sm_sinfo_fspace, + H5FS_t *lg_hdr_fspace, H5FS_t *lg_sinfo_fspace, hbool_t *continue_alloc_fsm); /* Free-space type manager routines */ static herr_t H5MF__create_fstype(H5F_t *f, H5F_mem_page_t type); @@ -794,12 +801,6 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ HDassert(f->shared->lf); HDassert(size > 0); - if(f->shared->first_alloc_dealloc) { - HDassert(! H5AC_cache_image_pending(f)); - if(H5MF_tidy_self_referential_fsm_hack(f) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "tidy of self referential fsm hack failed") - } /* end if */ - H5MF__alloc_to_fs_type(f, alloc_type, size, &fs_type); #ifdef H5MF_ALLOC_DEBUG_MORE @@ -1107,12 +1108,6 @@ HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUN HGOTO_DONE(SUCCEED) HDassert(addr != 0); /* Can't deallocate the superblock :-) */ - if(f->shared->first_alloc_dealloc) { - HDassert(!H5AC_cache_image_pending(f)); - if(H5MF_tidy_self_referential_fsm_hack(f) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed") - } /* end if */ - H5MF__alloc_to_fs_type(f, alloc_type, size, &fs_type); /* Set the ring type in the API context */ @@ -1302,12 +1297,6 @@ HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_r HDassert(f); HDassert(H5F_INTENT(f) & H5F_ACC_RDWR); - if(f->shared->first_alloc_dealloc) { - HDassert(! H5AC_cache_image_pending(f)); - if(H5MF_tidy_self_referential_fsm_hack(f) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed") - } /* end if */ - /* Set mapped type, treating global heap as raw data */ map_type = (alloc_type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : alloc_type; @@ -1680,16 +1669,6 @@ HDfprintf(stderr, "%s: Entering\n", FUNC); * H5MF_tidy_self_referential_fsm_hack() to discard the cache image * block. */ - if(f->shared->first_alloc_dealloc) { - if(H5AC_cache_image_pending(f)) { - if(H5AC_force_cache_image_load(f) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "forced cache image load failed") - } /* end if */ - else { - if(H5MF_tidy_self_referential_fsm_hack(f) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed") - } /* end else */ - } /* end if */ /* Set the ring type in the API context. In most cases, we will * need H5AC_RING_RDFSM, so initially set the ring in @@ -1834,7 +1813,7 @@ HDfprintf(stderr, "%s: Entering\n", FUNC); fsinfo.threshold = f->shared->fs_threshold; fsinfo.page_size = f->shared->fs_page_size; fsinfo.pgend_meta_thres = f->shared->pgend_meta_thres; - fsinfo.eoa_pre_fsm_fsalloc = f->shared->eoa_pre_fsm_fsalloc; + fsinfo.eoa_pre_fsm_fsalloc = f->shared->eoa_fsm_fsalloc; fsinfo.version = f->shared->fs_version; /* Write the free space manager message -- message must already exist */ @@ -1892,7 +1871,7 @@ HDfprintf(stderr, "%s: Entering\n", FUNC); * been no file space allocation or deallocation since file * open. */ - HDassert((f->shared->first_alloc_dealloc) || (final_eoa == f->shared->eoa_post_fsm_fsalloc)); + HDassert(H5F_NULL_FSM_ADDR(f) || final_eoa == f->shared->eoa_fsm_fsalloc); } /* end if */ else { /* super_vers can be 0, 1, 2 */ for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) @@ -2000,7 +1979,7 @@ HDfprintf(stderr, "%s: Entering\n", FUNC); */ for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) fsinfo.fs_addr[ptype-1] = f->shared->fs_addr[ptype]; - fsinfo.eoa_pre_fsm_fsalloc = f->shared->eoa_pre_fsm_fsalloc; + fsinfo.eoa_pre_fsm_fsalloc = f->shared->eoa_fsm_fsalloc; /* Write the free space manager message -- message must already exist */ if(H5F__super_ext_write_msg(f, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) @@ -2061,8 +2040,8 @@ HDfprintf(stderr, "%s: Entering\n", FUNC); * the file driver layer. However, as this possibility seems remote, * it is ignored in the following assert. */ - HDassert((f->shared->first_alloc_dealloc) || - (final_eoa == f->shared->eoa_post_fsm_fsalloc) || + HDassert((H5F_NULL_FSM_ADDR(f)) || + (final_eoa == f->shared->eoa_fsm_fsalloc) || ((H5F_addr_defined(f->shared->eoa_post_mdci_fsalloc)) && (final_eoa == f->shared->eoa_post_mdci_fsalloc))); } /* end if */ @@ -2395,16 +2374,6 @@ H5MF_get_free_sections(H5F_t *f, H5FD_mem_t type, size_t nsects, H5F_sect_info_t * H5MF_tidy_self_referential_fsm_hack() to discard the cache image * block. */ - if(f->shared->first_alloc_dealloc) { - if(H5AC_cache_image_pending(f)) { - if(H5AC_force_cache_image_load(f) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, (-1), "forced cache image load failed") - } /* end if */ - else { - if(H5MF_tidy_self_referential_fsm_hack(f) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, (-1), "tidy of self referential fsm hack failed") - } /* end else */ - } /* end if */ if(type == H5FD_MEM_DEFAULT) { start_type = H5F_MEM_PAGE_SUPER; @@ -2676,10 +2645,11 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hbool_t *fsm_settled) HDassert(f->shared); HDassert(fsm_settled); - /* Only need to settle things if we are persisting the free space info - * and allocation/deallocation has occurred. + /* + * Only need to settle things if we are persisting free space and + * the private property in f->shared->null_fsm_addr is not enabled. */ - if(f->shared->fs_persist && !f->shared->first_alloc_dealloc) { + if(f->shared->fs_persist && !H5F_NULL_FSM_ADDR(f)) { hbool_t fsm_opened[H5F_MEM_PAGE_NTYPES]; /* State of FSM */ hbool_t fsm_visited[H5F_MEM_PAGE_NTYPES]; /* State of FSM */ @@ -3017,6 +2987,7 @@ done: FUNC_LEAVE_NOAPI_TAG(ret_value) } /* H5MF_settle_raw_data_fsm() */ + /*------------------------------------------------------------------------- * Function: H5MF_settle_meta_data_fsm() @@ -3035,7 +3006,7 @@ done: * On entry to this function, the raw data settle routine * (H5MF_settle_raw_data_fsm()) should have: * - * 1) Freed the aggregators. + * 1) Freed the aggregators. * * 2) Freed all file space allocated to the free space managers. * @@ -3071,12 +3042,12 @@ done: * 1) Verify that the free space manager(s) involved in file * space allocation for free space managers are still floating. * - * 2) Free the aggregators. + * 2) Free the aggregators. * - * 3) Reduce the EOA to the extent possible, and make note + * 3) Reduce the EOA to the extent possible, and make note * of the resulting value. This value will be stored * in the fsinfo superblock extension message and be used - * in the subsequent file open. + * in the subsequent file open. * * 4) Re-allocate space for any free space manager(s) that: * @@ -3097,21 +3068,15 @@ done: * allocation has changed the size of the section info -- * forcing us to deallocate and start the loop over again. * - * To avoid this, simply allocate file space for these - * FSM(s) directly from the VFD layer if allocation is - * indicated. This avoids the issue by bypassing the FSMs - * in this case. - * - * Note that this may increase the size of the file needlessly. - * A better solution would be to modify the FSM code to + * The solution is to modify the FSM code to * save empty FSMs to file, and to allow section info blocks - * to be oversized. However, given that the FSM code is - * also used by the fractal heaps, and that we are under - * severe time pressure at the moment, the above brute - * force solution is attractive. + * to be oversized. That is, only allow section info to increase + * in size, not shrink. The solution is now implemented. * - * 5) Make note of the EOA -- used for sanity checking on - * FSM shutdown. + * 5) Make note of the EOA -- used for sanity checking on + * FSM shutdown. This is saved as eoa_pre_fsm_fsalloc in + * the free-space info message for backward compatibility + * with the 1.10 library that has the hack. * * Return: SUCCEED/FAIL * @@ -3131,10 +3096,9 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hbool_t *fsm_settled) H5FS_t *sm_sinfo_fspace = NULL; /* ptr to sm FSM sinfo alloc FSM */ H5FS_t *lg_hdr_fspace = NULL; /* ptr to lg FSM hdr alloc FSM */ H5FS_t *lg_sinfo_fspace = NULL; /* ptr to lg FSM sinfo alloc FSM */ - haddr_t eoa_pre_fsm_fsalloc; /* eoa pre file space allocation */ - /* for self referential FSMs */ - haddr_t eoa_post_fsm_fsalloc; /* eoa post file space allocation */ + haddr_t eoa_fsm_fsalloc; /* eoa after file space allocation */ /* for self referential FSMs */ + hbool_t continue_alloc_fsm = FALSE; /* Continue allocating addr and sect_addr for FSMs */ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ herr_t ret_value = SUCCEED; /* Return value */ @@ -3145,10 +3109,11 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hbool_t *fsm_settled) HDassert(f->shared); HDassert(fsm_settled); - /* Only need to settle things if we are persisting the free space info - * and allocation/deallocation has occurred. + /* + * Only need to settle things if we are persisting free space and + * the private property in f->shared->null_fsm_addr is not enabled. */ - if(f->shared->fs_persist && !f->shared->first_alloc_dealloc) { + if(f->shared->fs_persist && !H5F_NULL_FSM_ADDR(f)) { /* Sanity check */ HDassert(f->shared->lf); @@ -3196,7 +3161,6 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hbool_t *fsm_settled) { H5FS_stat_t fs_stat; /* Information for hdr FSM */ - /* Verify that sm_hdr_fspace is floating if it exists */ if(sm_hdr_fspace) { /* Query free space manager info for this type */ if(H5FS_stat_info(f, sm_hdr_fspace, &fs_stat) < 0) @@ -3253,7 +3217,6 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hbool_t *fsm_settled) * Note that the aggregators will not exist if paged aggregation * is enabled -- don't attempt to free if this is the case. */ - /* Vailin -- is this correct? */ /* (for space not at EOF, it may be put into free space managers) */ if((!H5F_PAGED_AGGR(f)) && (H5MF_free_aggrs(f) < 0)) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free aggregators") @@ -3262,21 +3225,7 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hbool_t *fsm_settled) if(H5MF__close_shrink_eoa(f) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") - /* At this point, the EOA should be set to a value that contains - * the allocation for all user data, all non self referential FSMs, - * the superblock and all superblock extension messages. - * - * Make note of the current EOA. We will store this value in the - * free space manager superblock extension message. Since space for - * everything other than the self referential FSMs (and possibly the - * cache image) has been allocated at this point, this allows us to - * to float the self referential FSMs on the first file space allocation / - * deallocation and then set the EOA to this value before we handle - * the allocation / deallocation. (If a cache image exists, the - * first allocation / deallocation will be the deallocation of space - * for the cache image). - * - * WARNING: This approach settling the self referential free space + /* WARNING: This approach settling the self referential free space * managers and allocating space for them in the file will * not work as currently implemented with the split and * multi file drivers, as the self referential free space @@ -3295,97 +3244,67 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hbool_t *fsm_settled) * We should be able to support the split file driver * without a file format change. However, the code to * do so does not exist at present. + * NOTE: not sure whether to remove or keep the above comments */ - if(HADDR_UNDEF == (eoa_pre_fsm_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT))) - HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get EOA") - - /* ******************* PROBLEM: ******************** - * - * If the file has an alignment other than 1, and if - * the EOA is not a multiple of this alignment, allocating space - * for the section via the VFD info has the potential of generating - * a fragment that will be added to the free space manager. This - * of course undoes everything we have been doing here. - * - * Need a way around this. Obvious solution is to force the EOA to - * be a multiple of the alignment. - * - * Fortunately, alignment is typically 1, so this is a non-issue in - * most cases. In cases where the alignment is not 1, for now we - * have decided to drop the fragment on the floor. - * - * Eventually, we should fix this by modifying the on disk representations - * of free space managers to allow for empty space, so as to bypass the - * issues created by self-referential free space managers, and make - * this issue moot. + /* + * Continue allocating file space for the header and section info until + * they are all settled, */ - /* HDassert(f->shared->alignment == 1); */ - + do { + continue_alloc_fsm = FALSE; + if(sm_hdr_fspace) + if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, sm_hdr_fspace, &(f->shared->fs_addr[sm_fshdr_fs_type])) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sm hdr FSM file space") + + if(sm_sinfo_fspace && (sm_sinfo_fspace != sm_hdr_fspace)) + if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, sm_sinfo_fspace, &(f->shared->fs_addr[sm_fssinfo_fs_type])) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sm sinfo FSM file space") + + if(H5F_PAGED_AGGR(f)) { + if(lg_hdr_fspace) + if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, lg_hdr_fspace, &(f->shared->fs_addr[lg_fshdr_fs_type])) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg hdr FSM file space") + + if(lg_sinfo_fspace && (lg_sinfo_fspace != lg_hdr_fspace)) + if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, lg_sinfo_fspace, &(f->shared->fs_addr[lg_fssinfo_fs_type])) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg sinfo FSM file space") + } /* end if */ - /* The free space manager(s) that handle space allocations for free - * space managers should be settled now, albeit without file space - * allocated to them. To avoid the possibility of changing the sizes - * of their section info blocks, allocate space for them now at the - * end of file via H5FD_alloc(). - * - * In the past, this issue of allocating space without touching the - * free space managers has been dealt with by calling - * H5MF_aggr_vfd_alloc(), which in turn calls H5MF_aggr_alloc(). - * This is problematic since (if I read the code correctly) it will - * re-constitute the metadata aggregator, which will add any leftover - * space to one of the free space managers when freed. - * - * This is a non-starter, since the entire objective is to settle the - * free space managers. - * - * Hence the decision to call H5FD_alloc() directly. - * - * As discussed in PROBLEM above, if f->shared->alignment is not 1, - * this has the possibility of generating a fragment of file space - * that would typically be inserted into one of the free space managers. - * - * This is isn't good, but due to schedule pressure, we will just drop - * the fragment on the floor for now. - */ - if(sm_hdr_fspace) - if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, sm_hdr_fspace, &(f->shared->fs_addr[sm_fshdr_fs_type])) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sm hdr FSM file space") + sm_hdr_fspace = f->shared->fs_man[sm_fshdr_fs_type]; + sm_sinfo_fspace = f->shared->fs_man[sm_fssinfo_fs_type]; + if(H5F_PAGED_AGGR(f)) { + lg_hdr_fspace = f->shared->fs_man[lg_fshdr_fs_type]; + lg_sinfo_fspace = f->shared->fs_man[lg_fssinfo_fs_type]; + } - if(sm_sinfo_fspace && (sm_sinfo_fspace != sm_hdr_fspace)) - if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, sm_sinfo_fspace, &(f->shared->fs_addr[sm_fssinfo_fs_type])) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sm sinfo FSM file space") + if(H5MF__continue_alloc_fsm(f, sm_hdr_fspace, sm_sinfo_fspace, lg_hdr_fspace, lg_sinfo_fspace, &continue_alloc_fsm) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg sinfo FSM file space") - if(H5F_PAGED_AGGR(f)) { - if(lg_hdr_fspace) - if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, lg_hdr_fspace, &(f->shared->fs_addr[lg_fshdr_fs_type])) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg hdr FSM file space") + } while(continue_alloc_fsm); - if(lg_sinfo_fspace && (lg_sinfo_fspace != lg_hdr_fspace)) - if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, lg_sinfo_fspace, &(f->shared->fs_addr[lg_fssinfo_fs_type])) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg sinfo FSM file space") - } /* end if */ + /* All free space managers should have file space allocated for them + * now, and should see no further allocations / deallocations. + * For backward compatibility, store the eoa in f->shared->eoa_fsm_fsalloc + * which will be set to fsinfo.eoa_pre_fsm_fsalloc when we actually write + * the free-space info message to the superblock extension. + * This will allow the 1.10 library with the hack to open the file with + * the new solution. + */ /* Get the eoa after allocation of file space for the self referential * free space managers. Assuming no cache image, this should be the * final EOA of the file. */ - if(HADDR_UNDEF == (eoa_post_fsm_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT))) + if(HADDR_UNDEF == (eoa_fsm_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT))) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file size") - - /* All free space managers should have file space allocated for them - * now, and should see no further allocations / deallocations. Store - * the pre and post file space allocation for self referential FSMs EOA - * for use when we actually write the free space manager superblock - * extension message. - */ - f->shared->eoa_pre_fsm_fsalloc = eoa_pre_fsm_fsalloc; - f->shared->eoa_post_fsm_fsalloc = eoa_post_fsm_fsalloc; + f->shared->eoa_fsm_fsalloc = eoa_fsm_fsalloc; /* Indicate that the FSM was settled successfully */ *fsm_settled = TRUE; } /* end if */ + done: /* Reset the ring in the API context */ if(orig_ring != H5AC_RING_INV) @@ -3396,6 +3315,61 @@ done: /*------------------------------------------------------------------------- + * Function: H5MF__continue_alloc_fsm + * + * Purpose: To determine whether any of the input FSMs has allocated + * its "addr" and "sect_addr". + * Return TRUE or FALSE in *continue_alloc_fsm. + * + * Return: SUCCEED/FAIL + * + * Programmer: Vailin Choi + * 6/24/2019 + *------------------------------------------------------------------------- + */ +static herr_t +H5MF__continue_alloc_fsm(H5F_t *f, H5FS_t *sm_hdr_fspace, H5FS_t *sm_sinfo_fspace, + H5FS_t *lg_hdr_fspace, H5FS_t *lg_sinfo_fspace, hbool_t *continue_alloc_fsm) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + HDassert(continue_alloc_fsm); + + /* Check sm_hdr_fspace */ + if(sm_hdr_fspace && sm_hdr_fspace->serial_sect_count > 0 && sm_hdr_fspace->sinfo) { + H5MF_CHECK_FSM(sm_hdr_fspace, continue_alloc_fsm); + } + + if(!(*continue_alloc_fsm)) { + if(sm_sinfo_fspace && sm_sinfo_fspace != sm_hdr_fspace && + sm_sinfo_fspace->serial_sect_count > 0 && sm_sinfo_fspace->sinfo) { + H5MF_CHECK_FSM(sm_hdr_fspace, continue_alloc_fsm); + } + } + + if(H5F_PAGED_AGGR(f) && !(*continue_alloc_fsm)) { + /* Check lg_hdr_fspace */ + if(lg_hdr_fspace && lg_hdr_fspace->serial_sect_count > 0 && lg_hdr_fspace->sinfo) { + H5MF_CHECK_FSM(lg_hdr_fspace, continue_alloc_fsm); + } + + /* Check lg_sinfo_fspace */ + if(!(*continue_alloc_fsm)) { + if(lg_sinfo_fspace && lg_sinfo_fspace != lg_hdr_fspace && + lg_sinfo_fspace->serial_sect_count > 0 && lg_sinfo_fspace->sinfo) { + H5MF_CHECK_FSM(lg_sinfo_fspace, continue_alloc_fsm); + } + } + } + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5MF__continue_alloc_fsm() */ + + +/*------------------------------------------------------------------------- * Function: H5MF__fsm_type_is_self_referential() * * Purpose: Return TRUE if the indicated free space manager allocates @@ -3500,303 +3474,3 @@ H5MF__fsm_is_self_referential(H5F_t *f, H5FS_t *fspace) FUNC_LEAVE_NOAPI(result) } /* H5MF__fsm_is_self_referential() */ - - -/*------------------------------------------------------------------------- - * Function: H5MF_tidy_self_referential_fsm_hack - * - * Purpose: As discussed in the comments of the settle routines above, - * the existence of self referential free space managers - * as currently implemented creates the possibility of - * infinite loops at file close. - * - * As a hack to avoid this, we have added code to settle - * self referential free space managers, and then allocate - * space for them directly from the file driver. - * - * To avoid dropping ever increasing amounts of file space - * on the floor with each subsequent file close/open cycle, - * we need to clean this up on file open. To avoid this, - * this function is called on the first file space allocation - * or deallocation after file open to float the self referential - * free space managers and reduce the EOA to the value it - * had before the direct allocation of space for the self - * referential free space managers. - * - * The function proceeds as follows: - * - * 1) Verify that f->shared->first_alloc_dealloc is TRUE, - * and then set it to FALSE. - * - * 2) Get the current EOA. Verify that it is greater than - * or equal to f->shared->eoa_pre_fsm_fsalloc. If the - * current eoa is equal to f->shared->eoa_pre_fsm_fsalloc, - * no self referential FSMs were stored, and we are done. - * - * NOTE: This will have to be reworked somewhat for - * cache image. - * - * 3) Load the self referential FSMs. In passing verify that - * the lowest address of a FSM header is equal to - * f->shared->eoa_pre_fsm_fsalloc. - * - * Note that we don't have to use any special I/O for - * this -- we can use the regular I/O methods even if - * paged aggregation and page buffering is enabled. - * - * 4) Float the FSMs. Ensure that the file space is NOT - * released. - * - * 5) Set EOA equal to f->shared->eoa_pre_fsm_fsalloc, - * and then set f->shared->eoa_pre_fsm_fsalloc to - * HADDR_UNDEF. - * - * If page buffering, verify that the new EOA is - * on a page boundary, and expunge any pages in the - * page buffer after the new EOA. - * - * Note that this function is also called from test code - * when it is necessary to startup a self referential - * free space manager prior to the first file space - * allocation / deallocation. Failure to do so will - * result in assertion failures in this function on - * the first file space allocation / deallocation. - * - * Return: SUCCEED/FAIL - * - * Programmer: John Mainzer - * 12/11/16 - * - *------------------------------------------------------------------------- - */ -herr_t -H5MF_tidy_self_referential_fsm_hack(H5F_t *f) -{ - haddr_t eoa; /* EOA of file */ - hsize_t tail_size = 0; /* Size of chunk to free */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - haddr_t first_srfsm_hdr = HADDR_UNDEF; /* Addr of first self referential */ - /* fsm header in file */ - H5FS_stat_t fs_stat; /* Information for hdr FSM */ - H5F_mem_page_t sm_fshdr_fs_type; /* Small fs hdr fsm */ - H5F_mem_page_t sm_fssinfo_fs_type; /* Small fs sinfo fsm */ - H5F_mem_page_t lg_fshdr_fs_type; /* Large fs hdr fsm */ - H5F_mem_page_t lg_fssinfo_fs_type; /* Large fs sinfo fsm */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI_TAG(H5AC__FREESPACE_TAG, FAIL) - - /* check args */ - HDassert(f); - HDassert(f->shared); - HDassert(f->shared->fs_persist); - HDassert(f->shared->first_alloc_dealloc); - - /* Set the ring type in the API context. Since we are only dealing with - * self referential FSMs, we will only need H5AC_RING_MDFSM. - */ - H5AC_set_ring(H5AC_RING_MDFSM, &orig_ring); - - /* 1) Verify that f->shared->first_alloc_dealloc is TRUE, - * and then set it to FALSE. - */ - HDassert(f->shared->first_alloc_dealloc); - f->shared->first_alloc_dealloc = FALSE; - - - /* 2) Get the current EOA. Verify that it is greater than - * or equal to f->shared->eoa_pre_fsm_fsalloc. If the - * current eoa is equal to f->shared->eoa_pre_fsm_fsalloc, - * no self referential FSMs were stored, and we are done. - * - * NOTE: This will have to be reworked somewhat for - * cache image. - */ - if(HADDR_UNDEF == (eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT))) - HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get EOA") - HDassert(H5F_addr_le(f->shared->eoa_pre_fsm_fsalloc, eoa)); - - if(H5F_addr_eq(f->shared->eoa_pre_fsm_fsalloc, eoa)) - HGOTO_DONE(SUCCEED) - - - /* 3) Load the self referential FSMs. In passing verify that - * the lowest address of a FSM header is equal to - * f->shared->eoa_pre_fsm_fsalloc.' - * - * Note that we don't have to use any special I/O for - * this -- we can use the regular I/O methods even if - * paged aggregation and page buffering is enabled. - */ - H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fs_type); - H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fs_type); - HDassert(sm_fshdr_fs_type > H5F_MEM_PAGE_DEFAULT); - HDassert(sm_fshdr_fs_type < H5F_MEM_PAGE_LARGE_SUPER); - - HDassert(sm_fssinfo_fs_type > H5F_MEM_PAGE_DEFAULT); - HDassert(sm_fssinfo_fs_type < H5F_MEM_PAGE_LARGE_SUPER); - - HDassert(NULL == f->shared->fs_man[sm_fshdr_fs_type]); - HDassert(NULL == f->shared->fs_man[sm_fssinfo_fs_type]); - - if(H5F_addr_defined(f->shared->fs_addr[sm_fshdr_fs_type])) { - first_srfsm_hdr = f->shared->fs_addr[sm_fshdr_fs_type]; - - /* open the FSM */ - if(H5MF__open_fstype(f, sm_fshdr_fs_type) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") - - HDassert(f->shared->fs_man[sm_fshdr_fs_type]); - } /* end if */ - - if((sm_fshdr_fs_type != sm_fssinfo_fs_type) && - (H5F_addr_defined(f->shared->fs_addr[sm_fssinfo_fs_type]))) { - - if(!H5F_addr_defined(first_srfsm_hdr) || - (H5F_addr_defined(first_srfsm_hdr) && - H5F_addr_lt(f->shared->fs_addr[sm_fssinfo_fs_type], first_srfsm_hdr))) - first_srfsm_hdr = f->shared->fs_addr[sm_fssinfo_fs_type]; - - HDassert(NULL == f->shared->fs_man[sm_fssinfo_fs_type]); - - /* open the FSM */ - if(H5MF__open_fstype(f, sm_fssinfo_fs_type) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") - - HDassert(f->shared->fs_man[sm_fssinfo_fs_type]); - } /* end if */ - - if(H5F_PAGED_AGGR(f)) { - H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fs_type); - H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fs_type); - - HDassert(lg_fshdr_fs_type >= H5F_MEM_PAGE_LARGE_SUPER); - HDassert(lg_fshdr_fs_type < H5F_MEM_PAGE_NTYPES); - - HDassert(lg_fssinfo_fs_type >= H5F_MEM_PAGE_LARGE_SUPER); - HDassert(lg_fssinfo_fs_type < H5F_MEM_PAGE_NTYPES); - - HDassert(NULL == f->shared->fs_man[lg_fshdr_fs_type]); - HDassert(NULL == f->shared->fs_man[lg_fssinfo_fs_type]); - - if(H5F_addr_defined(f->shared->fs_addr[lg_fshdr_fs_type])) { - if(!H5F_addr_defined(first_srfsm_hdr) || - (H5F_addr_defined(first_srfsm_hdr) && - H5F_addr_lt(f->shared->fs_addr[lg_fshdr_fs_type], first_srfsm_hdr))) - first_srfsm_hdr = f->shared->fs_addr[lg_fshdr_fs_type]; - - HDassert(NULL == f->shared->fs_man[lg_fshdr_fs_type]); - - /* open the FSM */ - if(H5MF__open_fstype(f, lg_fshdr_fs_type) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") - HDassert(f->shared->fs_man[lg_fshdr_fs_type]); - } /* end if */ - - if(lg_fshdr_fs_type != lg_fssinfo_fs_type && H5F_addr_defined(f->shared->fs_addr[lg_fssinfo_fs_type])) { - if(!H5F_addr_defined(first_srfsm_hdr) || - (H5F_addr_defined(first_srfsm_hdr) && - H5F_addr_lt(f->shared->fs_addr[lg_fssinfo_fs_type], first_srfsm_hdr))) - first_srfsm_hdr = f->shared->fs_addr[lg_fssinfo_fs_type]; - - HDassert(NULL == f->shared->fs_man[lg_fssinfo_fs_type]); - - /* open the FSM */ - if(H5MF__open_fstype(f, lg_fssinfo_fs_type) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") - HDassert(f->shared->fs_man[lg_fssinfo_fs_type]); - } /* end if */ - } /* end if */ - HDassert(H5F_addr_eq(first_srfsm_hdr, f->shared->eoa_pre_fsm_fsalloc)); - - /* 4) Float the FSMs. Ensure that the file space is NOT released. */ - if(f->shared->fs_man[sm_fshdr_fs_type]) { - /* Sanity check: Query free space manager info for this type */ - if(H5FS_stat_info(f, f->shared->fs_man[sm_fshdr_fs_type], &fs_stat) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") - - HDassert(H5F_addr_defined(fs_stat.addr)); - HDassert(H5F_addr_defined(fs_stat.sect_addr)); - if(H5FS_free(f, f->shared->fs_man[sm_fshdr_fs_type], FALSE) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers") - f->shared->fs_addr[sm_fshdr_fs_type] = HADDR_UNDEF; - } /* end if */ - - if(sm_fshdr_fs_type != sm_fssinfo_fs_type && f->shared->fs_man[sm_fssinfo_fs_type]) { - /* Sanity check: Query free space manager info for this type */ - if(H5FS_stat_info(f, f->shared->fs_man[sm_fssinfo_fs_type], &fs_stat) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") - - HDassert(H5F_addr_defined(fs_stat.addr)); - HDassert(H5F_addr_defined(fs_stat.sect_addr)); - if(H5FS_free(f, f->shared->fs_man[sm_fssinfo_fs_type], FALSE) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers") - f->shared->fs_addr[sm_fssinfo_fs_type] = HADDR_UNDEF; - } /* end if */ - - if(H5F_PAGED_AGGR(f)) { - if(f->shared->fs_man[lg_fshdr_fs_type]) { - /* Sanity check: Query free space manager info for this type */ - if(H5FS_stat_info(f, f->shared->fs_man[lg_fshdr_fs_type], &fs_stat) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") - - HDassert(H5F_addr_defined(fs_stat.addr)); - HDassert(H5F_addr_defined(fs_stat.sect_addr)); - if(H5FS_free(f, f->shared->fs_man[lg_fshdr_fs_type], FALSE) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't float free-space headers") - f->shared->fs_addr[lg_fshdr_fs_type] = HADDR_UNDEF; - } /* end if */ - - if(lg_fshdr_fs_type != lg_fssinfo_fs_type && f->shared->fs_man[lg_fssinfo_fs_type]) { - /* Sanity check: Query free space manager info for this type */ - if(H5FS_stat_info(f, f->shared->fs_man[lg_fssinfo_fs_type], &fs_stat) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") - - HDassert(H5F_addr_defined(fs_stat.addr)); - HDassert(H5F_addr_defined(fs_stat.sect_addr)); - if(H5FS_free(f, f->shared->fs_man[lg_fssinfo_fs_type], FALSE) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't float free-space headers") - f->shared->fs_addr[lg_fssinfo_fs_type] = HADDR_UNDEF; - } /* end if */ - } /* end if */ - - /* 5) Set EOA equal to f->shared->eoa_pre_fsm_fsalloc, - * and then set f->shared->eoa_pre_fsm_fsalloc to - * HADDR_UNDEF. - * - * If page buffering, verify that the new EOA is - * on a page boundary, and expunge any pages in the - * page buffer after the new EOA. - */ - if(!H5F_PAGED_AGGR(f)) { - /* Verify that the aggregators are still shutdown. */ - HDassert(f->shared->sdata_aggr.tot_size == 0); - HDassert(f->shared->sdata_aggr.addr == 0); - HDassert(f->shared->sdata_aggr.size == 0); - - HDassert(f->shared->meta_aggr.tot_size == 0); - HDassert(f->shared->meta_aggr.addr == 0); - HDassert(f->shared->meta_aggr.size == 0); - } /* end if */ - - tail_size = (hsize_t)(eoa - f->shared->eoa_pre_fsm_fsalloc); - - /* Release file space allocated to self referential FSMs */ - if(H5F__free(f, H5FD_MEM_DEFAULT, f->shared->eoa_pre_fsm_fsalloc, tail_size) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "driver free request failed") - if(HADDR_UNDEF == (eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT))) - HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get EOA") - HDassert(H5F_addr_eq(f->shared->eoa_pre_fsm_fsalloc, eoa)); - - f->shared->eoa_pre_fsm_fsalloc = HADDR_UNDEF; - - HDassert((!H5F_PAGED_AGGR(f)) || (0 == (eoa % f->shared->fs_page_size))); - -done: - /* Reset the ring in the API context */ - if(orig_ring != H5AC_RING_INV) - H5AC_set_ring(orig_ring, NULL); - - FUNC_LEAVE_NOAPI_TAG(ret_value) -} /* H5MF_tidy_self_referential_fsm_hack() */ - diff --git a/src/H5MFaggr.c b/src/H5MFaggr.c index 3db7f73..c726341 100644 --- a/src/H5MFaggr.c +++ b/src/H5MFaggr.c @@ -183,7 +183,21 @@ HDfprintf(stderr, "%s: type = %u, size = %Hu\n", FUNC, (unsigned)type, size); * allocate "generic" space and sub-allocate out of that, if possible. * Otherwise just allocate through H5F__alloc(). */ - if((f->shared->feature_flags & aggr->feature_flag) && f->shared->fs_strategy != H5F_FSPACE_STRATEGY_NONE) { + + /* + * Replace the following line with the line in #ifdef REPLACE/#endif. + * The line in #ifdef REPLACE triggers the following problem: + * test/objcopy.c: test_copy_group_deep() test fails with the family driver + * + * When closing the destination file after H5Ocopy, the library flushes the fractal + * heap direct block via H5HF__cache_dblock_pre_serialize(). While doing so, + * the cache eventually adjusts/evicts ageout entries and ends up flushing out the + * same entry that is being serialized (flush_in_progress). + */ + if((f->shared->feature_flags & aggr->feature_flag) && f->shared->fs_strategy != H5F_FSPACE_STRATEGY_NONE && (!f->closing || !f->shared->fs_persist)) { +#ifdef REPLACE + if((f->shared->feature_flags & aggr->feature_flag) && f->shared->fs_strategy != H5F_FSPACE_STRATEGY_NONE && !f->closing) { +#endif haddr_t aggr_frag_addr = HADDR_UNDEF; /* Address of aggregrator fragment */ hsize_t aggr_frag_size = 0; /* Size of aggregator fragment */ hsize_t alignment; /* Alignment of this section */ diff --git a/src/H5MFdbg.c b/src/H5MFdbg.c index eedb72e..e11476b 100644 --- a/src/H5MFdbg.c +++ b/src/H5MFdbg.c @@ -215,14 +215,14 @@ done: *------------------------------------------------------------------------- */ herr_t -H5MF_sects_dump(H5F_t *f, FILE *stream) +H5MF__sects_dump(H5F_t *f, FILE *stream) { haddr_t eoa; /* End of allocated space in the file */ int indent = 0; /* Amount to indent */ int fwidth = 50; /* Field width */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_PACKAGE_TAG(H5AC__FREESPACE_TAG, FAIL) + FUNC_ENTER_PACKAGE_TAG(H5AC__FREESPACE_TAG) #ifdef H5MF_ALLOC_DEBUG HDfprintf(stderr, "%s: Dumping file free space sections\n", FUNC); #endif /* H5MF_ALLOC_DEBUG */ @@ -261,7 +261,7 @@ HDfprintf(stderr, "%s: for type = H5FD_MEM_DEFAULT, eoa = %a\n", FUNC, eoa); udata.fwidth = MAX(0, fwidth - 6); /* Iterate over all the free space sections */ - if(H5FS_sect_iterate(f, f->shared->fs_man[ptype], H5MF_sects_debug_cb, &udata) < 0) + if(H5FS_sect_iterate(f, f->shared->fs_man[ptype], H5MF__sects_debug_cb, &udata) < 0) HGOTO_ERROR(H5E_HEAP, H5E_BADITER, FAIL, "can't iterate over heap's free space") } /* end if */ else @@ -314,7 +314,7 @@ HDfprintf(stderr, "%s: sda_addr = %a, sda_size = %Hu, end of sda = %a\n", FUNC, udata.fwidth = MAX(0, fwidth - 6); /* Iterate over all the free space sections */ - if(H5FS_sect_iterate(f, f->shared->fs_man[atype], H5MF_sects_debug_cb, &udata) < 0) + if(H5FS_sect_iterate(f, f->shared->fs_man[atype], H5MF__sects_debug_cb, &udata) < 0) HGOTO_ERROR(H5E_HEAP, H5E_BADITER, FAIL, "can't iterate over heap's free space") } /* end if */ else /* No sections of this type */ diff --git a/src/H5Ocache_image.c b/src/H5Ocache_image.c index 591ac4a..7336211 100644 --- a/src/H5Ocache_image.c +++ b/src/H5Ocache_image.c @@ -27,11 +27,12 @@ */ #include "H5Omodule.h" /* This source code file is part of the H5O module */ +#define H5F_FRIEND /*suppress error about including H5Fpkg */ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ -#include "H5Fprivate.h" /* Files */ +#include "H5Fpkg.h" /* Files */ #include "H5FLprivate.h" /* Free Lists */ #include "H5Opkg.h" /* Object headers */ #include "H5MFprivate.h" /* File space management */ @@ -275,16 +276,16 @@ H5O__mdci_free(void *mesg) *------------------------------------------------------------------------- */ static herr_t -H5O__mdci_delete(H5F_t *f, H5O_t *open_oh, void *_mesg) +H5O__mdci_delete(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, void *_mesg) { H5O_mdci_t *mesg = (H5O_mdci_t *)_mesg; - herr_t ret_value = SUCCEED; /* Return value */ + haddr_t final_eoa; /* For sanity check */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC /* check args */ HDassert(f); - HDassert(open_oh); HDassert(mesg); /* Free file space for cache image */ @@ -321,16 +322,24 @@ H5O__mdci_delete(H5F_t *f, H5O_t *open_oh, void *_mesg) * before the first metadata cache access. However, given * time constraints, I don't want to go there now. */ - if(H5F_FIRST_ALLOC_DEALLOC(f)) { - HDassert(HADDR_UNDEF != H5F_EOA_PRE_FSM_FSALLOC(f)); - HDassert(H5F_addr_ge(mesg->addr, H5F_EOA_PRE_FSM_FSALLOC(f))); - if(H5MF_tidy_self_referential_fsm_hack(f) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed") - } /* end if */ - else { + + + + if(f->closing) { + + /* Get the eoa, and verify that it has the expected value */ + if(HADDR_UNDEF == (final_eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)) ) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "unable to get file size") + + HDassert(H5F_addr_eq(final_eoa, mesg->addr + mesg->size)); + + if(H5FD_free(f->shared->lf, H5FD_MEM_SUPER, f, mesg->addr, mesg->size) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "can't free MDC image") + } else { if(H5MF_xfree(f, H5FD_MEM_SUPER, mesg->addr, mesg->size) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to free file space for cache image block") - } /* end else */ + } + } /* end if */ done: diff --git a/src/H5Ofsinfo.c b/src/H5Ofsinfo.c index ec2328c..2a71b05 100644 --- a/src/H5Ofsinfo.c +++ b/src/H5Ofsinfo.c @@ -65,6 +65,16 @@ const H5O_msg_class_t H5O_MSG_FSINFO[1] = {{ H5O__fsinfo_debug /* debug the message */ }}; +/* Format version bounds for fsinfo message */ +/* This message exists starting library release v1.10 */ +static const unsigned H5O_fsinfo_ver_bounds[] = { + H5O_INVALID_VERSION, /* H5F_LIBVER_EARLIEST */ + H5O_INVALID_VERSION, /* H5F_LIBVER_V18 */ + H5O_FSINFO_VERSION_1, /* H5F_LIBVER_V110 */ + H5O_FSINFO_VERSION_LATEST /* H5F_LIBVER_LATEST */ +}; +#define N_FSINFO_VERSION_BOUNDS 4 + /* Declare a free list to manage the H5O_fsinfo_t struct */ H5FL_DEFINE_STATIC(H5O_fsinfo_t); @@ -257,7 +267,7 @@ H5O_fsinfo_copy(const void *_mesg, void *_dest) /* check args */ HDassert(fsinfo); if(!dest && NULL == (dest = H5FL_CALLOC(H5O_fsinfo_t))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") /* copy */ *dest = *fsinfo; @@ -331,6 +341,79 @@ H5O__fsinfo_free(void *mesg) /*------------------------------------------------------------------------- + * Function: H5O_fsinfo_set_version + * + * Purpose: Set the version to encode the fsinfo message with. + * + * Return: SUCCEED/FAIL + * + * Programmer: Vailin Choi; June 2019 + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_fsinfo_set_version(H5F_libver_t low, H5F_libver_t high, H5O_fsinfo_t *fsinfo) +{ + unsigned version; /* Message version */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDcompile_assert(N_FSINFO_VERSION_BOUNDS == H5F_LIBVER_NBOUNDS); + HDassert(fsinfo); + + version = H5O_FSINFO_VERSION_1; + + /* Upgrade to the version indicated by the file's low bound if higher */ + if(H5O_fsinfo_ver_bounds[low] != H5O_INVALID_VERSION) + version = MAX(version, H5O_fsinfo_ver_bounds[low]); + + /* Version bounds check */ + if(H5O_fsinfo_ver_bounds[high] == H5O_INVALID_VERSION || version > H5O_fsinfo_ver_bounds[high]) + HGOTO_ERROR(H5E_OHDR, H5E_BADRANGE, FAIL, "File space info message's version out of bounds") + + /* Set the message version */ + fsinfo->version = version; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_fsinfo_set_version() */ + + +/*------------------------------------------------------------------------- + * Function: H5O_fsinfo_check_version + * + * Purpose: Validate the fsinfo message version + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Summer 2019 + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_fsinfo_check_version(H5F_libver_t high, H5O_fsinfo_t *fsinfo) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDcompile_assert(N_FSINFO_VERSION_BOUNDS == H5F_LIBVER_NBOUNDS); + HDassert(fsinfo); + + /* Check the version */ + if(H5O_fsinfo_ver_bounds[high] == H5O_INVALID_VERSION || fsinfo->version > H5O_fsinfo_ver_bounds[high]) + HGOTO_ERROR(H5E_OHDR, H5E_BADRANGE, FAIL, "File space info message's version out of bounds") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_fsinfo_check_version() */ + + +/*------------------------------------------------------------------------- * Function: H5O__fsinfo_debug * * Purpose: Prints debugging info for a message. diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index d143f31..354a00b 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -997,6 +997,10 @@ H5_DLL H5O_loc_t *H5O_get_loc(hid_t id); /* EFL operators */ H5_DLL hsize_t H5O_efl_total_size(H5O_efl_t *efl); +/* File space info routines */ +H5_DLL herr_t H5O_fsinfo_set_version(H5F_libver_t low, H5F_libver_t high, H5O_fsinfo_t *fsinfo); +H5_DLL herr_t H5O_fsinfo_check_version(H5F_libver_t high, H5O_fsinfo_t *fsinfo); + /* Fill value operators */ H5_DLL herr_t H5O_fill_reset_dyn(H5O_fill_t *fill); H5_DLL herr_t H5O_fill_convert(H5O_fill_t *fill, H5T_t *type, hbool_t *fill_changed); diff --git a/src/H5Rint.c b/src/H5Rint.c index 05c6ddf..2355ec1 100644 --- a/src/H5Rint.c +++ b/src/H5Rint.c @@ -84,7 +84,7 @@ H5R__init_package(void) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_NOAPI_NOINIT_NOERR /* Mark "top" of interface as initialized */ H5R_top_package_initialize_s = TRUE; diff --git a/test/cache_image.c b/test/cache_image.c index ee49502..f8252ac 100644 --- a/test/cache_image.c +++ b/test/cache_image.c @@ -93,6 +93,7 @@ static unsigned evict_on_close_test(hbool_t single_file_vfd); #define DSET_SIZE (40 * CHUNK_SIZE) #define MAX_NUM_DSETS 256 + static void create_datasets(hid_t file_id, int min_dset, int max_dset) { @@ -3717,8 +3718,7 @@ cache_image_smoke_check_2(hbool_t single_file_vfd) if ( show_progress ) HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); - - + /* 4) Open the file. * * Verify that the metadata cache is instructed to load the @@ -4486,6 +4486,7 @@ cache_image_smoke_check_4(hbool_t single_file_vfd) if ( show_progress ) HDfprintf(stdout, "%s:L3 cp = %d, max_dset = %d, pass = %d.\n", fcn_name, cp + 2, max_dset, pass); + } /* end while */ cp += 3; @@ -4638,7 +4639,6 @@ cache_image_smoke_check_4(hbool_t single_file_vfd) FUNC, failure_mssg); return !pass; - } /* cache_image_smoke_check_4() */ @@ -5111,8 +5111,7 @@ cache_image_smoke_check_5(hbool_t single_file_vfd) * extension message has been deleted. */ i = min_group; - while ( ( pass ) && ( i <= max_group ) ) { - + while ( ( pass ) && ( i <= max_group ) ) { sprintf(process_group_name, "/process_%d", i); validate_zoo(file_id, process_group_name, i++); } @@ -7513,10 +7512,9 @@ get_free_sections_test(hbool_t single_file_vfd) /* file_ptr->shared->first_alloc_dealloc is set to FALSE if the * file is opened R/O. */ - if ( ( file_ptr->shared->first_alloc_dealloc ) || - ( ! H5F_addr_defined(file_ptr->shared->eoa_pre_fsm_fsalloc) ) || + if ( ( ! H5F_addr_defined(file_ptr->shared->eoa_fsm_fsalloc) ) || ( ! H5F_addr_defined(file_ptr->shared->cache->image_addr) ) || - ( H5F_addr_gt(file_ptr->shared->eoa_pre_fsm_fsalloc, + ( H5F_addr_gt(file_ptr->shared->eoa_fsm_fsalloc, file_ptr->shared->cache->image_addr) ) ) { pass = FALSE; @@ -7554,11 +7552,8 @@ get_free_sections_test(hbool_t single_file_vfd) pass = FALSE; failure_mssg = "cache image not loaded (1).\n"; - } else if ( file_ptr->shared->first_alloc_dealloc ) { + } - pass = FALSE; - failure_mssg = "self referential FSMs not floated (1).\n"; - } } if ( show_progress ) @@ -7634,10 +7629,9 @@ get_free_sections_test(hbool_t single_file_vfd) */ if ( pass ) { - if ( ( ! file_ptr->shared->first_alloc_dealloc ) || - ( ! H5F_addr_defined(file_ptr->shared->eoa_pre_fsm_fsalloc) ) || + if ( ( ! H5F_addr_defined(file_ptr->shared->eoa_fsm_fsalloc) ) || ( ! H5F_addr_defined(file_ptr->shared->cache->image_addr) ) || - ( H5F_addr_gt(file_ptr->shared->eoa_pre_fsm_fsalloc, + ( H5F_addr_gt(file_ptr->shared->eoa_fsm_fsalloc, file_ptr->shared->cache->image_addr) ) ) { pass = FALSE; @@ -7675,11 +7669,8 @@ get_free_sections_test(hbool_t single_file_vfd) pass = FALSE; failure_mssg = "cache image not loaded (2).\n"; - } else if ( file_ptr->shared->first_alloc_dealloc ) { + } - pass = FALSE; - failure_mssg = "self referential FSMs not floated (2).\n"; - } } if ( show_progress ) diff --git a/test/cache_tagging.c b/test/cache_tagging.c index 734c760..dd886eb 100644 --- a/test/cache_tagging.c +++ b/test/cache_tagging.c @@ -1664,21 +1664,11 @@ check_attribute_rename_tags(hid_t fcpl, int type) if ( verify_tag(fid, H5AC_SOHM_LIST_ID, H5AC__SOHM_TAG) < 0 ) TEST_ERROR; /* - * 3 calls to verify_tag() for verifying free space: - * one freespace header tag for H5FD_MEM_DRAW manager, - * one freespace header tag for H5FD_MEM_SUPER manager - * one freespace section info tag for H5FD_MEM_SUPER manager + * one freespace header tag for H5FD_MEM_DRAW manager, + * one freespace header tag for H5FD_MEM_SUPER manager */ if ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 ) TEST_ERROR; - - /* If the free space managers are persistent, the - * H5MF_tidy_self_referential_fsm_hack() must have been run. - * Since this function floats all self referential free space - * managers, the H5FD_MEM_SUPER FSM will not be in the metadata - * cache. - */ - if(!persistent_fsms && verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0) TEST_ERROR; - if(!persistent_fsms && verify_tag(fid, H5AC_FSPACE_SINFO_ID, H5AC__FREESPACE_TAG) < 0) TEST_ERROR; + if ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 ) TEST_ERROR; /* verify btree header and leaf node belonging to group */ if ( verify_tag(fid, H5AC_BT2_HDR_ID, g_tag) < 0 ) TEST_ERROR; @@ -1828,13 +1818,16 @@ check_attribute_delete_tags(hid_t fcpl, int type) /* * 2 calls to verify_tag() for verifying free space: - * one freespace header tag for free-space header, - * one freespace header tag for free-space section info + * one freespace header tag for free-space header raw data + * one freespace header tag for free-space section info raw data + * one freespace header tag for free-space header metadata */ if ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 ) TEST_ERROR; if ( verify_tag(fid, H5AC_FSPACE_SINFO_ID, H5AC__FREESPACE_TAG) < 0 ) TEST_ERROR; + if ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 ) + TEST_ERROR; #if 0 /* If the free space managers are persistent, the diff --git a/test/fheap.c b/test/fheap.c index 846f03e..557fd37 100644 --- a/test/fheap.c +++ b/test/fheap.c @@ -1904,7 +1904,7 @@ test_create(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tparam) TEST_ERROR /* Close the fractal heap */ - if(H5HF_close(fh) < 0) + if((fh = H5HF_close(fh)) < 0) FAIL_STACK_ERROR /* Delete heap */ @@ -6643,7 +6643,7 @@ test_man_remove_one(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpara FAIL_STACK_ERROR /* Close the fractal heap */ - if(H5HF_close(fh) < 0) + if((fh = H5HF_close(fh)) < 0) TEST_ERROR /* Close the file */ @@ -6834,7 +6834,7 @@ test_man_remove_two(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpara FAIL_STACK_ERROR /* Close the fractal heap */ - if(H5HF_close(fh) < 0) + if((fh = H5HF_close(fh)) < 0) TEST_ERROR /* Close the file */ @@ -7001,7 +7001,7 @@ test_man_remove_one_larger(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t FAIL_STACK_ERROR /* Close the fractal heap */ - if(H5HF_close(fh) < 0) + if((fh = H5HF_close(fh)) < 0) FAIL_STACK_ERROR /* Close the file */ @@ -7239,7 +7239,7 @@ test_man_remove_two_larger(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t FAIL_STACK_ERROR /* Close the fractal heap */ - if(H5HF_close(fh) < 0) + if((fh = H5HF_close(fh)) < 0) TEST_ERROR /* Close the file */ @@ -7541,7 +7541,7 @@ test_man_remove_three_larger(hid_t fapl, H5HF_create_t *cparam, fheap_test_param FAIL_STACK_ERROR /* Close the fractal heap */ - if(H5HF_close(fh) < 0) + if((fh = H5HF_close(fh)) < 0) TEST_ERROR /* Close the file */ @@ -13683,23 +13683,23 @@ test_filtered_huge(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tparam H5F_t *f = NULL; /* Internal file object pointer */ H5HF_t *fh = NULL; /* Fractal heap wrapper */ haddr_t fh_addr; /* Address of fractal heap */ - H5HF_create_t tmp_cparam; /* Local heap creation parameters */ - fheap_heap_ids_t keep_ids; /* Structure to retain heap IDs */ - size_t id_len; /* Size of fractal heap IDs */ - h5_stat_size_t empty_size; /* Size of a file with an empty heap */ - h5_stat_size_t file_size; /* Size of file currently */ - unsigned char *heap_id = NULL; /* Heap ID for object */ - size_t obj_size; /* Size of object */ - size_t robj_size; /* Size of object read */ - unsigned char obj_type; /* Type of storage for object */ - fheap_heap_state_t state; /* State of fractal heap */ - unsigned deflate_level; /* Deflation level */ - size_t old_actual_id_len =0 ; /* Old actual ID length */ - hbool_t huge_ids_direct; /* Are 'huge' objects directly acccessed? */ - const char *base_desc = "insert 'huge' object into heap with I/O filters, then remove %s"; /* Test description */ + H5HF_create_t tmp_cparam; /* Local heap creation parameters */ + fheap_heap_ids_t keep_ids; /* Structure to retain heap IDs */ + size_t id_len; /* Size of fractal heap IDs */ + h5_stat_size_t empty_size; /* Size of a file with an empty heap */ + h5_stat_size_t file_size; /* Size of file currently */ + unsigned char *heap_id = NULL; /* Heap ID for object */ + size_t obj_size; /* Size of object */ + size_t robj_size; /* Size of object read */ + unsigned char obj_type; /* Type of storage for object */ + fheap_heap_state_t state; /* State of fractal heap */ + unsigned deflate_level; /* Deflation level */ + size_t old_actual_id_len =0 ; /* Old actual ID length */ + hbool_t huge_ids_direct; /* Are 'huge' objects directly acccessed? */ + const char *base_desc = "insert 'huge' object into heap with I/O filters, then remove %s"; /* Test description */ - /* Copy heap creation properties */ - HDmemcpy(&tmp_cparam, cparam, sizeof(H5HF_create_t)); + /* Copy heap creation properties */ + HDmemcpy(&tmp_cparam, cparam, sizeof(H5HF_create_t)); /* Set an I/O filter for heap data */ deflate_level = 6; @@ -13867,7 +13867,9 @@ error: H5MM_xfree(heap_id); if(fh) H5HF_close(fh); - H5Fclose(file); + if(&tmp_cparam.pline) + H5O_msg_reset(H5O_PLINE_ID, &tmp_cparam.pline); /* Release the I/O pipeline filter information */ + H5Fclose(file); } H5E_END_TRY; return(1); } /* test_filtered_huge() */ @@ -15571,9 +15573,12 @@ error: H5MM_xfree(keep_ids.ids); H5MM_xfree(keep_ids.lens); H5MM_xfree(keep_ids.offs); + + if(tparam->comp == FHEAP_TEST_COMPRESS) + H5O_msg_reset(H5O_PLINE_ID, &tmp_cparam.pline); /* Release the I/O pipeline filter information */ if(fh) H5HF_close(fh); - H5Fclose(file); + H5Fclose(file); } H5E_END_TRY; return(1); } /* test_random() */ @@ -15789,7 +15794,9 @@ error: H5MM_xfree(keep_ids.offs); if(fh) H5HF_close(fh); - H5Fclose(file); + if(tparam->comp == FHEAP_TEST_COMPRESS) + H5O_msg_reset(H5O_PLINE_ID, &tmp_cparam.pline); /* Release the I/O pipeline filter information */ + H5Fclose(file); } H5E_END_TRY; return(1); } /* test_random_pow2() */ @@ -6822,16 +6822,6 @@ test_mf_fs_persist(const char *env_h5_drvr, hid_t fapl, hbool_t new_format) if(!H5F_addr_defined(f->shared->fs_addr[tt])) TEST_ERROR - /* Since we are about to open a self referential free space - * manager prior to the first file space allocation / deallocation - * call H5MF_tidy_self_referential_fsm_hack() first so as to avoid - * assertion failures on the first file space alloc / dealloc. - */ - if((f->shared->first_alloc_dealloc) && - (SUCCEED != - H5MF_tidy_self_referential_fsm_hack(f))) - FAIL_STACK_ERROR - /* Start up H5FD_MEM_SUPER free-space manager */ if(!(f->shared->fs_man[tt])) if(H5MF__open_fstype(f, (H5F_mem_page_t)tt) < 0) @@ -6925,7 +6915,7 @@ test_mf_fs_gone(const char *env_h5_drvr, hid_t fapl, hbool_t new_format) haddr_t addrx; H5FD_mem_t fs_type; hbool_t contig_addr_vfd; - hbool_t ran_H5MF_tidy_self_referential_fsm_hack = FALSE; + if(new_format) TESTING("File's free-space is going away with new library format") @@ -6944,9 +6934,13 @@ test_mf_fs_gone(const char *env_h5_drvr, hid_t fapl, hbool_t new_format) /* Copy the file access property list */ if((fapl2 = H5Pcopy(fapl)) < 0) FAIL_STACK_ERROR - if(new_format) + if(new_format) { if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) FAIL_STACK_ERROR + } else { + if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_V18, H5F_LIBVER_LATEST) < 0) + FAIL_STACK_ERROR + } /* Set to aggregation and persisting free-space */ if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1) < 0) @@ -7022,17 +7016,6 @@ test_mf_fs_gone(const char *env_h5_drvr, hid_t fapl, hbool_t new_format) if(!H5F_addr_defined(f->shared->fs_addr[fs_type])) TEST_ERROR - /* Since we are about to open a self referential free space - * manager prior to the first file space allocation / deallocation - * call H5MF_tidy_self_referential_fsm_hack() first so as to avoid - * assertion failures on the first file space alloc / dealloc. - */ - if(f->shared->first_alloc_dealloc){ - if(SUCCEED != H5MF_tidy_self_referential_fsm_hack(f)) - FAIL_STACK_ERROR - ran_H5MF_tidy_self_referential_fsm_hack = TRUE; - } - /* Start up H5FD_MEM_SUPER free-space manager */ if(!(f->shared->fs_man[fs_type])) if(H5MF__open_fstype(f, (H5F_mem_page_t)fs_type) < 0) @@ -7042,24 +7025,18 @@ test_mf_fs_gone(const char *env_h5_drvr, hid_t fapl, hbool_t new_format) if(H5FS_stat_info(f, f->shared->fs_man[fs_type], &fs_stat) < 0) FAIL_STACK_ERROR - /* if we ran H5MF_tidy_self_referential_fsm_hack(), the - * H5FD_MEM_SUPER free space manager must be floating. - * Thus fs_stat.addr must be undefined. - */ - if((!ran_H5MF_tidy_self_referential_fsm_hack) && - (!H5F_addr_defined(fs_stat.addr))) + if(!H5F_addr_defined(fs_stat.addr)) TEST_ERROR if(fs_stat.tot_space < TBLOCK_SIZE3) TEST_ERROR - /* Put block #4 to H5FD_MEM_SUPER free-space manager */ - if(H5MF_xfree(f, type, addr4, (hsize_t)TBLOCK_SIZE4) < 0) - FAIL_STACK_ERROR - - if(!new_format) { - /* Need to take up this space so that the free-space manager will go away */ - if(HADDR_UNDEF == (addrx = H5MF_alloc(f, type, (hsize_t)103))) + /* Allocate/free space accordingly so that the free-space manager will go away */ + if(new_format) { + if(H5MF_xfree(f, type, addr4, (hsize_t)TBLOCK_SIZE4) < 0) + FAIL_STACK_ERROR + } else { + if(HADDR_UNDEF == (addrx = H5MF_alloc(f, type, (hsize_t)3))) FAIL_STACK_ERROR } @@ -7741,16 +7718,6 @@ test_page_alloc_xfree(const char *env_h5_drvr, hid_t fapl) if(!H5F_addr_defined(f->shared->fs_addr[fs_type])) TEST_ERROR - /* Since we are about to open a self referential free space - * manager prior to the first file space allocation / deallocation - * call H5MF_tidy_self_referential_fsm_hack() first so as to avoid - * assertion failures on the first file space alloc / dealloc. - */ - if(f->shared->first_alloc_dealloc){ - if(SUCCEED != H5MF_tidy_self_referential_fsm_hack(f)) - FAIL_STACK_ERROR - } - /* Set up to use the small metadata manager */ if(!(f->shared->fs_man[fs_type])) if(H5MF__open_fstype(f, fs_type) < 0) diff --git a/test/titerate.c b/test/titerate.c index 54e9b5e..716654d 100644 --- a/test/titerate.c +++ b/test/titerate.c @@ -939,7 +939,7 @@ static void test_links(hid_t fapl) *------------------------------------------------------------------------- */ static int -find_err_msg_cb(unsigned n, const H5E_error2_t *err_desc, void *_client_data) +find_err_msg_cb(unsigned H5_ATTR_UNUSED n, const H5E_error2_t *err_desc, void *_client_data) { int status = H5_ITER_CONT; searched_err_t *searched_err = (searched_err_t *)_client_data; diff --git a/test/tselect.c b/test/tselect.c index ad435cb..bbab868 100644 --- a/test/tselect.c +++ b/test/tselect.c @@ -14925,7 +14925,6 @@ test_sel_iter(void) hid_t sid; /* Dataspace ID */ hid_t iter_id; /* Dataspace selection iterator ID */ hsize_t dims1[] = {6, 12}; /* 2-D Dataspace dimensions */ - hsize_t dims2[] = {32}; /* 1-D dataspace dimensions */ hsize_t coord1[POINT1_NPOINTS][2]; /* Coordinates for point selection */ hsize_t start[2]; /* Hyperslab start */ hsize_t stride[2]; /* Hyperslab stride */ diff --git a/test/tsohm.c b/test/tsohm.c index 8cf4d2f..99df6b7 100644 --- a/test/tsohm.c +++ b/test/tsohm.c @@ -11,13 +11,9 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/*********************************************************** -* -* Test program: tsohm -* -* Test Shared Object Header Messages -* -*************************************************************/ +/* + * Purpose: Test Shared Object Header Messages + */ #include "testhdf5.h" @@ -582,12 +578,12 @@ error: * * Purpose: Creates object headers that use a large datatype message. * - * Set test_file_closing to 1 to add file closing and reopening + * Set test_file_closing to TRUE to add file closing and reopening * whenever possible (to test that SOHMs are written correctly * on disk and not just in memory). * - * Return: Success: file ID (may not be the same one passed in) - * Failure: Negative + * Return: Success: file ID (may not be the same one passed in) + * Failure: H5I_INVALID_HID * * Programmer: James Laird * Monday, April 10, 2006 @@ -595,13 +591,13 @@ error: *------------------------------------------------------------------------- */ static hid_t -size1_helper(hid_t file, const char *filename, hid_t fapl_id, unsigned test_file_closing) +size1_helper(hid_t file, const char *filename, hid_t fapl_id, hbool_t test_file_closing) { dtype1_struct wdata; dtype1_struct rdata; - hid_t dtype1_id = -1; - hid_t space_id = -1; - hid_t dset_id = -1; + hid_t dtype1_id = H5I_INVALID_HID; + hid_t space_id = H5I_INVALID_HID; + hid_t dset_id = H5I_INVALID_HID; hsize_t dim1[1]; int x; @@ -635,7 +631,7 @@ size1_helper(hid_t file, const char *filename, hid_t fapl_id, unsigned test_file * local disks. Don't close and reopen if express testing is enabled. */ if(GetTestExpress() > 1) - test_file_closing = 0; + test_file_closing = FALSE; /* Intialize wdata */ HDmemset(&wdata, 0, sizeof(wdata)); @@ -718,6 +714,7 @@ size1_helper(hid_t file, const char *filename, hid_t fapl_id, unsigned test_file if(H5Dclose(dset_id) < 0) TEST_ERROR if(H5Tclose(dtype1_id) < 0) TEST_ERROR + return file; error: @@ -727,7 +724,8 @@ size1_helper(hid_t file, const char *filename, hid_t fapl_id, unsigned test_file H5Dclose(dset_id); H5Fclose(file); } H5E_END_TRY - return -1; + + return H5I_INVALID_HID; #undef TSOHM_S1H_VERIFY_DATA /* macro is exclusive to this function */ } /* size1_helper */ @@ -744,25 +742,30 @@ size1_helper(hid_t file, const char *filename, hid_t fapl_id, unsigned test_file *---------------------------------------------------------------------------- */ static h5_stat_size_t -getsize_testsize1(const char *filename, hid_t fcpl_id, hid_t fapl_id, unsigned open_close, H5O_info_t *oinfo) +getsize_testsize1(const char *filename, hid_t fcpl_id, hid_t fapl_id, + hbool_t test_file_closing, H5O_info_t *oinfo) { - hid_t file = -1; + hid_t fid = H5I_INVALID_HID; herr_t ret; - file = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, fapl_id); - CHECK_I(file, "H5Fcreate"); + fid = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl_id, fapl_id); + CHECK(fid, H5I_INVALID_HID, "H5Fcreate"); - file = size1_helper(file, filename, fapl_id, open_close); - CHECK_I(file, "size1_helper"); + /* If test_file_closing is TRUE, you will get back a different ID, + * which will need to be closed. The helper will close your passed-in + * ID. + */ + fid = size1_helper(fid, filename, fapl_id, test_file_closing); + CHECK(fid, H5I_INVALID_HID, "size1_helper"); - ret = H5Oget_info_by_name2(file, DSETNAME[0], oinfo, H5O_INFO_HDR, H5P_DEFAULT); - CHECK_I(ret, "H5Oget_info_by_name"); + ret = H5Oget_info_by_name2(fid, DSETNAME[0], oinfo, H5O_INFO_HDR, H5P_DEFAULT); + CHECK(ret, FAIL, "H5Oget_info_by_name"); - ret = H5Fclose(file); - CHECK_I(ret, "H5Fclose"); + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); return h5_get_file_size(filename, fapl_id); -} /* getsize_testsize1 */ +} /* getsize_testsize1() */ /*------------------------------------------------------------------------- @@ -784,7 +787,6 @@ test_sohm_size1(void) unsigned use_shared = 0; unsigned use_btree = 0; - unsigned open_close = 0; h5_stat_size_t file_sizes[9]; unsigned size_index = 0; hsize_t oh_sizes[3]; @@ -827,6 +829,7 @@ test_sohm_size1(void) for (use_shared = 0; use_shared < 2; use_shared++) { for (use_btree = 0; use_btree < 2; use_btree++) { + hbool_t test_open_close; /* cannot use btree indexing without shared messages; skip case */ if (use_btree && !use_shared) @@ -862,8 +865,10 @@ test_sohm_size1(void) file_sizes[size_index++] = h5_get_file_size(FILENAME, fapl_id); /* size of populated file, with different populating behaviors */ - for (open_close = 0; open_close < 2; open_close++) - file_sizes[size_index++] = getsize_testsize1(FILENAME, fcpl_id, fapl_id, open_close, &oinfo); + test_open_close = TRUE; + file_sizes[size_index++] = getsize_testsize1(FILENAME, fcpl_id, fapl_id, test_open_close, &oinfo); + test_open_close = FALSE; + file_sizes[size_index++] = getsize_testsize1(FILENAME, fcpl_id, fapl_id, test_open_close, &oinfo); oh_sizes[oh_size_index++] = oinfo.hdr.space.total; ret = H5Pclose(fcpl_id); @@ -960,6 +965,7 @@ test_sohm_size1(void) * *--------------------------------------------------------------------------- */ +#if 0 /* TODO: REVEALS BUG TO BE FIXED - SEE JIRA HDFFV-10645 */ static void test_sohm_size_consistency_open_create(void) { @@ -1015,7 +1021,7 @@ test_sohm_size_consistency_open_create(void) * Add messages to previously-created file */ file = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl_id); CHECK_I(file, "H5Fopen"); - file = size1_helper(file, FILENAME, fapl_id, 0); + file = size1_helper(file, FILENAME, fapl_id, FALSE); CHECK_I(file, "size1_helper"); /* Get the size of a dataset object header */ @@ -1030,7 +1036,7 @@ test_sohm_size_consistency_open_create(void) * Add messages to a newly-created file */ file = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl_id, fapl_id); CHECK_I(file, "H5Fcreate"); - file = size1_helper(file, FILENAME, fapl_id, 0); + file = size1_helper(file, FILENAME, fapl_id, FALSE); CHECK_I(file, "size1_helper"); /* Get the size of a dataset object header */ @@ -1181,8 +1187,8 @@ sohm_attr_helper(hid_t fcpl_id) static void test_sohm_attrs(void) { - hid_t fcpl_id; - hid_t file_id; + hid_t bad_fid = H5I_INVALID_HID; + hid_t fcpl_id = H5I_INVALID_HID; unsigned i = 0; #define TSOHM_TSA_NFLAGS_1 7 unsigned flags1[TSOHM_TSA_NFLAGS_1] = { @@ -1300,8 +1306,8 @@ test_sohm_attrs(void) ret = H5Pset_shared_mesg_index(fcpl_id, 1, H5O_SHMESG_ATTR_FLAG, 2); CHECK_I(ret, "H5Pset_shared_mesg_nindexes"); - file_id = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl_id, H5P_DEFAULT); - VERIFY(file_id, -1, "H5Fcreate"); + bad_fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl_id, H5P_DEFAULT); + VERIFY(bad_fid, H5I_INVALID_HID, "H5Fcreate"); ret = H5Pclose(fcpl_id); CHECK_I(ret, "H5Pclose"); diff --git a/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl b/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl index affa93b..e78c5e2 100644 --- a/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl +++ b/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl @@ -11,6 +11,6 @@ File space page size: 4096 bytes Summary of file space information: File metadata: 1602 bytes Raw data: 400 bytes - Amount/Percent of tracked free space: 6307 bytes/38.5% - Unaccounted space: 8075 bytes -Total space: 16384 bytes + Amount/Percent of tracked free space: 6190 bytes/75.6% + Unaccounted space: 0 bytes +Total space: 8192 bytes diff --git a/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl b/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl index 1ce06b2..3ecf9b1 100644 --- a/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl +++ b/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl @@ -11,6 +11,6 @@ File space page size: 4096 bytes Summary of file space information: File metadata: 1485 bytes Raw data: 400 bytes - Amount/Percent of tracked free space: 680 bytes/26.5% + Amount/Percent of tracked free space: 563 bytes/23.0% Unaccounted space: 0 bytes -Total space: 2565 bytes +Total space: 2448 bytes diff --git a/tools/test/perform/pio_standalone.c b/tools/test/perform/pio_standalone.c index d5f681e..dd6a61a 100644 --- a/tools/test/perform/pio_standalone.c +++ b/tools/test/perform/pio_standalone.c @@ -163,121 +163,3 @@ print_version(const char *progname) } - -/** From h5test.c **/ - -#ifdef H5_HAVE_PARALLEL -MPI_Info h5_io_info_g=MPI_INFO_NULL;/* MPI INFO object for IO */ -#endif - -#if 0 -int -h5_set_info_object(void) -{ - char *envp; /* environment pointer */ - int ret_value=0; - - /* handle any MPI INFO hints via $HDF5_MPI_INFO */ - if ((envp = getenv("HDF5_MPI_INFO")) != NULL){ - char *next, *valp; - - - valp = envp = next = HDstrdup(envp); - - /* create an INFO object if not created yet */ - if (h5_io_info_g == MPI_INFO_NULL) - MPI_Info_create(&h5_io_info_g); - - do { - size_t len; - char *key_val, *endp, *namep; - - if (*valp == ';') - valp++; - - /* copy key/value pair into temporary buffer */ - len = strcspn(valp, ";"); - next = &valp[len]; - key_val = calloc(1, len + 1); - - /* increment the next pointer past the terminating semicolon */ - if (*next == ';') - ++next; - - namep = HDstrncpy(key_val, valp, len); - - /* pass up any beginning whitespaces */ - while (*namep && (*namep == ' ' || *namep == '\t')) - namep++; - - /* eat up any ending white spaces */ - endp = &namep[strlen(namep) - 1]; - - while (endp && (*endp == ' ' || *endp == '\t')) - *endp-- = '\0'; - - /* find the '=' */ - - valp = HDstrchr(namep, '='); - - if (valp != NULL) { /* it's a valid key/value pairing */ - char *tmp_val = valp + 1; - - /* change '=' to \0, move valp down one */ - *valp-- = '\0'; - - /* eat up ending whitespace on the "key" part */ - while (*valp == ' ' || *valp == '\t') - *valp-- = '\0'; - - valp = tmp_val; - - /* eat up beginning whitespace on the "value" part */ - while (*valp == ' ' || *valp == '\t') - *valp++ = '\0'; - - /* actually set the darned thing */ - if (MPI_SUCCESS != MPI_Info_set(h5_io_info_g, namep, valp)) { - printf("MPI_Info_set failed\n"); - ret_value = -1; - } - } - - valp = next; - HDfree(key_val); - } while (next && *next); - - HDfree(envp); - } - - return ret_value; -} - - -void -h5_dump_info_object(MPI_Info info) -{ - char key[MPI_MAX_INFO_KEY+1]; - char value[MPI_MAX_INFO_VAL+1]; - int flag; - int i, nkeys; - - printf("Dumping MPI Info Object(%d) (up to %d bytes per item):\n", (int)info, - MPI_MAX_INFO_VAL); - if (info==MPI_INFO_NULL){ - printf("object is MPI_INFO_NULL\n"); - } - else { - MPI_Info_get_nkeys(info, &nkeys); - printf("object has %d items\n", nkeys); - for (i=0; i<nkeys; i++){ - MPI_Info_get_nthkey(info, i, key); - MPI_Info_get(info, key, MPI_MAX_INFO_VAL, value, &flag); - printf("%s=%s\n", key, value); - } - - } -} - -#endif - diff --git a/tools/test/perform/sio_standalone.c b/tools/test/perform/sio_standalone.c index dfdbc55..ac191e5 100644 --- a/tools/test/perform/sio_standalone.c +++ b/tools/test/perform/sio_standalone.c @@ -162,122 +162,3 @@ print_version(const char *progname) H5_VERS_SUBRELEASE[0] ? "-" : "", H5_VERS_SUBRELEASE); } - - -/** From h5test.c **/ - -#ifdef H5_HAVE_PARALLEL -MPI_Info h5_io_info_g=MPI_INFO_NULL;/* MPI INFO object for IO */ -#endif - -#if 0 -int -h5_set_info_object(void) -{ - char *envp; /* environment pointer */ - int ret_value=0; - - /* handle any MPI INFO hints via $HDF5_MPI_INFO */ - if ((envp = getenv("HDF5_MPI_INFO")) != NULL){ - char *next, *valp; - - - valp = envp = next = HDstrdup(envp); - - /* create an INFO object if not created yet */ - if (h5_io_info_g == MPI_INFO_NULL) - MPI_Info_create(&h5_io_info_g); - - do { - size_t len; - char *key_val, *endp, *namep; - - if (*valp == ';') - valp++; - - /* copy key/value pair into temporary buffer */ - len = strcspn(valp, ";"); - next = &valp[len]; - key_val = calloc(1, len + 1); - - /* increment the next pointer past the terminating semicolon */ - if (*next == ';') - ++next; - - namep = HDstrncpy(key_val, valp, len); - - /* pass up any beginning whitespaces */ - while (*namep && (*namep == ' ' || *namep == '\t')) - namep++; - - /* eat up any ending white spaces */ - endp = &namep[strlen(namep) - 1]; - - while (endp && (*endp == ' ' || *endp == '\t')) - *endp-- = '\0'; - - /* find the '=' */ - - valp = HDstrchr(namep, '='); - - if (valp != NULL) { /* it's a valid key/value pairing */ - char *tmp_val = valp + 1; - - /* change '=' to \0, move valp down one */ - *valp-- = '\0'; - - /* eat up ending whitespace on the "key" part */ - while (*valp == ' ' || *valp == '\t') - *valp-- = '\0'; - - valp = tmp_val; - - /* eat up beginning whitespace on the "value" part */ - while (*valp == ' ' || *valp == '\t') - *valp++ = '\0'; - - /* actually set the darned thing */ - if (MPI_SUCCESS != MPI_Info_set(h5_io_info_g, namep, valp)) { - printf("MPI_Info_set failed\n"); - ret_value = -1; - } - } - - valp = next; - HDfree(key_val); - } while (next && *next); - - HDfree(envp); - } - - return ret_value; -} - - -void -h5_dump_info_object(MPI_Info info) -{ - char key[MPI_MAX_INFO_KEY+1]; - char value[MPI_MAX_INFO_VAL+1]; - int flag; - int i, nkeys; - - printf("Dumping MPI Info Object(%d) (up to %d bytes per item):\n", (int)info, - MPI_MAX_INFO_VAL); - if (info==MPI_INFO_NULL){ - printf("object is MPI_INFO_NULL\n"); - } - else { - MPI_Info_get_nkeys(info, &nkeys); - printf("object has %d items\n", nkeys); - for (i=0; i<nkeys; i++){ - MPI_Info_get_nthkey(info, i, key); - MPI_Info_get(info, key, MPI_MAX_INFO_VAL, value, &flag); - printf("%s=%s\n", key, value); - } - - } -} - -#endif - |