summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorVailin Choi <vchoi@hdfgroup.org>2019-06-28 14:54:52 (GMT)
committerVailin Choi <vchoi@hdfgroup.org>2019-06-28 14:54:52 (GMT)
commitb6268dbb1b4548be0a7bb5605001cb1859daec30 (patch)
treee11f5f9a42b50563d9e4081c54a758234469e236
parent2c69584fb124c20e1dd06c31f2baf240b50b381c (diff)
parentc69b1025f58ec8467be1e056214510c70ed00867 (diff)
downloadhdf5-b6268dbb1b4548be0a7bb5605001cb1859daec30.zip
hdf5-b6268dbb1b4548be0a7bb5605001cb1859daec30.tar.gz
hdf5-b6268dbb1b4548be0a7bb5605001cb1859daec30.tar.bz2
Merge pull request #1757 in HDFFV/hdf5 from ~VCHOI/my_third_fork:bugfix/new_shutdown_fsm to develop
* commit 'c69b1025f58ec8467be1e056214510c70ed00867': (1) Add/remove comments. (2) A temporary fix to address the test/objcopy.c: test_copy_group_deep() test failure with the family driver. The test failure occurs with these configurations in objcopy.c: --with shared src messages (CONFIG_SHARE_SRC) --without shared dst messages (CONFIG_SHARE_DST) --with latest format for source file (CONFIG_SRC_NEW_FORMAT) --without dense attributes (CONFIG_DENSE) --with latest format for destination file (CONFIG_DST_NEW_FORMAT) The temporary fix is in src/H5MFaggr.c (see comments above #ifdef REPLACE/#endif). Checkin for new shutting down free-space manager.
-rw-r--r--src/H5AC.c68
-rw-r--r--src/H5ACprivate.h1
-rw-r--r--src/H5Cprivate.h1
-rw-r--r--src/H5Cquery.c28
-rw-r--r--src/H5FScache.c60
-rw-r--r--src/H5FSsection.c333
-rw-r--r--src/H5Fint.c6
-rw-r--r--src/H5Fpkg.h11
-rw-r--r--src/H5Fprivate.h9
-rw-r--r--src/H5Fquery.c36
-rw-r--r--src/H5Fsuper.c67
-rw-r--r--src/H5MF.c592
-rw-r--r--src/H5MFaggr.c16
-rw-r--r--src/H5MFdbg.c8
-rw-r--r--src/H5Ocache_image.c33
-rw-r--r--test/cache_image.c29
-rw-r--r--test/cache_tagging.c23
-rw-r--r--test/fheap.c57
-rw-r--r--test/mf.c59
-rw-r--r--tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl6
-rw-r--r--tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl4
21 files changed, 540 insertions, 907 deletions
diff --git a/src/H5AC.c b/src/H5AC.c
index dc88fd4..f6243ad 100644
--- a/src/H5AC.c
+++ b/src/H5AC.c
@@ -698,46 +698,6 @@ done:
/*-------------------------------------------------------------------------
- * Function: H5AC_force_cache_image_load()
- *
- * Purpose: On rare occasions, it is necessary to run
- * H5MF_tidy_self_referential_fsm_hack() prior to the first
- * metadata cache access. This is a problem as if there is a
- * cache image at the end of the file, that routine will
- * discard it.
- *
- * We solve this issue by calling this function, which will
- * load the cache image and then call
- * H5MF_tidy_self_referential_fsm_hack() to discard it.
- *
- * Return: SUCCEED on success, and FAIL on failure.
- *
- * Programmer: John Mainzer
- * 1/11/17
- *
- *-------------------------------------------------------------------------
- */
-herr_t
-H5AC_force_cache_image_load(H5F_t *f)
-{
- herr_t ret_value = SUCCEED; /* Return value */
-
- FUNC_ENTER_NOAPI(FAIL)
-
- /* Sanity checks */
- HDassert(f);
- HDassert(f->shared);
- HDassert(f->shared->cache);
-
- if(H5C_force_cache_image_load(f) < 0)
- HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, FAIL, "Can't load cache image")
-
-done:
- FUNC_LEAVE_NOAPI(ret_value)
-} /* H5AC_force_cache_image_load() */
-
-
-/*-------------------------------------------------------------------------
* Function: H5AC_get_entry_status
*
* Purpose: Given a file address, determine whether the metadata
@@ -1744,6 +1704,33 @@ done:
/*-------------------------------------------------------------------------
+ * Function: H5AC_get_cache_flush_in_progess
+ *
+ * Purpose: Wrapper function for H5C_get_cache_flush_in_progress().
+ *
+ * Return: SUCCEED on success, and FAIL on failure.
+ *
+ * Programmer: John Mainzer
+ * 3/11/05
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5AC_get_cache_flush_in_progress(H5AC_t *cache_ptr, hbool_t *flush_in_progress_ptr)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ if(H5C_get_cache_flush_in_progress((H5C_t *)cache_ptr, flush_in_progress_ptr) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_get_cache_flush_in_progress() failed")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5AC_get_cache_flush_in_progress() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5AC_get_cache_hit_rate
*
* Purpose: Wrapper function for H5C_get_cache_hit_rate().
@@ -2682,4 +2669,3 @@ H5AC_get_mdc_image_info(H5AC_t *cache_ptr, haddr_t *image_addr, hsize_t *image_l
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5AC_get_mdc_image_info() */
-
diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h
index e1fdedf..691f7c6 100644
--- a/src/H5ACprivate.h
+++ b/src/H5ACprivate.h
@@ -411,6 +411,7 @@ H5_DLL herr_t H5AC_get_cache_auto_resize_config(const H5AC_t * cache_ptr,
H5AC_cache_config_t *config_ptr);
H5_DLL herr_t H5AC_get_cache_size(H5AC_t *cache_ptr, size_t *max_size_ptr,
size_t *min_clean_size_ptr, size_t *cur_size_ptr, uint32_t *cur_num_entries_ptr);
+H5_DLL herr_t H5AC_get_cache_flush_in_progress(H5AC_t *cache_ptr, hbool_t *flush_in_progress_ptr);
H5_DLL herr_t H5AC_get_cache_hit_rate(H5AC_t *cache_ptr, double *hit_rate_ptr);
H5_DLL herr_t H5AC_reset_cache_hit_rate_stats(H5AC_t *cache_ptr);
H5_DLL herr_t H5AC_set_cache_auto_resize_config(H5AC_t *cache_ptr,
diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h
index d4ed6fc..bd200e0 100644
--- a/src/H5Cprivate.h
+++ b/src/H5Cprivate.h
@@ -2258,6 +2258,7 @@ H5_DLL herr_t H5C_get_cache_image_config(const H5C_t * cache_ptr,
H5_DLL herr_t H5C_get_cache_size(H5C_t *cache_ptr, size_t *max_size_ptr,
size_t *min_clean_size_ptr, size_t *cur_size_ptr,
uint32_t *cur_num_entries_ptr);
+H5_DLL herr_t H5C_get_cache_flush_in_progress(H5C_t *cache_ptr, hbool_t *flush_in_progress_ptr);
H5_DLL herr_t H5C_get_cache_hit_rate(H5C_t *cache_ptr, double *hit_rate_ptr);
H5_DLL herr_t H5C_get_entry_status(const H5F_t *f, haddr_t addr,
size_t *size_ptr, hbool_t *in_cache_ptr, hbool_t *is_dirty_ptr,
diff --git a/src/H5Cquery.c b/src/H5Cquery.c
index 51ce1c2..a1267c5 100644
--- a/src/H5Cquery.c
+++ b/src/H5Cquery.c
@@ -156,6 +156,34 @@ done:
/*-------------------------------------------------------------------------
+ * Function: H5C_get_cache_flush_in_progress
+ *
+ * Purpose: Return flush_in_progress in *flush_in_progress_ptr
+ * If the parameter is NULL, skip that value.
+ *
+ * Return: SUCCEED on success, and FAIL on failure.
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5C_get_cache_flush_in_progress(H5C_t * cache_ptr, hbool_t *flush_in_progress_ptr)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ if((cache_ptr == NULL) || (cache_ptr->magic != H5C__H5C_T_MAGIC))
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Bad cache_ptr on entry.")
+
+ if(flush_in_progress_ptr != NULL)
+ *flush_in_progress_ptr = cache_ptr->flush_in_progress;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5C_get_cache_flush_in_progress() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5C_get_cache_hit_rate
*
* Purpose: Compute and return the current cache hit rate in
diff --git a/src/H5FScache.c b/src/H5FScache.c
index 7525a9a..b520458 100644
--- a/src/H5FScache.c
+++ b/src/H5FScache.c
@@ -517,14 +517,36 @@ H5FS__cache_hdr_pre_serialize(H5F_t *f, void *_thing,
if(!H5F_addr_defined(fspace->sect_addr)) { /* case 1 */
haddr_t tag = HADDR_UNDEF;
+ haddr_t sect_addr;
+ hsize_t saved_sect_size, new_sect_size;
/* allocate file space for the section info, and insert it
* into the metadata cache.
*/
- if(HADDR_UNDEF == (fspace->sect_addr = H5MF_alloc((H5F_t *)f, H5FD_MEM_FSPACE_SINFO, fspace->sect_size)))
+ saved_sect_size = fspace->sect_size;
+ if(HADDR_UNDEF == (sect_addr = H5MF_alloc((H5F_t *)f, H5FD_MEM_FSPACE_SINFO, fspace->sect_size)))
HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "file allocation failed for free space sections")
- fspace->alloc_sect_size = (size_t)fspace->sect_size;
+ /* fspace->sect_size may change in size after H5MF_alloc().
+ * If increased in size, free the previous allocation and
+ * allocate again with the bigger fspace->sect_size.
+ */
+ if(fspace->sect_size > saved_sect_size) {
+
+ new_sect_size = fspace->sect_size;
+
+ if(H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, sect_addr, saved_sect_size) < 0)
+ HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to free free space sections")
+
+ if(HADDR_UNDEF == (sect_addr = H5MF_alloc((H5F_t *)f, H5FD_MEM_FSPACE_SINFO, new_sect_size)))
+ HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "file allocation failed for free space sections")
+ fspace->sect_size = new_sect_size;
+ fspace->alloc_sect_size = new_sect_size;
+ } else {
+ fspace->alloc_sect_size = saved_sect_size;
+ fspace->sect_size = saved_sect_size;
+ }
+ fspace->sect_addr = sect_addr;
/* Get the tag for this free space manager and use it to insert the entry */
if(H5AC_get_tag((const void *)fspace, &tag) < 0)
@@ -593,7 +615,6 @@ H5FS__cache_hdr_pre_serialize(H5F_t *f, void *_thing,
* a nonsense section info address.
*/
if(!H5F_POINT_OF_NO_RETURN(f)) {
- HDassert(fspace->serial_sect_count > 0);
HDassert(fspace->sect_size > 0);
HDassert(fspace->alloc_sect_size == (size_t)fspace->sect_size);
} /* end if */
@@ -703,9 +724,8 @@ H5FS__cache_hdr_serialize(const H5F_t *f, void *_image, size_t len,
if(!H5F_POINT_OF_NO_RETURN(f))
HDassert((! H5F_addr_defined(fspace->sect_addr)) ||
- ((fspace->serial_sect_count > 0) &&
- (fspace->sect_size > 0) &&
- (fspace->alloc_sect_size == (size_t)fspace->sect_size)));
+ ((fspace->sect_size > 0) &&
+ (fspace->alloc_sect_size == (size_t)fspace->sect_size)));
/* Magic number */
H5MM_memcpy(image, H5FS_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC);
@@ -968,6 +988,7 @@ H5FS__cache_sinfo_deserialize(const void *_image, size_t len, void *_udata,
haddr_t fs_addr; /* Free space header address */
size_t old_sect_size; /* Old section size */
const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */
+ const uint8_t *chksum_image; /* Points to chksum location */
uint32_t stored_chksum; /* Stored metadata checksum */
void * ret_value = NULL; /* Return value */
@@ -1064,10 +1085,14 @@ H5FS__cache_sinfo_deserialize(const void *_image, size_t len, void *_udata,
if(H5FS_sect_add(udata->f, fspace, new_sect, H5FS_ADD_DESERIALIZING, udata) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, NULL, "can't add section to free space manager")
} /* end for */
+
+ if(fspace->tot_sect_count == old_tot_sect_count)
+ break;
+
} while(image < (((const uint8_t *)_image + old_sect_size) - H5FS_SIZEOF_CHKSUM));
/* Sanity check */
- HDassert((size_t)(image - (const uint8_t *)_image) == (old_sect_size - H5FS_SIZEOF_CHKSUM));
+ HDassert((size_t)(image - (const uint8_t *)_image) <= (old_sect_size - H5FS_SIZEOF_CHKSUM));
HDassert(old_sect_size == fspace->sect_size);
HDassert(old_tot_sect_count == fspace->tot_sect_count);
HDassert(old_serial_sect_count == fspace->serial_sect_count);
@@ -1077,11 +1102,14 @@ H5FS__cache_sinfo_deserialize(const void *_image, size_t len, void *_udata,
/* checksum verification already done in verify_chksum cb */
+ /* There may be empty space between entries and chksum */
+ chksum_image = (const uint8_t *)(_image) + old_sect_size - H5FS_SIZEOF_CHKSUM;
/* Metadata checksum */
- UINT32DECODE(image, stored_chksum);
+ UINT32DECODE(chksum_image, stored_chksum);
/* Sanity check */
- HDassert((size_t)(image - (const uint8_t *)_image) == old_sect_size);
+ HDassert((image == chksum_image) ||
+ ((size_t)((image - (const uint8_t *)_image) + (chksum_image - image)) == old_sect_size));
/* Set return value */
ret_value = sinfo;
@@ -1177,10 +1205,6 @@ H5FS__cache_sinfo_pre_serialize(H5F_t *f, void *_thing, haddr_t addr,
HDassert(new_len);
HDassert(flags);
- /* we shouldn't be called if the section info is empty, unless we hit the point of no return. */
- if(!H5F_POINT_OF_NO_RETURN(f))
- HDassert(fspace->serial_sect_count > 0);
-
sinfo_addr = addr; /* this will change if we relocate the section data */
/* Check for section info at temporary address */
@@ -1246,6 +1270,7 @@ H5FS__cache_sinfo_serialize(const H5F_t *f, void *_image, size_t len,
H5FS_t *fspace; /* Free space header */
H5FS_iter_ud_t udata; /* User data for callbacks */
uint8_t *image = (uint8_t *)_image; /* Pointer into raw data buffer */
+ uint8_t *chksum_image = NULL; /* Points to chksum location */
uint32_t metadata_chksum; /* Computed metadata checksum value */
unsigned bin; /* Current bin we are on */
herr_t ret_value = SUCCEED; /* Return value */
@@ -1290,13 +1315,16 @@ H5FS__cache_sinfo_serialize(const H5F_t *f, void *_image, size_t len,
/* Compute checksum */
- metadata_chksum = H5_checksum_metadata(_image, (size_t)(image - (uint8_t *)_image), 0);
+ /* There may be empty space between entries and chksum */
+ chksum_image = (uint8_t *)(_image) + len - H5FS_SIZEOF_CHKSUM;
+ metadata_chksum = H5_checksum_metadata(_image, (size_t)(chksum_image - (uint8_t *)_image), 0);
/* Metadata checksum */
- UINT32ENCODE(image, metadata_chksum);
+ UINT32ENCODE(chksum_image, metadata_chksum);
/* Sanity check */
- HDassert((size_t)(image - (uint8_t *)_image) == sinfo->fspace->sect_size);
+ HDassert((chksum_image == image) ||
+ ((size_t)((image - (uint8_t *)_image) + (chksum_image - image)) == sinfo->fspace->sect_size));
HDassert(sinfo->fspace->sect_size <= sinfo->fspace->alloc_sect_size);
done:
diff --git a/src/H5FSsection.c b/src/H5FSsection.c
index 27486f1..d15e299 100644
--- a/src/H5FSsection.c
+++ b/src/H5FSsection.c
@@ -307,6 +307,20 @@ HDfprintf(stderr, "%s: fspace->alloc_sect_size = %Hu, fspace->sect_size = %Hu\n"
* Purpose: Release the section info, either giving ownership back to
* the cache or letting the free space header keep it.
*
+ * Add the fix in this routine to resolve the potential infinite loop
+ * problem when allocating file space for the meta data of the
+ * self-referential free-space managers at file closing.
+ *
+ * On file close or flushing, when the section info is modified
+ * and protected/unprotected, does not allow the section info size
+ * to shrink:
+ * --if the current allocated section info size in fspace->sect_size is
+ * larger than the previous section info size in fpsace->alloc_sect_size,
+ * release the section info
+ * --Otherwise, set the fspace->sect_size to be the same as
+ * fpsace->alloc_sect_size. This means fspace->sect_size may be larger
+ * than what is actually needed.
+ *
* Return: SUCCEED/FAIL
*
* Programmer: Quincey Koziol
@@ -357,6 +371,11 @@ HDfprintf(stderr, "%s: fspace->alloc_sect_size = %Hu, fspace->sect_size = %Hu\n"
/* Check if section info lock count dropped to zero */
if(fspace->sinfo_lock_count == 0) {
hbool_t release_sinfo_space = FALSE; /* Flag to indicate section info space in file should be released */
+ hbool_t flush_in_progress = FALSE; /* Is flushing in progress */
+
+ /* Check whether cache is flush_in_progress */
+ if(H5AC_get_cache_flush_in_progress(f->shared->cache, &flush_in_progress) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't get flush_in_progress")
/* Check if we actually protected the section info */
if(fspace->sinfo_protected) {
@@ -370,9 +389,17 @@ HDfprintf(stderr, "%s: fspace->alloc_sect_size = %Hu, fspace->sect_size = %Hu\n"
/* Note that we've modified the section info */
cache_flags |= H5AC__DIRTIED_FLAG;
+ /* On file close or flushing, does not allow section info to shrink in size */
+ if(f->closing || flush_in_progress) {
+ if(fspace->sect_size > fspace->alloc_sect_size)
+ cache_flags |= H5AC__DELETED_FLAG | H5AC__TAKE_OWNERSHIP_FLAG;
+ else
+ fspace->sect_size = fspace->alloc_sect_size;
+
/* Check if the section info size in the file has changed */
- if(fspace->sect_size != fspace->alloc_sect_size)
+ } else if(fspace->sect_size != fspace->alloc_sect_size)
cache_flags |= H5AC__DELETED_FLAG | H5AC__TAKE_OWNERSHIP_FLAG;
+
} /* end if */
/* Sanity check */
@@ -410,11 +437,20 @@ HDfprintf(stderr, "%s: Relinquishing section info ownership\n", FUNC);
/* Check if the section info was modified */
if(fspace->sinfo_modified) {
/* Check if we need to release section info in the file */
- if(H5F_addr_defined(fspace->sect_addr))
+ if(H5F_addr_defined(fspace->sect_addr)) {
/* Set flag to release section info space in file */
- release_sinfo_space = TRUE;
- else
+ /* On file close or flushing, only need to release section info with size
+ bigger than previous section */
+ if(f->closing || flush_in_progress) {
+ if(fspace->sect_size > fspace->alloc_sect_size)
+ release_sinfo_space = TRUE;
+ else
+ fspace->sect_size = fspace->alloc_sect_size;
+ } else
+ release_sinfo_space = TRUE;
+ } else
HDassert(fspace->alloc_sect_size == 0);
+
} /* end if */
else {
/* Sanity checks... */
@@ -2414,84 +2450,31 @@ done:
/*-------------------------------------------------------------------------
* Function: H5FS_vfd_alloc_hdr_and_section_info_if_needed
*
- * Purpose: This function is part of a hack to patch over a design
- * flaw in the free space managers for file space allocation.
- * Specifically, if a free space manager allocates space for
- * its own section info, it is possible for it to
- * go into an infinite loop as it:
- *
- * 1) computes the size of the section info
- *
- * 2) allocates file space for the section info
- *
- * 3) notices that the size of the section info
- * has changed
- *
- * 4) deallocates the section info file space and
- * returns to 1) above.
- *
- * Similarly, if it allocates space for its own header, it
- * can go into an infinite loop as it:
- *
- * 1) allocates space for the header
- *
- * 2) notices that the free space manager is empty
- * and thus should not have file space allocated
- * to its header
- *
- * 3) frees the space allocated to the header
- *
- * 4) notices that the free space manager is not
- * empty and thus must have file space allocated
- * to it, and thus returns to 1) above.
- *
- * In a nutshell, the solution applied in this hack is to
- * deallocate file space for the free space manager(s) that
- * handle FSM header and/or section info file space allocations,
- * wait until all other allocation/deallocation requests have
- * been handled, and then test to see if the free space manager(s)
- * in question are empty. If they are, do nothing. If they
- * are not, allocate space for them at end of file bypassing the
- * usual file space allocation calls, and thus avoiding the
- * potential infinite loops.
- *
- * The purpose of this function is to allocate file space for
+ * Purpose: The purpose of this function is to allocate file space for
* the header and section info of the target free space manager
- * directly from the VFD if needed. In this case the function
- * also re-inserts the header and section info in the metadata
- * cache with this allocation.
- *
- * When paged allocation is not enabled, allocation of space
- * for the free space manager header and section info is
- * straight forward -- we simply allocate the space directly
- * from file driver.
- *
- * Note that if f->shared->alignment > 1, and EOA is not a
- * multiple of the alignment, it is possible that performing
- * these allocations may generate a fragment of file space in
- * addition to the space allocated for the section info. This
- * excess space is dropped on the floor. As shall be seen,
- * it will usually be reclaimed later.
- *
- * When page allocation is enabled, things are more difficult,
- * as there is the possibility that page buffering will be
- * enabled when the free space managers are read. To allow
- * for this, we must ensure that space allocated for the
- * free space manager header and section info is either larger
- * than a page, or resides completely within a page.
- *
- * Do this by allocating space for the free space header and
- * section info starting at page boundaries, and extending
- * allocation to the next page boundary. This of course wastes
- * space, but see below.
- *
- * On the first free space allocation / deallocation after the
- * next file open, we will read the self referential free space
- * managers, float them and reduce the EOA to its value prior
- * to allocation of file space for the self referential free
- * space managers on the preceeding file close. This EOA value
- * is stored in the free space manager superblock extension
- * message.
+ * if they are not allocated yet.
+ *
+ * The previous hack in this routine to avoid the potential infinite
+ * loops by allocating file space directly from the end of the file
+ * is removed. The allocation can now be done via the usual
+ * file space allocation call H5MF_alloc().
+ *
+ * The design flaw is addressed by not allowing the size
+ * of section info to shrink. This means, when trying to allocate
+ * section info size X via H5MF_alloc() and the section info size
+ * after H5MF_alloc() changes to Y:
+ * --if Y is larger than X, free the just allocated file space X
+ * via H5MF_xfree() and set fspace->sect_size to Y.
+ * This routine will be called again later from
+ * H5MF_settle_meta_data_fsm() to allocate section info with the
+ * larger fpsace->sect_size Y.
+ * --if Y is smaller than X, no further allocation is needed and
+ * fspace->sect_size and fspace->alloc_sect_size are set to X.
+ * This means fspace->sect_size may be larger than what is
+ * actually needed.
+ *
+ * This routine also re-inserts the header and section info in the
+ * metadata cache with this allocation.
*
* Return: Success: non-negative
* Failure: negative
@@ -2508,8 +2491,6 @@ H5FS_vfd_alloc_hdr_and_section_info_if_needed(H5F_t *f, H5FS_t *fspace,
hsize_t hdr_alloc_size;
hsize_t sinfo_alloc_size;
haddr_t sect_addr = HADDR_UNDEF; /* address of sinfo */
- haddr_t eoa_frag_addr = HADDR_UNDEF; /* Address of fragment at EOA */
- hsize_t eoa_frag_size = 0; /* Size of fragment at EOA */
haddr_t eoa = HADDR_UNDEF; /* Initial EOA for the file */
herr_t ret_value = SUCCEED; /* Return value */
@@ -2525,12 +2506,6 @@ H5FS_vfd_alloc_hdr_and_section_info_if_needed(H5F_t *f, H5FS_t *fspace,
/* the section info should be unlocked */
HDassert(fspace->sinfo_lock_count == 0);
- /* no space should be allocated */
- HDassert(*fs_addr_ptr == HADDR_UNDEF);
- HDassert(fspace->addr == HADDR_UNDEF);
- HDassert(fspace->sect_addr == HADDR_UNDEF);
- HDassert(fspace->alloc_sect_size == 0);
-
/* persistent free space managers must be enabled */
HDassert(f->shared->fs_persist);
@@ -2542,125 +2517,109 @@ H5FS_vfd_alloc_hdr_and_section_info_if_needed(H5F_t *f, H5FS_t *fspace,
HDassert((f->shared->fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR) ||
(f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE));
- if(fspace->serial_sect_count > 0) {
+ if(fspace->serial_sect_count > 0 && fspace->sinfo) {
/* the section info is floating, so space->sinfo should be defined */
- HDassert(fspace->sinfo);
-
- /* start by allocating file space for the header */
-
- /* Get the EOA for the file -- need for sanity check below */
- if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_FSPACE_HDR)))
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "Unable to get eoa")
- /* check for overlap into temporary allocation space */
- if(H5F_IS_TMP_ADDR(f, (eoa + fspace->sect_size)))
- HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, FAIL, "hdr file space alloc will overlap into 'temporary' file space")
+ if(!H5F_addr_defined(fspace->addr)) {
- hdr_alloc_size = H5FS_HEADER_SIZE(f);
-
- /* if page allocation is enabled, extend the hdr_alloc_size to the
- * next page boundary.
- */
- if(H5F_PAGED_AGGR(f)) {
- HDassert(0 == (eoa % f->shared->fs_page_size));
+ /* start by allocating file space for the header */
- hdr_alloc_size = ((hdr_alloc_size / f->shared->fs_page_size) + 1) * f->shared->fs_page_size;
+ /* Get the EOA for the file -- need for sanity check below */
+ if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_FSPACE_HDR)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "Unable to get eoa")
- HDassert(hdr_alloc_size >= H5FS_HEADER_SIZE(f));
- HDassert((hdr_alloc_size % f->shared->fs_page_size) == 0);
- } /* end if */
+ /* check for overlap into temporary allocation space */
+ if(H5F_IS_TMP_ADDR(f, (eoa + fspace->sect_size)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, FAIL, "hdr file space alloc will overlap into 'temporary' file space")
- /* allocate space for the hdr */
- if(HADDR_UNDEF == (fspace->addr = H5FD_alloc(f->shared->lf, H5FD_MEM_FSPACE_HDR,
- f, hdr_alloc_size, &eoa_frag_addr, &eoa_frag_size)))
- HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space for hdr")
+ hdr_alloc_size = H5FS_HEADER_SIZE(f);
- /* if the file alignment is 1, there should be no
- * eoa fragment. Otherwise, drop any fragment on the floor.
- */
- HDassert((eoa_frag_size == 0) || (f->shared->alignment != 1));
+ if(H5F_PAGED_AGGR(f))
+ HDassert(0 == (eoa % f->shared->fs_page_size));
- /* Cache the new free space header (pinned) */
- if(H5AC_insert_entry(f, H5AC_FSPACE_HDR, fspace->addr, fspace, H5AC__PIN_ENTRY_FLAG) < 0)
- HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space header to cache")
+ /* Allocate space for the free space header */
+ if(HADDR_UNDEF == (fspace->addr = H5MF_alloc(f, H5FD_MEM_FSPACE_HDR, hdr_alloc_size)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "file allocation failed for free space header")
- *fs_addr_ptr = fspace->addr;
+ /* Cache the new free space header (pinned) */
+ if(H5AC_insert_entry(f, H5AC_FSPACE_HDR, fspace->addr, fspace, H5AC__PIN_ENTRY_FLAG) < 0)
+ HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space header to cache")
- /* now allocate file space for the section info */
+ *fs_addr_ptr = fspace->addr;
+ }
- /* Get the EOA for the file -- need for sanity check below */
- if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_FSPACE_SINFO)))
- HGOTO_ERROR(H5E_FSPACE, H5E_CANTGET, FAIL, "Unable to get eoa")
+ if(!H5F_addr_defined(fspace->sect_addr)) {
- /* check for overlap into temporary allocation space */
- if(H5F_IS_TMP_ADDR(f, (eoa + fspace->sect_size)))
- HGOTO_ERROR(H5E_FSPACE, H5E_BADRANGE, FAIL, "sinfo file space alloc will overlap into 'temporary' file space")
+ /* now allocate file space for the section info */
- sinfo_alloc_size = fspace->sect_size;
+ /* Get the EOA for the file -- need for sanity check below */
+ if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_FSPACE_SINFO)))
+ HGOTO_ERROR(H5E_FSPACE, H5E_CANTGET, FAIL, "Unable to get eoa")
- /* if paged allocation is enabled, extend the sinfo_alloc_size to the
- * next page boundary.
- */
- if(H5F_PAGED_AGGR(f)) {
- HDassert(0 == (eoa % f->shared->fs_page_size));
+ /* check for overlap into temporary allocation space */
+ if(H5F_IS_TMP_ADDR(f, (eoa + fspace->sect_size)))
+ HGOTO_ERROR(H5E_FSPACE, H5E_BADRANGE, FAIL, "sinfo file space alloc will overlap into 'temporary' file space")
- sinfo_alloc_size = ((sinfo_alloc_size / f->shared->fs_page_size) + 1) * f->shared->fs_page_size;
+ sinfo_alloc_size = fspace->sect_size;
- HDassert(sinfo_alloc_size >= fspace->sect_size);
- HDassert((sinfo_alloc_size % f->shared->fs_page_size) == 0);
- } /* end if */
+ if(H5F_PAGED_AGGR(f))
+ HDassert(0 == (eoa % f->shared->fs_page_size));
- /* allocate space for the section info */
- if(HADDR_UNDEF == (sect_addr = H5FD_alloc(f->shared->lf, H5FD_MEM_FSPACE_SINFO,
- f, sinfo_alloc_size, &eoa_frag_addr, &eoa_frag_size)))
- HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space")
+ /* allocate space for the section info */
+ if(HADDR_UNDEF == (sect_addr = H5MF_alloc(f, H5FD_MEM_FSPACE_SINFO, sinfo_alloc_size)))
+ HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "file allocation failed for section info")
- /* if the file alignment is 1, there should be no
- * eoa fragment. Otherwise, drop the fragment on the floor.
- */
- HDassert((eoa_frag_size == 0) || (f->shared->alignment != 1));
+ /* update fspace->alloc_sect_size and fspace->sect_addr to reflect
+ * the allocation
+ */
+ if(fspace->sect_size > sinfo_alloc_size) {
+ hsize_t saved_sect_size = fspace->sect_size;
- /* update fspace->alloc_sect_size and fspace->sect_addr to reflect
- * the allocation
- */
- fspace->alloc_sect_size = fspace->sect_size;
- fspace->sect_addr = sect_addr;
-
- /* insert the new section info into the metadata cache. */
-
- /* Question: Do we need to worry about this insertion causing an
- * eviction from the metadata cache? Think on this. If so, add a
- * flag to H5AC_insert() to force it to skip the call to make space in
- * cache.
- *
- * On reflection, no.
- *
- * On a regular file close, any eviction will not change the
- * the contents of the free space manager(s), as all entries
- * should have correct file space allocated by the time this
- * function is called.
- *
- * In the cache image case, the selection of entries for inclusion
- * in the cache image will not take place until after this call.
- * (Recall that this call is made during the metadata fsm settle
- * routine, which is called during the serialization routine in
- * the cache image case. Entries are not selected for inclusion
- * in the image until after the cache is serialized.)
- *
- * JRM -- 11/4/16
- */
- if(H5AC_insert_entry(f, H5AC_FSPACE_SINFO, sect_addr, fspace->sinfo, H5AC__NO_FLAGS_SET) < 0)
- HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space sinfo to cache")
+ if(H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, sect_addr, sinfo_alloc_size) < 0)
+ HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to free free space sections")
+ fspace->sect_size = saved_sect_size;
+ } else {
+ fspace->alloc_sect_size = sinfo_alloc_size;
+ fspace->sect_size = sinfo_alloc_size;
+ fspace->sect_addr = sect_addr;
+
+ /* insert the new section info into the metadata cache. */
+
+ /* Question: Do we need to worry about this insertion causing an
+ * eviction from the metadata cache? Think on this. If so, add a
+ * flag to H5AC_insert() to force it to skip the call to make space in
+ * cache.
+ *
+ * On reflection, no.
+ *
+ * On a regular file close, any eviction will not change the
+ * the contents of the free space manager(s), as all entries
+ * should have correct file space allocated by the time this
+ * function is called.
+ *
+ * In the cache image case, the selection of entries for inclusion
+ * in the cache image will not take place until after this call.
+ * (Recall that this call is made during the metadata fsm settle
+ * routine, which is called during the serialization routine in
+ * the cache image case. Entries are not selected for inclusion
+ * in the image until after the cache is serialized.)
+ *
+ * JRM -- 11/4/16
+ */
+ if(H5AC_insert_entry(f, H5AC_FSPACE_SINFO, sect_addr, fspace->sinfo, H5AC__NO_FLAGS_SET) < 0)
+ HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space sinfo to cache")
- /* We have changed the sinfo address -- Mark free space header dirty */
- if(H5AC_mark_entry_dirty(fspace) < 0)
- HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty")
+ /* We have changed the sinfo address -- Mark free space header dirty */
+ if(H5AC_mark_entry_dirty(fspace) < 0)
+ HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty")
- /* since space has been allocated for the section info and the sinfo
- * has been inserted into the cache, relinquish ownership (i.e. float)
- * the section info.
- */
- fspace->sinfo = NULL;
+ /* since space has been allocated for the section info and the sinfo
+ * has been inserted into the cache, relinquish ownership (i.e. float)
+ * the section info.
+ */
+ fspace->sinfo = NULL;
+ }
+ }
} /* end if */
done:
diff --git a/src/H5Fint.c b/src/H5Fint.c
index 030fda8..4021e41 100644
--- a/src/H5Fint.c
+++ b/src/H5Fint.c
@@ -934,9 +934,9 @@ H5F__new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_
f->shared->fs_addr[u] = HADDR_UNDEF;
f->shared->fs_man[u] = NULL;
}
- f->shared->first_alloc_dealloc = FALSE;
- f->shared->eoa_pre_fsm_fsalloc = HADDR_UNDEF;
- f->shared->eoa_post_fsm_fsalloc = HADDR_UNDEF;
+ /* This will be stored as eoa_pre_fsm_fsalloc in the fsinfo message */
+ /* This is done to be backward compatible with 1.10 library that has the FSM hack */
+ f->shared->eoa_fsm_fsalloc = HADDR_UNDEF;
f->shared->eoa_post_mdci_fsalloc = HADDR_UNDEF;
/* Initialization for handling file space (for paged aggregation) */
diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h
index 6c1156f..5f25ddc 100644
--- a/src/H5Fpkg.h
+++ b/src/H5Fpkg.h
@@ -328,14 +328,9 @@ struct H5F_file_t {
H5F_fs_state_t fs_state[H5F_MEM_PAGE_NTYPES]; /* State of free space manager for each type */
haddr_t fs_addr[H5F_MEM_PAGE_NTYPES]; /* Address of free space manager info for each type */
H5FS_t *fs_man[H5F_MEM_PAGE_NTYPES]; /* Free space manager for each file space type */
- hbool_t first_alloc_dealloc; /* TRUE iff free space managers */
- /* are persistent and have not */
- /* been used accessed for either */
- /* allocation or deallocation */
- /* since file open. */
- haddr_t eoa_pre_fsm_fsalloc; /* eoa pre file space allocation */
- /* for self referential FSMs */
- haddr_t eoa_post_fsm_fsalloc; /* eoa post file space allocation */
+ hbool_t null_fsm_addr; /* Used by h5clear tool to tell the library */
+ /* to drop free-space to the floor */
+ haddr_t eoa_fsm_fsalloc; /* eoa after file space allocation */
/* for self referential FSMs */
haddr_t eoa_post_mdci_fsalloc; /* eoa past file space allocation */
/* for metadata cache image, or */
diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
index e9b6c32..a8ac158 100644
--- a/src/H5Fprivate.h
+++ b/src/H5Fprivate.h
@@ -330,8 +330,7 @@ typedef struct H5F_t H5F_t;
#define H5F_THRESHOLD(F) ((F)->shared->threshold)
#define H5F_PGEND_META_THRES(F) ((F)->shared->fs.pgend_meta_thres)
#define H5F_POINT_OF_NO_RETURN(F) ((F)->shared->fs.point_of_no_return)
-#define H5F_FIRST_ALLOC_DEALLOC(F) ((F)->shared->first_alloc_dealloc)
-#define H5F_EOA_PRE_FSM_FSALLOC(F) ((F)->shared->eoa_pre_fsm_fsalloc)
+#define H5F_NULL_FSM_ADDR(F) ((F)->shared->null_fsm_addr)
#define H5F_GET_MIN_DSET_OHDR(F) ((F)->shared->crt_dset_min_ohdr_flag)
#define H5F_SET_MIN_DSET_OHDR(F, V) ((F)->shared->crt_dset_min_ohdr_flag = (V))
#else /* H5F_MODULE */
@@ -389,8 +388,7 @@ typedef struct H5F_t H5F_t;
#define H5F_THRESHOLD(F) (H5F_get_threshold(F))
#define H5F_PGEND_META_THRES(F) (H5F_get_pgend_meta_thres(F))
#define H5F_POINT_OF_NO_RETURN(F) (H5F_get_point_of_no_return(F))
-#define H5F_FIRST_ALLOC_DEALLOC(F) (H5F_get_first_alloc_dealloc(F))
-#define H5F_EOA_PRE_FSM_FSALLOC(F) (H5F_get_eoa_pre_fsm_fsalloc(F))
+#define H5F_NULL_FSM_ADDR(F) (H5F_get_null_fsm_addr(F))
#define H5F_GET_MIN_DSET_OHDR(F) (H5F_get_min_dset_ohdr(F))
#define H5F_SET_MIN_DSET_OHDR(F, V) (H5F_set_min_dset_ohdr((F), (V)))
#endif /* H5F_MODULE */
@@ -744,8 +742,7 @@ H5_DLL herr_t H5F_get_obj_count(const H5F_t *f, unsigned types, hbool_t app_ref,
H5_DLL herr_t H5F_get_obj_ids(const H5F_t *f, unsigned types, size_t max_objs, hid_t *oid_list, hbool_t app_ref, size_t *obj_id_count_ptr);
H5_DLL hsize_t H5F_get_pgend_meta_thres(const H5F_t *f);
H5_DLL hbool_t H5F_get_point_of_no_return(const H5F_t *f);
-H5_DLL hbool_t H5F_get_first_alloc_dealloc(const H5F_t *f);
-H5_DLL haddr_t H5F_get_eoa_pre_fsm_fsalloc(const H5F_t *f);
+H5_DLL hbool_t H5F_get_null_fsm_addr(const H5F_t *f);
H5_DLL hbool_t H5F_get_min_dset_ohdr(const H5F_t *f);
H5_DLL herr_t H5F_set_min_dset_ohdr(H5F_t *f, hbool_t minimize);
diff --git a/src/H5Fquery.c b/src/H5Fquery.c
index 103ad3b..c168535 100644
--- a/src/H5Fquery.c
+++ b/src/H5Fquery.c
@@ -1190,40 +1190,17 @@ H5F_get_point_of_no_return(const H5F_t *f)
FUNC_LEAVE_NOAPI(f->shared->point_of_no_return)
} /* end H5F_get_point_of_no_return() */
-
/*-------------------------------------------------------------------------
- * Function: H5F_get_first_alloc_dealloc
+ * Function: H5F_get_null_fsm_addr
*
- * Purpose: Retrieve the 'first alloc / dealloc' value for the file.
+ * Purpose: Retrieve the 'null_fsm_addr' value for the file.
*
- * Return: Success: Non-negative, the 'first_alloc_dealloc'
+ * Return: Success: Non-negative, the 'null_fsm_addr'
* Failure: (can't happen)
*-------------------------------------------------------------------------
*/
hbool_t
-H5F_get_first_alloc_dealloc(const H5F_t *f)
-{
- /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */
- FUNC_ENTER_NOAPI_NOINIT_NOERR
-
- HDassert(f);
- HDassert(f->shared);
-
- FUNC_LEAVE_NOAPI(f->shared->first_alloc_dealloc)
-} /* end H5F_get_first_alloc_dealloc() */
-
-
-/*-------------------------------------------------------------------------
- * Function: H5F_get_eoa_pre_fsm_fsalloc
- *
- * Purpose: Retrieve the 'EOA pre-FSM fsalloc' value for the file.
- *
- * Return: Success: Non-negative, the 'EOA pre-FSM fsalloc'
- * Failure: (can't happen)
- *-------------------------------------------------------------------------
- */
-haddr_t
-H5F_get_eoa_pre_fsm_fsalloc(const H5F_t *f)
+H5F_get_null_fsm_addr(const H5F_t *f)
{
/* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */
FUNC_ENTER_NOAPI_NOINIT_NOERR
@@ -1231,6 +1208,5 @@ H5F_get_eoa_pre_fsm_fsalloc(const H5F_t *f)
HDassert(f);
HDassert(f->shared);
- FUNC_LEAVE_NOAPI(f->shared->eoa_pre_fsm_fsalloc)
-} /* end H5F_get_eoa_pre_fsm_fsalloc() */
-
+ FUNC_LEAVE_NOAPI(f->shared->null_fsm_addr)
+} /* end H5F_get_null_fsm_addr() */
diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c
index f4ff645..7685384 100644
--- a/src/H5Fsuper.c
+++ b/src/H5Fsuper.c
@@ -777,13 +777,13 @@ H5F__super_read(H5F_t *f, H5P_genplist_t *fa_plist, hbool_t initial_read)
/* If message is NOT marked "unknown"--set up file space info */
if(!(flags & H5O_MSG_FLAG_WAS_UNKNOWN)) {
H5O_fsinfo_t fsinfo; /* File space info message from superblock extension */
- hbool_t null_fsm_addr = FALSE; /* Whether to drop free-space to the floor */
+ /* f->shared->null_fsm_addr: Whether to drop free-space to the floor */
/* The h5clear tool uses this property to tell the library
* to drop free-space to the floor
*/
if(H5P_exist_plist(fa_plist, H5F_ACS_NULL_FSM_ADDR_NAME) > 0)
- if(H5P_get(fa_plist, H5F_ACS_NULL_FSM_ADDR_NAME, &null_fsm_addr) < 0)
+ if(H5P_get(fa_plist, H5F_ACS_NULL_FSM_ADDR_NAME, &f->shared->null_fsm_addr) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get clearance for persisting fsm addr")
/* Retrieve the 'file space info' structure */
@@ -834,35 +834,42 @@ H5F__super_read(H5F_t *f, H5P_genplist_t *fa_plist, hbool_t initial_read)
/* Initialize page end meta threshold */
f->shared->pgend_meta_thres = fsinfo.pgend_meta_thres;
- if(f->shared->eoa_pre_fsm_fsalloc != fsinfo.eoa_pre_fsm_fsalloc)
- f->shared->eoa_pre_fsm_fsalloc = fsinfo.eoa_pre_fsm_fsalloc;
+ if(f->shared->eoa_fsm_fsalloc != fsinfo.eoa_pre_fsm_fsalloc)
+ f->shared->eoa_fsm_fsalloc = fsinfo.eoa_pre_fsm_fsalloc;
- /* f->shared->eoa_pre_fsm_fsalloc must always be HADDR_UNDEF
- * in the absence of persistent free space managers.
- */
- /* If the following two conditions are true:
+ /*
+ * If the following two conditions are true:
* (1) skipping EOF check (skip_eof_check)
- * (2) dropping free-space to the floor (null_fsm_addr)
- * skip the asserts as "eoa_pre_fsm_fsalloc" may be undefined
+ * (2) dropping free-space to the floor (f->shared->null_fsm_addr)
+ * skip the asserts as "eoa_fsm_fsalloc" may be undefined
* for a crashed file with persistent free space managers.
- * #1 and #2 are enabled when the tool h5clear --increment
+ * The above two conditions are enabled when the tool h5clear --increment
* option is used.
*/
- if(!skip_eof_check && !null_fsm_addr) {
- HDassert((!f->shared->fs_persist) || (f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF));
- HDassert(!f->shared->first_alloc_dealloc);
- }
-
- /* As "eoa_pre_fsm_fsalloc" may be undefined for a crashed file
- * with persistent free space managers, therefore, set
- * "first_alloc_dealloc" when the condition
- * "dropping free-space to the floor is true.
- * This will ensure that no action is done to settle things on file
- * close via H5MF_settle_meta_data_fsm() and H5MF_settle_raw_data_fsm().
+ if(!skip_eof_check && !f->shared->null_fsm_addr)
+ HDassert((!f->shared->fs_persist) || (f->shared->eoa_fsm_fsalloc != HADDR_UNDEF));
+
+ /*
+ * A crashed file with persistent free-space managers may have
+ * undefined f->shared->eoa_fsm_fsalloc.
+ * eoa_fsm_fsalloc is the final eoa which is saved in the free-space
+ * info message's eoa_pre_fsm_fsalloc field for backward compatibility.
+ * If the tool h5clear sets to dropping free-space to the floor
+ * as indicated by f->shared->null_fsm_addr, we are not going to
+ * perform actions to settle things on file close in the routines
+ * H5MF_settle_meta_data_fsm() and H5MF_settle_raw_data_fsm().
+ *
+ * We remove the following check:
+ * if((f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF || null_fsm_addr) &&
+ * (H5F_INTENT(f) & H5F_ACC_RDWR))
+ * f->shared->first_alloc_dealloc = TRUE;
+ *
+ * Because:
+ * --there is no more f->shared->eoa_pre_fsm_fsalloc and
+ * f->shared->first_alloc_dealloc
+ * --the check for null_fsm_addr is directly done in H5MF_settle_meta_data_fsm()
+ * and H5MF_settle_raw_data_fsm()
*/
- if((f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF || null_fsm_addr) &&
- (H5F_INTENT(f) & H5F_ACC_RDWR))
- f->shared->first_alloc_dealloc = TRUE;
f->shared->fs_addr[0] = HADDR_UNDEF;
for(u = 1; u < NELMTS(f->shared->fs_addr); u++)
@@ -870,17 +877,17 @@ H5F__super_read(H5F_t *f, H5P_genplist_t *fa_plist, hbool_t initial_read)
/* If the following two conditions are true:
* (1) file is persisting free-space
- * (2) dropping free-space to the floor (null_fsm_addr)
+ * (2) dropping free-space to the floor (f->shared->null_fsm_addr)
* nullify the addresses of the FSMs
*/
- if(f->shared->fs_persist && null_fsm_addr)
+ if(f->shared->fs_persist && f->shared->null_fsm_addr)
for(u = 0; u < NELMTS(fsinfo.fs_addr); u++)
f->shared->fs_addr[u] = fsinfo.fs_addr[u] = HADDR_UNDEF;
/* For fsinfo.mapped: remove the FSINFO message from the superblock extension
and write a new message to the extension */
- /* For null_fsm_addr: just update FSINFO message in the superblock extension */
- if(((fsinfo.mapped || null_fsm_addr) && (rw_flags & H5AC__READ_ONLY_FLAG) == 0)) {
+ /* For f->shared->null_fsm_addr: just update FSINFO message in the superblock extension */
+ if(((fsinfo.mapped || f->shared->null_fsm_addr) && (rw_flags & H5AC__READ_ONLY_FLAG) == 0)) {
/* Do the same kluge until we know for sure. VC */
#if 1 /* bug fix test code -- tidy this up if all goes well */ /* JRM */
@@ -894,7 +901,7 @@ H5F__super_read(H5F_t *f, H5P_genplist_t *fa_plist, hbool_t initial_read)
f->shared->sblock = sblock;
#endif /* JRM */
- if(null_fsm_addr) {
+ if(f->shared->null_fsm_addr) {
if(H5F__super_ext_write_msg(f, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0)
HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing fsinfo message to superblock extension")
}
diff --git a/src/H5MF.c b/src/H5MF.c
index 8d7ab96..3d9ddb0 100644
--- a/src/H5MF.c
+++ b/src/H5MF.c
@@ -50,6 +50,11 @@
#define H5MF_FSPACE_SHRINK 80 /* Percent of "normal" size to shrink serialized free space size */
#define H5MF_FSPACE_EXPAND 120 /* Percent of "normal" size to expand serialized free space size */
+#define H5MF_CHECK_FSM(FSM, CF) \
+ HDassert(*CF == FALSE); \
+ if(!H5F_addr_defined(FSM->addr) || !H5F_addr_defined(FSM->sect_addr)) \
+ *CF = TRUE;
+
/******************/
/* Local Typedefs */
/******************/
@@ -90,6 +95,8 @@ static herr_t H5MF__close_shrink_eoa(H5F_t *f);
static herr_t H5MF__get_free_sects(H5F_t *f, H5FS_t *fspace, H5MF_sect_iter_ud_t *sect_udata, size_t *nums);
static hbool_t H5MF__fsm_type_is_self_referential(H5F_t *f, H5F_mem_page_t fsm_type);
static hbool_t H5MF__fsm_is_self_referential(H5F_t *f, H5FS_t *fspace);
+static herr_t H5MF__continue_alloc_fsm(H5F_t *f, H5FS_t *sm_hdr_fspace, H5FS_t *sm_sinfo_fspace,
+ H5FS_t *lg_hdr_fspace, H5FS_t *lg_sinfo_fspace, hbool_t *continue_alloc_fsm);
/* Free-space type manager routines */
static herr_t H5MF__create_fstype(H5F_t *f, H5F_mem_page_t type);
@@ -794,12 +801,6 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ
HDassert(f->shared->lf);
HDassert(size > 0);
- if(f->shared->first_alloc_dealloc) {
- HDassert(! H5AC_cache_image_pending(f));
- if(H5MF_tidy_self_referential_fsm_hack(f) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "tidy of self referential fsm hack failed")
- } /* end if */
-
H5MF__alloc_to_fs_type(f, alloc_type, size, &fs_type);
#ifdef H5MF_ALLOC_DEBUG_MORE
@@ -1107,12 +1108,6 @@ HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUN
HGOTO_DONE(SUCCEED)
HDassert(addr != 0); /* Can't deallocate the superblock :-) */
- if(f->shared->first_alloc_dealloc) {
- HDassert(!H5AC_cache_image_pending(f));
- if(H5MF_tidy_self_referential_fsm_hack(f) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed")
- } /* end if */
-
H5MF__alloc_to_fs_type(f, alloc_type, size, &fs_type);
/* Set the ring type in the API context */
@@ -1302,12 +1297,6 @@ HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_r
HDassert(f);
HDassert(H5F_INTENT(f) & H5F_ACC_RDWR);
- if(f->shared->first_alloc_dealloc) {
- HDassert(! H5AC_cache_image_pending(f));
- if(H5MF_tidy_self_referential_fsm_hack(f) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed")
- } /* end if */
-
/* Set mapped type, treating global heap as raw data */
map_type = (alloc_type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : alloc_type;
@@ -1680,16 +1669,6 @@ HDfprintf(stderr, "%s: Entering\n", FUNC);
* H5MF_tidy_self_referential_fsm_hack() to discard the cache image
* block.
*/
- if(f->shared->first_alloc_dealloc) {
- if(H5AC_cache_image_pending(f)) {
- if(H5AC_force_cache_image_load(f) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "forced cache image load failed")
- } /* end if */
- else {
- if(H5MF_tidy_self_referential_fsm_hack(f) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed")
- } /* end else */
- } /* end if */
/* Set the ring type in the API context. In most cases, we will
* need H5AC_RING_RDFSM, so initially set the ring in
@@ -1834,7 +1813,7 @@ HDfprintf(stderr, "%s: Entering\n", FUNC);
fsinfo.threshold = f->shared->fs_threshold;
fsinfo.page_size = f->shared->fs_page_size;
fsinfo.pgend_meta_thres = f->shared->pgend_meta_thres;
- fsinfo.eoa_pre_fsm_fsalloc = f->shared->eoa_pre_fsm_fsalloc;
+ fsinfo.eoa_pre_fsm_fsalloc = f->shared->eoa_fsm_fsalloc;
fsinfo.version = f->shared->fs_version;
/* Write the free space manager message -- message must already exist */
@@ -1892,7 +1871,7 @@ HDfprintf(stderr, "%s: Entering\n", FUNC);
* been no file space allocation or deallocation since file
* open.
*/
- HDassert((f->shared->first_alloc_dealloc) || (final_eoa == f->shared->eoa_post_fsm_fsalloc));
+ HDassert(H5F_NULL_FSM_ADDR(f) || final_eoa == f->shared->eoa_fsm_fsalloc);
} /* end if */
else { /* super_vers can be 0, 1, 2 */
for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
@@ -2000,7 +1979,7 @@ HDfprintf(stderr, "%s: Entering\n", FUNC);
*/
for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype))
fsinfo.fs_addr[ptype-1] = f->shared->fs_addr[ptype];
- fsinfo.eoa_pre_fsm_fsalloc = f->shared->eoa_pre_fsm_fsalloc;
+ fsinfo.eoa_pre_fsm_fsalloc = f->shared->eoa_fsm_fsalloc;
/* Write the free space manager message -- message must already exist */
if(H5F__super_ext_write_msg(f, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0)
@@ -2061,8 +2040,8 @@ HDfprintf(stderr, "%s: Entering\n", FUNC);
* the file driver layer. However, as this possibility seems remote,
* it is ignored in the following assert.
*/
- HDassert((f->shared->first_alloc_dealloc) ||
- (final_eoa == f->shared->eoa_post_fsm_fsalloc) ||
+ HDassert((H5F_NULL_FSM_ADDR(f)) ||
+ (final_eoa == f->shared->eoa_fsm_fsalloc) ||
((H5F_addr_defined(f->shared->eoa_post_mdci_fsalloc)) &&
(final_eoa == f->shared->eoa_post_mdci_fsalloc)));
} /* end if */
@@ -2395,16 +2374,6 @@ H5MF_get_free_sections(H5F_t *f, H5FD_mem_t type, size_t nsects, H5F_sect_info_t
* H5MF_tidy_self_referential_fsm_hack() to discard the cache image
* block.
*/
- if(f->shared->first_alloc_dealloc) {
- if(H5AC_cache_image_pending(f)) {
- if(H5AC_force_cache_image_load(f) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, (-1), "forced cache image load failed")
- } /* end if */
- else {
- if(H5MF_tidy_self_referential_fsm_hack(f) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, (-1), "tidy of self referential fsm hack failed")
- } /* end else */
- } /* end if */
if(type == H5FD_MEM_DEFAULT) {
start_type = H5F_MEM_PAGE_SUPER;
@@ -2676,10 +2645,11 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hbool_t *fsm_settled)
HDassert(f->shared);
HDassert(fsm_settled);
- /* Only need to settle things if we are persisting the free space info
- * and allocation/deallocation has occurred.
+ /*
+ * Only need to settle things if we are persisting free space and
+ * the private property in f->shared->null_fsm_addr is not enabled.
*/
- if(f->shared->fs_persist && !f->shared->first_alloc_dealloc) {
+ if(f->shared->fs_persist && !H5F_NULL_FSM_ADDR(f)) {
hbool_t fsm_opened[H5F_MEM_PAGE_NTYPES]; /* State of FSM */
hbool_t fsm_visited[H5F_MEM_PAGE_NTYPES]; /* State of FSM */
@@ -3017,6 +2987,7 @@ done:
FUNC_LEAVE_NOAPI_TAG(ret_value)
} /* H5MF_settle_raw_data_fsm() */
+
/*-------------------------------------------------------------------------
* Function: H5MF_settle_meta_data_fsm()
@@ -3035,7 +3006,7 @@ done:
* On entry to this function, the raw data settle routine
* (H5MF_settle_raw_data_fsm()) should have:
*
- * 1) Freed the aggregators.
+ * 1) Freed the aggregators.
*
* 2) Freed all file space allocated to the free space managers.
*
@@ -3071,12 +3042,12 @@ done:
* 1) Verify that the free space manager(s) involved in file
* space allocation for free space managers are still floating.
*
- * 2) Free the aggregators.
+ * 2) Free the aggregators.
*
- * 3) Reduce the EOA to the extent possible, and make note
+ * 3) Reduce the EOA to the extent possible, and make note
* of the resulting value. This value will be stored
* in the fsinfo superblock extension message and be used
- * in the subsequent file open.
+ * in the subsequent file open.
*
* 4) Re-allocate space for any free space manager(s) that:
*
@@ -3097,21 +3068,15 @@ done:
* allocation has changed the size of the section info --
* forcing us to deallocate and start the loop over again.
*
- * To avoid this, simply allocate file space for these
- * FSM(s) directly from the VFD layer if allocation is
- * indicated. This avoids the issue by bypassing the FSMs
- * in this case.
- *
- * Note that this may increase the size of the file needlessly.
- * A better solution would be to modify the FSM code to
+ * The solution is to modify the FSM code to
* save empty FSMs to file, and to allow section info blocks
- * to be oversized. However, given that the FSM code is
- * also used by the fractal heaps, and that we are under
- * severe time pressure at the moment, the above brute
- * force solution is attractive.
+ * to be oversized. That is, only allow section info to increase
+ * in size, not shrink. The solution is now implemented.
*
- * 5) Make note of the EOA -- used for sanity checking on
- * FSM shutdown.
+ * 5) Make note of the EOA -- used for sanity checking on
+ * FSM shutdown. This is saved as eoa_pre_fsm_fsalloc in
+ * the free-space info message for backward compatibility
+ * with the 1.10 library that has the hack.
*
* Return: SUCCEED/FAIL
*
@@ -3131,10 +3096,9 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hbool_t *fsm_settled)
H5FS_t *sm_sinfo_fspace = NULL; /* ptr to sm FSM sinfo alloc FSM */
H5FS_t *lg_hdr_fspace = NULL; /* ptr to lg FSM hdr alloc FSM */
H5FS_t *lg_sinfo_fspace = NULL; /* ptr to lg FSM sinfo alloc FSM */
- haddr_t eoa_pre_fsm_fsalloc; /* eoa pre file space allocation */
- /* for self referential FSMs */
- haddr_t eoa_post_fsm_fsalloc; /* eoa post file space allocation */
+ haddr_t eoa_fsm_fsalloc; /* eoa after file space allocation */
/* for self referential FSMs */
+ hbool_t continue_alloc_fsm = FALSE; /* Continue allocating addr and sect_addr for FSMs */
H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
herr_t ret_value = SUCCEED; /* Return value */
@@ -3145,10 +3109,11 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hbool_t *fsm_settled)
HDassert(f->shared);
HDassert(fsm_settled);
- /* Only need to settle things if we are persisting the free space info
- * and allocation/deallocation has occurred.
+ /*
+ * Only need to settle things if we are persisting free space and
+ * the private property in f->shared->null_fsm_addr is not enabled.
*/
- if(f->shared->fs_persist && !f->shared->first_alloc_dealloc) {
+ if(f->shared->fs_persist && !H5F_NULL_FSM_ADDR(f)) {
/* Sanity check */
HDassert(f->shared->lf);
@@ -3196,7 +3161,6 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hbool_t *fsm_settled)
{
H5FS_stat_t fs_stat; /* Information for hdr FSM */
- /* Verify that sm_hdr_fspace is floating if it exists */
if(sm_hdr_fspace) {
/* Query free space manager info for this type */
if(H5FS_stat_info(f, sm_hdr_fspace, &fs_stat) < 0)
@@ -3253,7 +3217,6 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hbool_t *fsm_settled)
* Note that the aggregators will not exist if paged aggregation
* is enabled -- don't attempt to free if this is the case.
*/
- /* Vailin -- is this correct? */
/* (for space not at EOF, it may be put into free space managers) */
if((!H5F_PAGED_AGGR(f)) && (H5MF_free_aggrs(f) < 0))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free aggregators")
@@ -3262,21 +3225,7 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hbool_t *fsm_settled)
if(H5MF__close_shrink_eoa(f) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
- /* At this point, the EOA should be set to a value that contains
- * the allocation for all user data, all non self referential FSMs,
- * the superblock and all superblock extension messages.
- *
- * Make note of the current EOA. We will store this value in the
- * free space manager superblock extension message. Since space for
- * everything other than the self referential FSMs (and possibly the
- * cache image) has been allocated at this point, this allows us to
- * to float the self referential FSMs on the first file space allocation /
- * deallocation and then set the EOA to this value before we handle
- * the allocation / deallocation. (If a cache image exists, the
- * first allocation / deallocation will be the deallocation of space
- * for the cache image).
- *
- * WARNING: This approach settling the self referential free space
+ /* WARNING: This approach settling the self referential free space
* managers and allocating space for them in the file will
* not work as currently implemented with the split and
* multi file drivers, as the self referential free space
@@ -3295,97 +3244,67 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hbool_t *fsm_settled)
* We should be able to support the split file driver
* without a file format change. However, the code to
* do so does not exist at present.
+ * NOTE: not sure whether to remove or keep the above comments
*/
- if(HADDR_UNDEF == (eoa_pre_fsm_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)))
- HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get EOA")
-
- /* ******************* PROBLEM: ********************
- *
- * If the file has an alignment other than 1, and if
- * the EOA is not a multiple of this alignment, allocating space
- * for the section via the VFD info has the potential of generating
- * a fragment that will be added to the free space manager. This
- * of course undoes everything we have been doing here.
- *
- * Need a way around this. Obvious solution is to force the EOA to
- * be a multiple of the alignment.
- *
- * Fortunately, alignment is typically 1, so this is a non-issue in
- * most cases. In cases where the alignment is not 1, for now we
- * have decided to drop the fragment on the floor.
- *
- * Eventually, we should fix this by modifying the on disk representations
- * of free space managers to allow for empty space, so as to bypass the
- * issues created by self-referential free space managers, and make
- * this issue moot.
+ /*
+ * Continue allocating file space for the header and section info until
+ * they are all settled,
*/
- /* HDassert(f->shared->alignment == 1); */
-
+ do {
+ continue_alloc_fsm = FALSE;
+ if(sm_hdr_fspace)
+ if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, sm_hdr_fspace, &(f->shared->fs_addr[sm_fshdr_fs_type])) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sm hdr FSM file space")
+
+ if(sm_sinfo_fspace && (sm_sinfo_fspace != sm_hdr_fspace))
+ if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, sm_sinfo_fspace, &(f->shared->fs_addr[sm_fssinfo_fs_type])) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sm sinfo FSM file space")
+
+ if(H5F_PAGED_AGGR(f)) {
+ if(lg_hdr_fspace)
+ if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, lg_hdr_fspace, &(f->shared->fs_addr[lg_fshdr_fs_type])) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg hdr FSM file space")
+
+ if(lg_sinfo_fspace && (lg_sinfo_fspace != lg_hdr_fspace))
+ if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, lg_sinfo_fspace, &(f->shared->fs_addr[lg_fssinfo_fs_type])) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg sinfo FSM file space")
+ } /* end if */
- /* The free space manager(s) that handle space allocations for free
- * space managers should be settled now, albeit without file space
- * allocated to them. To avoid the possibility of changing the sizes
- * of their section info blocks, allocate space for them now at the
- * end of file via H5FD_alloc().
- *
- * In the past, this issue of allocating space without touching the
- * free space managers has been dealt with by calling
- * H5MF_aggr_vfd_alloc(), which in turn calls H5MF_aggr_alloc().
- * This is problematic since (if I read the code correctly) it will
- * re-constitute the metadata aggregator, which will add any leftover
- * space to one of the free space managers when freed.
- *
- * This is a non-starter, since the entire objective is to settle the
- * free space managers.
- *
- * Hence the decision to call H5FD_alloc() directly.
- *
- * As discussed in PROBLEM above, if f->shared->alignment is not 1,
- * this has the possibility of generating a fragment of file space
- * that would typically be inserted into one of the free space managers.
- *
- * This is isn't good, but due to schedule pressure, we will just drop
- * the fragment on the floor for now.
- */
- if(sm_hdr_fspace)
- if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, sm_hdr_fspace, &(f->shared->fs_addr[sm_fshdr_fs_type])) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sm hdr FSM file space")
+ sm_hdr_fspace = f->shared->fs_man[sm_fshdr_fs_type];
+ sm_sinfo_fspace = f->shared->fs_man[sm_fssinfo_fs_type];
+ if(H5F_PAGED_AGGR(f)) {
+ lg_hdr_fspace = f->shared->fs_man[lg_fshdr_fs_type];
+ lg_sinfo_fspace = f->shared->fs_man[lg_fssinfo_fs_type];
+ }
- if(sm_sinfo_fspace && (sm_sinfo_fspace != sm_hdr_fspace))
- if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, sm_sinfo_fspace, &(f->shared->fs_addr[sm_fssinfo_fs_type])) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sm sinfo FSM file space")
+ if(H5MF__continue_alloc_fsm(f, sm_hdr_fspace, sm_sinfo_fspace, lg_hdr_fspace, lg_sinfo_fspace, &continue_alloc_fsm) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg sinfo FSM file space")
- if(H5F_PAGED_AGGR(f)) {
- if(lg_hdr_fspace)
- if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, lg_hdr_fspace, &(f->shared->fs_addr[lg_fshdr_fs_type])) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg hdr FSM file space")
+ } while(continue_alloc_fsm);
- if(lg_sinfo_fspace && (lg_sinfo_fspace != lg_hdr_fspace))
- if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, lg_sinfo_fspace, &(f->shared->fs_addr[lg_fssinfo_fs_type])) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg sinfo FSM file space")
- } /* end if */
+ /* All free space managers should have file space allocated for them
+ * now, and should see no further allocations / deallocations.
+ * For backward compatibility, store the eoa in f->shared->eoa_fsm_fsalloc
+ * which will be set to fsinfo.eoa_pre_fsm_fsalloc when we actually write
+ * the free-space info message to the superblock extension.
+ * This will allow the 1.10 library with the hack to open the file with
+ * the new solution.
+ */
/* Get the eoa after allocation of file space for the self referential
* free space managers. Assuming no cache image, this should be the
* final EOA of the file.
*/
- if(HADDR_UNDEF == (eoa_post_fsm_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)))
+ if(HADDR_UNDEF == (eoa_fsm_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)))
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file size")
-
- /* All free space managers should have file space allocated for them
- * now, and should see no further allocations / deallocations. Store
- * the pre and post file space allocation for self referential FSMs EOA
- * for use when we actually write the free space manager superblock
- * extension message.
- */
- f->shared->eoa_pre_fsm_fsalloc = eoa_pre_fsm_fsalloc;
- f->shared->eoa_post_fsm_fsalloc = eoa_post_fsm_fsalloc;
+ f->shared->eoa_fsm_fsalloc = eoa_fsm_fsalloc;
/* Indicate that the FSM was settled successfully */
*fsm_settled = TRUE;
} /* end if */
+
done:
/* Reset the ring in the API context */
if(orig_ring != H5AC_RING_INV)
@@ -3396,6 +3315,61 @@ done:
/*-------------------------------------------------------------------------
+ * Function: H5MF__continue_alloc_fsm
+ *
+ * Purpose: To determine whether any of the input FSMs has allocated
+ * its "addr" and "sect_addr".
+ * Return TRUE or FALSE in *continue_alloc_fsm.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ * Programmer: Vailin Choi
+ * 6/24/2019
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5MF__continue_alloc_fsm(H5F_t *f, H5FS_t *sm_hdr_fspace, H5FS_t *sm_sinfo_fspace,
+ H5FS_t *lg_hdr_fspace, H5FS_t *lg_sinfo_fspace, hbool_t *continue_alloc_fsm)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ /* Sanity checks */
+ HDassert(f);
+ HDassert(f->shared);
+ HDassert(continue_alloc_fsm);
+
+ /* Check sm_hdr_fspace */
+ if(sm_hdr_fspace && sm_hdr_fspace->serial_sect_count > 0 && sm_hdr_fspace->sinfo) {
+ H5MF_CHECK_FSM(sm_hdr_fspace, continue_alloc_fsm);
+ }
+
+ if(!(*continue_alloc_fsm)) {
+ if(sm_sinfo_fspace && sm_sinfo_fspace != sm_hdr_fspace &&
+ sm_sinfo_fspace->serial_sect_count > 0 && sm_sinfo_fspace->sinfo) {
+ H5MF_CHECK_FSM(sm_hdr_fspace, continue_alloc_fsm);
+ }
+ }
+
+ if(H5F_PAGED_AGGR(f) && !(*continue_alloc_fsm)) {
+ /* Check lg_hdr_fspace */
+ if(lg_hdr_fspace && lg_hdr_fspace->serial_sect_count > 0 && lg_hdr_fspace->sinfo) {
+ H5MF_CHECK_FSM(lg_hdr_fspace, continue_alloc_fsm);
+ }
+
+ /* Check lg_sinfo_fspace */
+ if(!(*continue_alloc_fsm)) {
+ if(lg_sinfo_fspace && lg_sinfo_fspace != lg_hdr_fspace &&
+ lg_sinfo_fspace->serial_sect_count > 0 && lg_sinfo_fspace->sinfo) {
+ H5MF_CHECK_FSM(lg_sinfo_fspace, continue_alloc_fsm);
+ }
+ }
+ }
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5MF__continue_alloc_fsm() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5MF__fsm_type_is_self_referential()
*
* Purpose: Return TRUE if the indicated free space manager allocates
@@ -3500,303 +3474,3 @@ H5MF__fsm_is_self_referential(H5F_t *f, H5FS_t *fspace)
FUNC_LEAVE_NOAPI(result)
} /* H5MF__fsm_is_self_referential() */
-
-
-/*-------------------------------------------------------------------------
- * Function: H5MF_tidy_self_referential_fsm_hack
- *
- * Purpose: As discussed in the comments of the settle routines above,
- * the existence of self referential free space managers
- * as currently implemented creates the possibility of
- * infinite loops at file close.
- *
- * As a hack to avoid this, we have added code to settle
- * self referential free space managers, and then allocate
- * space for them directly from the file driver.
- *
- * To avoid dropping ever increasing amounts of file space
- * on the floor with each subsequent file close/open cycle,
- * we need to clean this up on file open. To avoid this,
- * this function is called on the first file space allocation
- * or deallocation after file open to float the self referential
- * free space managers and reduce the EOA to the value it
- * had before the direct allocation of space for the self
- * referential free space managers.
- *
- * The function proceeds as follows:
- *
- * 1) Verify that f->shared->first_alloc_dealloc is TRUE,
- * and then set it to FALSE.
- *
- * 2) Get the current EOA. Verify that it is greater than
- * or equal to f->shared->eoa_pre_fsm_fsalloc. If the
- * current eoa is equal to f->shared->eoa_pre_fsm_fsalloc,
- * no self referential FSMs were stored, and we are done.
- *
- * NOTE: This will have to be reworked somewhat for
- * cache image.
- *
- * 3) Load the self referential FSMs. In passing verify that
- * the lowest address of a FSM header is equal to
- * f->shared->eoa_pre_fsm_fsalloc.
- *
- * Note that we don't have to use any special I/O for
- * this -- we can use the regular I/O methods even if
- * paged aggregation and page buffering is enabled.
- *
- * 4) Float the FSMs. Ensure that the file space is NOT
- * released.
- *
- * 5) Set EOA equal to f->shared->eoa_pre_fsm_fsalloc,
- * and then set f->shared->eoa_pre_fsm_fsalloc to
- * HADDR_UNDEF.
- *
- * If page buffering, verify that the new EOA is
- * on a page boundary, and expunge any pages in the
- * page buffer after the new EOA.
- *
- * Note that this function is also called from test code
- * when it is necessary to startup a self referential
- * free space manager prior to the first file space
- * allocation / deallocation. Failure to do so will
- * result in assertion failures in this function on
- * the first file space allocation / deallocation.
- *
- * Return: SUCCEED/FAIL
- *
- * Programmer: John Mainzer
- * 12/11/16
- *
- *-------------------------------------------------------------------------
- */
-herr_t
-H5MF_tidy_self_referential_fsm_hack(H5F_t *f)
-{
- haddr_t eoa; /* EOA of file */
- hsize_t tail_size = 0; /* Size of chunk to free */
- H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
- haddr_t first_srfsm_hdr = HADDR_UNDEF; /* Addr of first self referential */
- /* fsm header in file */
- H5FS_stat_t fs_stat; /* Information for hdr FSM */
- H5F_mem_page_t sm_fshdr_fs_type; /* Small fs hdr fsm */
- H5F_mem_page_t sm_fssinfo_fs_type; /* Small fs sinfo fsm */
- H5F_mem_page_t lg_fshdr_fs_type; /* Large fs hdr fsm */
- H5F_mem_page_t lg_fssinfo_fs_type; /* Large fs sinfo fsm */
- herr_t ret_value = SUCCEED; /* Return value */
-
- FUNC_ENTER_NOAPI_TAG(H5AC__FREESPACE_TAG, FAIL)
-
- /* check args */
- HDassert(f);
- HDassert(f->shared);
- HDassert(f->shared->fs_persist);
- HDassert(f->shared->first_alloc_dealloc);
-
- /* Set the ring type in the API context. Since we are only dealing with
- * self referential FSMs, we will only need H5AC_RING_MDFSM.
- */
- H5AC_set_ring(H5AC_RING_MDFSM, &orig_ring);
-
- /* 1) Verify that f->shared->first_alloc_dealloc is TRUE,
- * and then set it to FALSE.
- */
- HDassert(f->shared->first_alloc_dealloc);
- f->shared->first_alloc_dealloc = FALSE;
-
-
- /* 2) Get the current EOA. Verify that it is greater than
- * or equal to f->shared->eoa_pre_fsm_fsalloc. If the
- * current eoa is equal to f->shared->eoa_pre_fsm_fsalloc,
- * no self referential FSMs were stored, and we are done.
- *
- * NOTE: This will have to be reworked somewhat for
- * cache image.
- */
- if(HADDR_UNDEF == (eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)))
- HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get EOA")
- HDassert(H5F_addr_le(f->shared->eoa_pre_fsm_fsalloc, eoa));
-
- if(H5F_addr_eq(f->shared->eoa_pre_fsm_fsalloc, eoa))
- HGOTO_DONE(SUCCEED)
-
-
- /* 3) Load the self referential FSMs. In passing verify that
- * the lowest address of a FSM header is equal to
- * f->shared->eoa_pre_fsm_fsalloc.'
- *
- * Note that we don't have to use any special I/O for
- * this -- we can use the regular I/O methods even if
- * paged aggregation and page buffering is enabled.
- */
- H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fs_type);
- H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fs_type);
- HDassert(sm_fshdr_fs_type > H5F_MEM_PAGE_DEFAULT);
- HDassert(sm_fshdr_fs_type < H5F_MEM_PAGE_LARGE_SUPER);
-
- HDassert(sm_fssinfo_fs_type > H5F_MEM_PAGE_DEFAULT);
- HDassert(sm_fssinfo_fs_type < H5F_MEM_PAGE_LARGE_SUPER);
-
- HDassert(NULL == f->shared->fs_man[sm_fshdr_fs_type]);
- HDassert(NULL == f->shared->fs_man[sm_fssinfo_fs_type]);
-
- if(H5F_addr_defined(f->shared->fs_addr[sm_fshdr_fs_type])) {
- first_srfsm_hdr = f->shared->fs_addr[sm_fshdr_fs_type];
-
- /* open the FSM */
- if(H5MF__open_fstype(f, sm_fshdr_fs_type) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager")
-
- HDassert(f->shared->fs_man[sm_fshdr_fs_type]);
- } /* end if */
-
- if((sm_fshdr_fs_type != sm_fssinfo_fs_type) &&
- (H5F_addr_defined(f->shared->fs_addr[sm_fssinfo_fs_type]))) {
-
- if(!H5F_addr_defined(first_srfsm_hdr) ||
- (H5F_addr_defined(first_srfsm_hdr) &&
- H5F_addr_lt(f->shared->fs_addr[sm_fssinfo_fs_type], first_srfsm_hdr)))
- first_srfsm_hdr = f->shared->fs_addr[sm_fssinfo_fs_type];
-
- HDassert(NULL == f->shared->fs_man[sm_fssinfo_fs_type]);
-
- /* open the FSM */
- if(H5MF__open_fstype(f, sm_fssinfo_fs_type) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager")
-
- HDassert(f->shared->fs_man[sm_fssinfo_fs_type]);
- } /* end if */
-
- if(H5F_PAGED_AGGR(f)) {
- H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fs_type);
- H5MF__alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fs_type);
-
- HDassert(lg_fshdr_fs_type >= H5F_MEM_PAGE_LARGE_SUPER);
- HDassert(lg_fshdr_fs_type < H5F_MEM_PAGE_NTYPES);
-
- HDassert(lg_fssinfo_fs_type >= H5F_MEM_PAGE_LARGE_SUPER);
- HDassert(lg_fssinfo_fs_type < H5F_MEM_PAGE_NTYPES);
-
- HDassert(NULL == f->shared->fs_man[lg_fshdr_fs_type]);
- HDassert(NULL == f->shared->fs_man[lg_fssinfo_fs_type]);
-
- if(H5F_addr_defined(f->shared->fs_addr[lg_fshdr_fs_type])) {
- if(!H5F_addr_defined(first_srfsm_hdr) ||
- (H5F_addr_defined(first_srfsm_hdr) &&
- H5F_addr_lt(f->shared->fs_addr[lg_fshdr_fs_type], first_srfsm_hdr)))
- first_srfsm_hdr = f->shared->fs_addr[lg_fshdr_fs_type];
-
- HDassert(NULL == f->shared->fs_man[lg_fshdr_fs_type]);
-
- /* open the FSM */
- if(H5MF__open_fstype(f, lg_fshdr_fs_type) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager")
- HDassert(f->shared->fs_man[lg_fshdr_fs_type]);
- } /* end if */
-
- if(lg_fshdr_fs_type != lg_fssinfo_fs_type && H5F_addr_defined(f->shared->fs_addr[lg_fssinfo_fs_type])) {
- if(!H5F_addr_defined(first_srfsm_hdr) ||
- (H5F_addr_defined(first_srfsm_hdr) &&
- H5F_addr_lt(f->shared->fs_addr[lg_fssinfo_fs_type], first_srfsm_hdr)))
- first_srfsm_hdr = f->shared->fs_addr[lg_fssinfo_fs_type];
-
- HDassert(NULL == f->shared->fs_man[lg_fssinfo_fs_type]);
-
- /* open the FSM */
- if(H5MF__open_fstype(f, lg_fssinfo_fs_type) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager")
- HDassert(f->shared->fs_man[lg_fssinfo_fs_type]);
- } /* end if */
- } /* end if */
- HDassert(H5F_addr_eq(first_srfsm_hdr, f->shared->eoa_pre_fsm_fsalloc));
-
- /* 4) Float the FSMs. Ensure that the file space is NOT released. */
- if(f->shared->fs_man[sm_fshdr_fs_type]) {
- /* Sanity check: Query free space manager info for this type */
- if(H5FS_stat_info(f, f->shared->fs_man[sm_fshdr_fs_type], &fs_stat) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
-
- HDassert(H5F_addr_defined(fs_stat.addr));
- HDassert(H5F_addr_defined(fs_stat.sect_addr));
- if(H5FS_free(f, f->shared->fs_man[sm_fshdr_fs_type], FALSE) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers")
- f->shared->fs_addr[sm_fshdr_fs_type] = HADDR_UNDEF;
- } /* end if */
-
- if(sm_fshdr_fs_type != sm_fssinfo_fs_type && f->shared->fs_man[sm_fssinfo_fs_type]) {
- /* Sanity check: Query free space manager info for this type */
- if(H5FS_stat_info(f, f->shared->fs_man[sm_fssinfo_fs_type], &fs_stat) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
-
- HDassert(H5F_addr_defined(fs_stat.addr));
- HDassert(H5F_addr_defined(fs_stat.sect_addr));
- if(H5FS_free(f, f->shared->fs_man[sm_fssinfo_fs_type], FALSE) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers")
- f->shared->fs_addr[sm_fssinfo_fs_type] = HADDR_UNDEF;
- } /* end if */
-
- if(H5F_PAGED_AGGR(f)) {
- if(f->shared->fs_man[lg_fshdr_fs_type]) {
- /* Sanity check: Query free space manager info for this type */
- if(H5FS_stat_info(f, f->shared->fs_man[lg_fshdr_fs_type], &fs_stat) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info")
-
- HDassert(H5F_addr_defined(fs_stat.addr));
- HDassert(H5F_addr_defined(fs_stat.sect_addr));
- if(H5FS_free(f, f->shared->fs_man[lg_fshdr_fs_type], FALSE) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't float free-space headers")
- f->shared->fs_addr[lg_fshdr_fs_type] = HADDR_UNDEF;
- } /* end if */
-
- if(lg_fshdr_fs_type != lg_fssinfo_fs_type && f->shared->fs_man[lg_fssinfo_fs_type]) {
- /* Sanity check: Query free space manager info for this type */
- if(H5FS_stat_info(f, f->shared->fs_man[lg_fssinfo_fs_type], &fs_stat) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info")
-
- HDassert(H5F_addr_defined(fs_stat.addr));
- HDassert(H5F_addr_defined(fs_stat.sect_addr));
- if(H5FS_free(f, f->shared->fs_man[lg_fssinfo_fs_type], FALSE) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't float free-space headers")
- f->shared->fs_addr[lg_fssinfo_fs_type] = HADDR_UNDEF;
- } /* end if */
- } /* end if */
-
- /* 5) Set EOA equal to f->shared->eoa_pre_fsm_fsalloc,
- * and then set f->shared->eoa_pre_fsm_fsalloc to
- * HADDR_UNDEF.
- *
- * If page buffering, verify that the new EOA is
- * on a page boundary, and expunge any pages in the
- * page buffer after the new EOA.
- */
- if(!H5F_PAGED_AGGR(f)) {
- /* Verify that the aggregators are still shutdown. */
- HDassert(f->shared->sdata_aggr.tot_size == 0);
- HDassert(f->shared->sdata_aggr.addr == 0);
- HDassert(f->shared->sdata_aggr.size == 0);
-
- HDassert(f->shared->meta_aggr.tot_size == 0);
- HDassert(f->shared->meta_aggr.addr == 0);
- HDassert(f->shared->meta_aggr.size == 0);
- } /* end if */
-
- tail_size = (hsize_t)(eoa - f->shared->eoa_pre_fsm_fsalloc);
-
- /* Release file space allocated to self referential FSMs */
- if(H5F__free(f, H5FD_MEM_DEFAULT, f->shared->eoa_pre_fsm_fsalloc, tail_size) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "driver free request failed")
- if(HADDR_UNDEF == (eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)))
- HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get EOA")
- HDassert(H5F_addr_eq(f->shared->eoa_pre_fsm_fsalloc, eoa));
-
- f->shared->eoa_pre_fsm_fsalloc = HADDR_UNDEF;
-
- HDassert((!H5F_PAGED_AGGR(f)) || (0 == (eoa % f->shared->fs_page_size)));
-
-done:
- /* Reset the ring in the API context */
- if(orig_ring != H5AC_RING_INV)
- H5AC_set_ring(orig_ring, NULL);
-
- FUNC_LEAVE_NOAPI_TAG(ret_value)
-} /* H5MF_tidy_self_referential_fsm_hack() */
-
diff --git a/src/H5MFaggr.c b/src/H5MFaggr.c
index 3db7f73..c726341 100644
--- a/src/H5MFaggr.c
+++ b/src/H5MFaggr.c
@@ -183,7 +183,21 @@ HDfprintf(stderr, "%s: type = %u, size = %Hu\n", FUNC, (unsigned)type, size);
* allocate "generic" space and sub-allocate out of that, if possible.
* Otherwise just allocate through H5F__alloc().
*/
- if((f->shared->feature_flags & aggr->feature_flag) && f->shared->fs_strategy != H5F_FSPACE_STRATEGY_NONE) {
+
+ /*
+ * Replace the following line with the line in #ifdef REPLACE/#endif.
+ * The line in #ifdef REPLACE triggers the following problem:
+ * test/objcopy.c: test_copy_group_deep() test fails with the family driver
+ *
+ * When closing the destination file after H5Ocopy, the library flushes the fractal
+ * heap direct block via H5HF__cache_dblock_pre_serialize(). While doing so,
+ * the cache eventually adjusts/evicts ageout entries and ends up flushing out the
+ * same entry that is being serialized (flush_in_progress).
+ */
+ if((f->shared->feature_flags & aggr->feature_flag) && f->shared->fs_strategy != H5F_FSPACE_STRATEGY_NONE && (!f->closing || !f->shared->fs_persist)) {
+#ifdef REPLACE
+ if((f->shared->feature_flags & aggr->feature_flag) && f->shared->fs_strategy != H5F_FSPACE_STRATEGY_NONE && !f->closing) {
+#endif
haddr_t aggr_frag_addr = HADDR_UNDEF; /* Address of aggregrator fragment */
hsize_t aggr_frag_size = 0; /* Size of aggregator fragment */
hsize_t alignment; /* Alignment of this section */
diff --git a/src/H5MFdbg.c b/src/H5MFdbg.c
index eedb72e..e11476b 100644
--- a/src/H5MFdbg.c
+++ b/src/H5MFdbg.c
@@ -215,14 +215,14 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
-H5MF_sects_dump(H5F_t *f, FILE *stream)
+H5MF__sects_dump(H5F_t *f, FILE *stream)
{
haddr_t eoa; /* End of allocated space in the file */
int indent = 0; /* Amount to indent */
int fwidth = 50; /* Field width */
herr_t ret_value = SUCCEED; /* Return value */
- FUNC_ENTER_PACKAGE_TAG(H5AC__FREESPACE_TAG, FAIL)
+ FUNC_ENTER_PACKAGE_TAG(H5AC__FREESPACE_TAG)
#ifdef H5MF_ALLOC_DEBUG
HDfprintf(stderr, "%s: Dumping file free space sections\n", FUNC);
#endif /* H5MF_ALLOC_DEBUG */
@@ -261,7 +261,7 @@ HDfprintf(stderr, "%s: for type = H5FD_MEM_DEFAULT, eoa = %a\n", FUNC, eoa);
udata.fwidth = MAX(0, fwidth - 6);
/* Iterate over all the free space sections */
- if(H5FS_sect_iterate(f, f->shared->fs_man[ptype], H5MF_sects_debug_cb, &udata) < 0)
+ if(H5FS_sect_iterate(f, f->shared->fs_man[ptype], H5MF__sects_debug_cb, &udata) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_BADITER, FAIL, "can't iterate over heap's free space")
} /* end if */
else
@@ -314,7 +314,7 @@ HDfprintf(stderr, "%s: sda_addr = %a, sda_size = %Hu, end of sda = %a\n", FUNC,
udata.fwidth = MAX(0, fwidth - 6);
/* Iterate over all the free space sections */
- if(H5FS_sect_iterate(f, f->shared->fs_man[atype], H5MF_sects_debug_cb, &udata) < 0)
+ if(H5FS_sect_iterate(f, f->shared->fs_man[atype], H5MF__sects_debug_cb, &udata) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_BADITER, FAIL, "can't iterate over heap's free space")
} /* end if */
else /* No sections of this type */
diff --git a/src/H5Ocache_image.c b/src/H5Ocache_image.c
index 591ac4a..7336211 100644
--- a/src/H5Ocache_image.c
+++ b/src/H5Ocache_image.c
@@ -27,11 +27,12 @@
*/
#include "H5Omodule.h" /* This source code file is part of the H5O module */
+#define H5F_FRIEND /*suppress error about including H5Fpkg */
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
-#include "H5Fprivate.h" /* Files */
+#include "H5Fpkg.h" /* Files */
#include "H5FLprivate.h" /* Free Lists */
#include "H5Opkg.h" /* Object headers */
#include "H5MFprivate.h" /* File space management */
@@ -275,16 +276,16 @@ H5O__mdci_free(void *mesg)
*-------------------------------------------------------------------------
*/
static herr_t
-H5O__mdci_delete(H5F_t *f, H5O_t *open_oh, void *_mesg)
+H5O__mdci_delete(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, void *_mesg)
{
H5O_mdci_t *mesg = (H5O_mdci_t *)_mesg;
- herr_t ret_value = SUCCEED; /* Return value */
+ haddr_t final_eoa; /* For sanity check */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
/* check args */
HDassert(f);
- HDassert(open_oh);
HDassert(mesg);
/* Free file space for cache image */
@@ -321,16 +322,24 @@ H5O__mdci_delete(H5F_t *f, H5O_t *open_oh, void *_mesg)
* before the first metadata cache access. However, given
* time constraints, I don't want to go there now.
*/
- if(H5F_FIRST_ALLOC_DEALLOC(f)) {
- HDassert(HADDR_UNDEF != H5F_EOA_PRE_FSM_FSALLOC(f));
- HDassert(H5F_addr_ge(mesg->addr, H5F_EOA_PRE_FSM_FSALLOC(f)));
- if(H5MF_tidy_self_referential_fsm_hack(f) < 0)
- HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed")
- } /* end if */
- else {
+
+
+
+ if(f->closing) {
+
+ /* Get the eoa, and verify that it has the expected value */
+ if(HADDR_UNDEF == (final_eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)) )
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "unable to get file size")
+
+ HDassert(H5F_addr_eq(final_eoa, mesg->addr + mesg->size));
+
+ if(H5FD_free(f->shared->lf, H5FD_MEM_SUPER, f, mesg->addr, mesg->size) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "can't free MDC image")
+ } else {
if(H5MF_xfree(f, H5FD_MEM_SUPER, mesg->addr, mesg->size) < 0)
HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to free file space for cache image block")
- } /* end else */
+ }
+
} /* end if */
done:
diff --git a/test/cache_image.c b/test/cache_image.c
index ee49502..f8252ac 100644
--- a/test/cache_image.c
+++ b/test/cache_image.c
@@ -93,6 +93,7 @@ static unsigned evict_on_close_test(hbool_t single_file_vfd);
#define DSET_SIZE (40 * CHUNK_SIZE)
#define MAX_NUM_DSETS 256
+
static void
create_datasets(hid_t file_id, int min_dset, int max_dset)
{
@@ -3717,8 +3718,7 @@ cache_image_smoke_check_2(hbool_t single_file_vfd)
if ( show_progress )
HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass);
-
-
+
/* 4) Open the file.
*
* Verify that the metadata cache is instructed to load the
@@ -4486,6 +4486,7 @@ cache_image_smoke_check_4(hbool_t single_file_vfd)
if ( show_progress )
HDfprintf(stdout, "%s:L3 cp = %d, max_dset = %d, pass = %d.\n",
fcn_name, cp + 2, max_dset, pass);
+
} /* end while */
cp += 3;
@@ -4638,7 +4639,6 @@ cache_image_smoke_check_4(hbool_t single_file_vfd)
FUNC, failure_mssg);
return !pass;
-
} /* cache_image_smoke_check_4() */
@@ -5111,8 +5111,7 @@ cache_image_smoke_check_5(hbool_t single_file_vfd)
* extension message has been deleted.
*/
i = min_group;
- while ( ( pass ) && ( i <= max_group ) ) {
-
+ while ( ( pass ) && ( i <= max_group ) ) {
sprintf(process_group_name, "/process_%d", i);
validate_zoo(file_id, process_group_name, i++);
}
@@ -7513,10 +7512,9 @@ get_free_sections_test(hbool_t single_file_vfd)
/* file_ptr->shared->first_alloc_dealloc is set to FALSE if the
* file is opened R/O.
*/
- if ( ( file_ptr->shared->first_alloc_dealloc ) ||
- ( ! H5F_addr_defined(file_ptr->shared->eoa_pre_fsm_fsalloc) ) ||
+ if ( ( ! H5F_addr_defined(file_ptr->shared->eoa_fsm_fsalloc) ) ||
( ! H5F_addr_defined(file_ptr->shared->cache->image_addr) ) ||
- ( H5F_addr_gt(file_ptr->shared->eoa_pre_fsm_fsalloc,
+ ( H5F_addr_gt(file_ptr->shared->eoa_fsm_fsalloc,
file_ptr->shared->cache->image_addr) ) ) {
pass = FALSE;
@@ -7554,11 +7552,8 @@ get_free_sections_test(hbool_t single_file_vfd)
pass = FALSE;
failure_mssg = "cache image not loaded (1).\n";
- } else if ( file_ptr->shared->first_alloc_dealloc ) {
+ }
- pass = FALSE;
- failure_mssg = "self referential FSMs not floated (1).\n";
- }
}
if ( show_progress )
@@ -7634,10 +7629,9 @@ get_free_sections_test(hbool_t single_file_vfd)
*/
if ( pass ) {
- if ( ( ! file_ptr->shared->first_alloc_dealloc ) ||
- ( ! H5F_addr_defined(file_ptr->shared->eoa_pre_fsm_fsalloc) ) ||
+ if ( ( ! H5F_addr_defined(file_ptr->shared->eoa_fsm_fsalloc) ) ||
( ! H5F_addr_defined(file_ptr->shared->cache->image_addr) ) ||
- ( H5F_addr_gt(file_ptr->shared->eoa_pre_fsm_fsalloc,
+ ( H5F_addr_gt(file_ptr->shared->eoa_fsm_fsalloc,
file_ptr->shared->cache->image_addr) ) ) {
pass = FALSE;
@@ -7675,11 +7669,8 @@ get_free_sections_test(hbool_t single_file_vfd)
pass = FALSE;
failure_mssg = "cache image not loaded (2).\n";
- } else if ( file_ptr->shared->first_alloc_dealloc ) {
+ }
- pass = FALSE;
- failure_mssg = "self referential FSMs not floated (2).\n";
- }
}
if ( show_progress )
diff --git a/test/cache_tagging.c b/test/cache_tagging.c
index 734c760..dd886eb 100644
--- a/test/cache_tagging.c
+++ b/test/cache_tagging.c
@@ -1664,21 +1664,11 @@ check_attribute_rename_tags(hid_t fcpl, int type)
if ( verify_tag(fid, H5AC_SOHM_LIST_ID, H5AC__SOHM_TAG) < 0 ) TEST_ERROR;
/*
- * 3 calls to verify_tag() for verifying free space:
- * one freespace header tag for H5FD_MEM_DRAW manager,
- * one freespace header tag for H5FD_MEM_SUPER manager
- * one freespace section info tag for H5FD_MEM_SUPER manager
+ * one freespace header tag for H5FD_MEM_DRAW manager,
+ * one freespace header tag for H5FD_MEM_SUPER manager
*/
if ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 ) TEST_ERROR;
-
- /* If the free space managers are persistent, the
- * H5MF_tidy_self_referential_fsm_hack() must have been run.
- * Since this function floats all self referential free space
- * managers, the H5FD_MEM_SUPER FSM will not be in the metadata
- * cache.
- */
- if(!persistent_fsms && verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0) TEST_ERROR;
- if(!persistent_fsms && verify_tag(fid, H5AC_FSPACE_SINFO_ID, H5AC__FREESPACE_TAG) < 0) TEST_ERROR;
+ if ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 ) TEST_ERROR;
/* verify btree header and leaf node belonging to group */
if ( verify_tag(fid, H5AC_BT2_HDR_ID, g_tag) < 0 ) TEST_ERROR;
@@ -1828,13 +1818,16 @@ check_attribute_delete_tags(hid_t fcpl, int type)
/*
* 2 calls to verify_tag() for verifying free space:
- * one freespace header tag for free-space header,
- * one freespace header tag for free-space section info
+ * one freespace header tag for free-space header raw data
+ * one freespace header tag for free-space section info raw data
+ * one freespace header tag for free-space header metadata
*/
if ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 )
TEST_ERROR;
if ( verify_tag(fid, H5AC_FSPACE_SINFO_ID, H5AC__FREESPACE_TAG) < 0 )
TEST_ERROR;
+ if ( verify_tag(fid, H5AC_FSPACE_HDR_ID, H5AC__FREESPACE_TAG) < 0 )
+ TEST_ERROR;
#if 0
/* If the free space managers are persistent, the
diff --git a/test/fheap.c b/test/fheap.c
index 846f03e..557fd37 100644
--- a/test/fheap.c
+++ b/test/fheap.c
@@ -1904,7 +1904,7 @@ test_create(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tparam)
TEST_ERROR
/* Close the fractal heap */
- if(H5HF_close(fh) < 0)
+ if((fh = H5HF_close(fh)) < 0)
FAIL_STACK_ERROR
/* Delete heap */
@@ -6643,7 +6643,7 @@ test_man_remove_one(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpara
FAIL_STACK_ERROR
/* Close the fractal heap */
- if(H5HF_close(fh) < 0)
+ if((fh = H5HF_close(fh)) < 0)
TEST_ERROR
/* Close the file */
@@ -6834,7 +6834,7 @@ test_man_remove_two(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpara
FAIL_STACK_ERROR
/* Close the fractal heap */
- if(H5HF_close(fh) < 0)
+ if((fh = H5HF_close(fh)) < 0)
TEST_ERROR
/* Close the file */
@@ -7001,7 +7001,7 @@ test_man_remove_one_larger(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t
FAIL_STACK_ERROR
/* Close the fractal heap */
- if(H5HF_close(fh) < 0)
+ if((fh = H5HF_close(fh)) < 0)
FAIL_STACK_ERROR
/* Close the file */
@@ -7239,7 +7239,7 @@ test_man_remove_two_larger(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t
FAIL_STACK_ERROR
/* Close the fractal heap */
- if(H5HF_close(fh) < 0)
+ if((fh = H5HF_close(fh)) < 0)
TEST_ERROR
/* Close the file */
@@ -7541,7 +7541,7 @@ test_man_remove_three_larger(hid_t fapl, H5HF_create_t *cparam, fheap_test_param
FAIL_STACK_ERROR
/* Close the fractal heap */
- if(H5HF_close(fh) < 0)
+ if((fh = H5HF_close(fh)) < 0)
TEST_ERROR
/* Close the file */
@@ -13683,23 +13683,23 @@ test_filtered_huge(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tparam
H5F_t *f = NULL; /* Internal file object pointer */
H5HF_t *fh = NULL; /* Fractal heap wrapper */
haddr_t fh_addr; /* Address of fractal heap */
- H5HF_create_t tmp_cparam; /* Local heap creation parameters */
- fheap_heap_ids_t keep_ids; /* Structure to retain heap IDs */
- size_t id_len; /* Size of fractal heap IDs */
- h5_stat_size_t empty_size; /* Size of a file with an empty heap */
- h5_stat_size_t file_size; /* Size of file currently */
- unsigned char *heap_id = NULL; /* Heap ID for object */
- size_t obj_size; /* Size of object */
- size_t robj_size; /* Size of object read */
- unsigned char obj_type; /* Type of storage for object */
- fheap_heap_state_t state; /* State of fractal heap */
- unsigned deflate_level; /* Deflation level */
- size_t old_actual_id_len =0 ; /* Old actual ID length */
- hbool_t huge_ids_direct; /* Are 'huge' objects directly acccessed? */
- const char *base_desc = "insert 'huge' object into heap with I/O filters, then remove %s"; /* Test description */
+ H5HF_create_t tmp_cparam; /* Local heap creation parameters */
+ fheap_heap_ids_t keep_ids; /* Structure to retain heap IDs */
+ size_t id_len; /* Size of fractal heap IDs */
+ h5_stat_size_t empty_size; /* Size of a file with an empty heap */
+ h5_stat_size_t file_size; /* Size of file currently */
+ unsigned char *heap_id = NULL; /* Heap ID for object */
+ size_t obj_size; /* Size of object */
+ size_t robj_size; /* Size of object read */
+ unsigned char obj_type; /* Type of storage for object */
+ fheap_heap_state_t state; /* State of fractal heap */
+ unsigned deflate_level; /* Deflation level */
+ size_t old_actual_id_len =0 ; /* Old actual ID length */
+ hbool_t huge_ids_direct; /* Are 'huge' objects directly acccessed? */
+ const char *base_desc = "insert 'huge' object into heap with I/O filters, then remove %s"; /* Test description */
- /* Copy heap creation properties */
- HDmemcpy(&tmp_cparam, cparam, sizeof(H5HF_create_t));
+ /* Copy heap creation properties */
+ HDmemcpy(&tmp_cparam, cparam, sizeof(H5HF_create_t));
/* Set an I/O filter for heap data */
deflate_level = 6;
@@ -13867,7 +13867,9 @@ error:
H5MM_xfree(heap_id);
if(fh)
H5HF_close(fh);
- H5Fclose(file);
+ if(&tmp_cparam.pline)
+ H5O_msg_reset(H5O_PLINE_ID, &tmp_cparam.pline); /* Release the I/O pipeline filter information */
+ H5Fclose(file);
} H5E_END_TRY;
return(1);
} /* test_filtered_huge() */
@@ -15571,9 +15573,12 @@ error:
H5MM_xfree(keep_ids.ids);
H5MM_xfree(keep_ids.lens);
H5MM_xfree(keep_ids.offs);
+
+ if(tparam->comp == FHEAP_TEST_COMPRESS)
+ H5O_msg_reset(H5O_PLINE_ID, &tmp_cparam.pline); /* Release the I/O pipeline filter information */
if(fh)
H5HF_close(fh);
- H5Fclose(file);
+ H5Fclose(file);
} H5E_END_TRY;
return(1);
} /* test_random() */
@@ -15789,7 +15794,9 @@ error:
H5MM_xfree(keep_ids.offs);
if(fh)
H5HF_close(fh);
- H5Fclose(file);
+ if(tparam->comp == FHEAP_TEST_COMPRESS)
+ H5O_msg_reset(H5O_PLINE_ID, &tmp_cparam.pline); /* Release the I/O pipeline filter information */
+ H5Fclose(file);
} H5E_END_TRY;
return(1);
} /* test_random_pow2() */
diff --git a/test/mf.c b/test/mf.c
index 4dccae6..8c17df5 100644
--- a/test/mf.c
+++ b/test/mf.c
@@ -6822,16 +6822,6 @@ test_mf_fs_persist(const char *env_h5_drvr, hid_t fapl, hbool_t new_format)
if(!H5F_addr_defined(f->shared->fs_addr[tt]))
TEST_ERROR
- /* Since we are about to open a self referential free space
- * manager prior to the first file space allocation / deallocation
- * call H5MF_tidy_self_referential_fsm_hack() first so as to avoid
- * assertion failures on the first file space alloc / dealloc.
- */
- if((f->shared->first_alloc_dealloc) &&
- (SUCCEED !=
- H5MF_tidy_self_referential_fsm_hack(f)))
- FAIL_STACK_ERROR
-
/* Start up H5FD_MEM_SUPER free-space manager */
if(!(f->shared->fs_man[tt]))
if(H5MF__open_fstype(f, (H5F_mem_page_t)tt) < 0)
@@ -6925,7 +6915,7 @@ test_mf_fs_gone(const char *env_h5_drvr, hid_t fapl, hbool_t new_format)
haddr_t addrx;
H5FD_mem_t fs_type;
hbool_t contig_addr_vfd;
- hbool_t ran_H5MF_tidy_self_referential_fsm_hack = FALSE;
+
if(new_format)
TESTING("File's free-space is going away with new library format")
@@ -6944,9 +6934,13 @@ test_mf_fs_gone(const char *env_h5_drvr, hid_t fapl, hbool_t new_format)
/* Copy the file access property list */
if((fapl2 = H5Pcopy(fapl)) < 0) FAIL_STACK_ERROR
- if(new_format)
+ if(new_format) {
if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
FAIL_STACK_ERROR
+ } else {
+ if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_V18, H5F_LIBVER_LATEST) < 0)
+ FAIL_STACK_ERROR
+ }
/* Set to aggregation and persisting free-space */
if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_FSM_AGGR, TRUE, (hsize_t)1) < 0)
@@ -7022,17 +7016,6 @@ test_mf_fs_gone(const char *env_h5_drvr, hid_t fapl, hbool_t new_format)
if(!H5F_addr_defined(f->shared->fs_addr[fs_type]))
TEST_ERROR
- /* Since we are about to open a self referential free space
- * manager prior to the first file space allocation / deallocation
- * call H5MF_tidy_self_referential_fsm_hack() first so as to avoid
- * assertion failures on the first file space alloc / dealloc.
- */
- if(f->shared->first_alloc_dealloc){
- if(SUCCEED != H5MF_tidy_self_referential_fsm_hack(f))
- FAIL_STACK_ERROR
- ran_H5MF_tidy_self_referential_fsm_hack = TRUE;
- }
-
/* Start up H5FD_MEM_SUPER free-space manager */
if(!(f->shared->fs_man[fs_type]))
if(H5MF__open_fstype(f, (H5F_mem_page_t)fs_type) < 0)
@@ -7042,24 +7025,18 @@ test_mf_fs_gone(const char *env_h5_drvr, hid_t fapl, hbool_t new_format)
if(H5FS_stat_info(f, f->shared->fs_man[fs_type], &fs_stat) < 0)
FAIL_STACK_ERROR
- /* if we ran H5MF_tidy_self_referential_fsm_hack(), the
- * H5FD_MEM_SUPER free space manager must be floating.
- * Thus fs_stat.addr must be undefined.
- */
- if((!ran_H5MF_tidy_self_referential_fsm_hack) &&
- (!H5F_addr_defined(fs_stat.addr)))
+ if(!H5F_addr_defined(fs_stat.addr))
TEST_ERROR
if(fs_stat.tot_space < TBLOCK_SIZE3)
TEST_ERROR
- /* Put block #4 to H5FD_MEM_SUPER free-space manager */
- if(H5MF_xfree(f, type, addr4, (hsize_t)TBLOCK_SIZE4) < 0)
- FAIL_STACK_ERROR
-
- if(!new_format) {
- /* Need to take up this space so that the free-space manager will go away */
- if(HADDR_UNDEF == (addrx = H5MF_alloc(f, type, (hsize_t)103)))
+ /* Allocate/free space accordingly so that the free-space manager will go away */
+ if(new_format) {
+ if(H5MF_xfree(f, type, addr4, (hsize_t)TBLOCK_SIZE4) < 0)
+ FAIL_STACK_ERROR
+ } else {
+ if(HADDR_UNDEF == (addrx = H5MF_alloc(f, type, (hsize_t)3)))
FAIL_STACK_ERROR
}
@@ -7741,16 +7718,6 @@ test_page_alloc_xfree(const char *env_h5_drvr, hid_t fapl)
if(!H5F_addr_defined(f->shared->fs_addr[fs_type]))
TEST_ERROR
- /* Since we are about to open a self referential free space
- * manager prior to the first file space allocation / deallocation
- * call H5MF_tidy_self_referential_fsm_hack() first so as to avoid
- * assertion failures on the first file space alloc / dealloc.
- */
- if(f->shared->first_alloc_dealloc){
- if(SUCCEED != H5MF_tidy_self_referential_fsm_hack(f))
- FAIL_STACK_ERROR
- }
-
/* Set up to use the small metadata manager */
if(!(f->shared->fs_man[fs_type]))
if(H5MF__open_fstype(f, fs_type) < 0)
diff --git a/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl b/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl
index affa93b..e78c5e2 100644
--- a/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl
+++ b/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl
@@ -11,6 +11,6 @@ File space page size: 4096 bytes
Summary of file space information:
File metadata: 1602 bytes
Raw data: 400 bytes
- Amount/Percent of tracked free space: 6307 bytes/38.5%
- Unaccounted space: 8075 bytes
-Total space: 16384 bytes
+ Amount/Percent of tracked free space: 6190 bytes/75.6%
+ Unaccounted space: 0 bytes
+Total space: 8192 bytes
diff --git a/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl b/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl
index 1ce06b2..3ecf9b1 100644
--- a/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl
+++ b/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl
@@ -11,6 +11,6 @@ File space page size: 4096 bytes
Summary of file space information:
File metadata: 1485 bytes
Raw data: 400 bytes
- Amount/Percent of tracked free space: 680 bytes/26.5%
+ Amount/Percent of tracked free space: 563 bytes/23.0%
Unaccounted space: 0 bytes
-Total space: 2565 bytes
+Total space: 2448 bytes