summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorJordan Henderson <jhenderson@hdfgroup.org>2017-03-30 17:58:20 (GMT)
committerJordan Henderson <jhenderson@hdfgroup.org>2017-03-30 17:58:20 (GMT)
commit6bc14164e6707ab201234e02d8eadf719a0caa05 (patch)
tree55b8773c59982a455635c15377ed43be0a6b9980 /src
parent588e829b740e600484c965e7d2f90913c45de89f (diff)
parent3a01afc0b15fc63a0fe6800ae3f8f55fc6bfed0a (diff)
downloadhdf5-6bc14164e6707ab201234e02d8eadf719a0caa05.zip
hdf5-6bc14164e6707ab201234e02d8eadf719a0caa05.tar.gz
hdf5-6bc14164e6707ab201234e02d8eadf719a0caa05.tar.bz2
Merge branch 'feature/parallel_filters' of ssh://bitbucket.hdfgroup.org:7999/~jhenderson/hdf5.git into feature/parallel_filters
Diffstat (limited to 'src')
-rw-r--r--src/CMakeLists.txt13
-rw-r--r--src/H5AC.c129
-rw-r--r--src/H5ACdbg.c2
-rw-r--r--src/H5ACprivate.h5
-rw-r--r--src/H5C.c252
-rw-r--r--src/H5Cdbg.c106
-rw-r--r--src/H5Cimage.c231
-rw-r--r--src/H5Cmpio.c991
-rw-r--r--src/H5Cpkg.h33
-rw-r--r--src/H5Cprivate.h63
-rw-r--r--src/H5Ctag.c2
-rw-r--r--src/H5F.c106
-rw-r--r--src/H5FD.c66
-rw-r--r--src/H5FDint.c59
-rw-r--r--src/H5FDlog.c8
-rw-r--r--src/H5FDmulti.c10
-rw-r--r--src/H5FDprivate.h37
-rw-r--r--src/H5FDpublic.h14
-rw-r--r--src/H5FDspace.c81
-rw-r--r--src/H5FDstdio.c7
-rw-r--r--src/H5FS.c197
-rw-r--r--src/H5FScache.c28
-rw-r--r--src/H5FSpkg.h3
-rw-r--r--src/H5FSprivate.h28
-rw-r--r--src/H5FSsection.c307
-rw-r--r--src/H5Faccum.c88
-rw-r--r--src/H5Fint.c246
-rw-r--r--src/H5Fio.c77
-rw-r--r--src/H5Fmount.c10
-rw-r--r--src/H5Fpkg.h79
-rw-r--r--src/H5Fprivate.h93
-rw-r--r--src/H5Fpublic.h15
-rw-r--r--src/H5Fquery.c147
-rw-r--r--src/H5Fspace.c226
-rw-r--r--src/H5Fsuper.c220
-rw-r--r--src/H5HFsection.c42
-rw-r--r--src/H5MF.c3498
-rw-r--r--src/H5MFaggr.c181
-rw-r--r--src/H5MFdbg.c105
-rw-r--r--src/H5MFpkg.h67
-rw-r--r--src/H5MFprivate.h10
-rw-r--r--src/H5MFsection.c714
-rw-r--r--src/H5Oalloc.c8
-rw-r--r--src/H5Ocache_image.c48
-rw-r--r--src/H5Ofsinfo.c190
-rw-r--r--src/H5Omessage.c52
-rw-r--r--src/H5Oprivate.h14
-rw-r--r--src/H5PB.c1533
-rw-r--r--src/H5PBmodule.h34
-rw-r--r--src/H5PBpkg.h60
-rw-r--r--src/H5PBprivate.h108
-rw-r--r--src/H5Pdeprec.c130
-rw-r--r--src/H5Pfapl.c132
-rw-r--r--src/H5Pfcpl.c250
-rw-r--r--src/H5Ppublic.h10
-rw-r--r--src/H5Z.c2
-rw-r--r--src/H5err.txt1
-rw-r--r--src/H5private.h8
-rw-r--r--src/H5system.c33
-rw-r--r--src/H5trace.c37
-rw-r--r--src/H5win32defs.h16
-rw-r--r--src/Makefile.am3
62 files changed, 8793 insertions, 2472 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index ef361b2..9321bbd 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -195,6 +195,7 @@ set (H5F_SOURCES
${HDF5_SRC_DIR}/H5Fmpi.c
${HDF5_SRC_DIR}/H5Fquery.c
${HDF5_SRC_DIR}/H5Fsfile.c
+ ${HDF5_SRC_DIR}/H5Fspace.c
${HDF5_SRC_DIR}/H5Fsuper.c
${HDF5_SRC_DIR}/H5Fsuper_cache.c
${HDF5_SRC_DIR}/H5Ftest.c
@@ -504,6 +505,15 @@ set (H5P_HDRS
)
IDE_GENERATED_PROPERTIES ("H5P" "${H5P_HDRS}" "${H5P_SOURCES}" )
+set (H5PB_SOURCES
+ ${HDF5_SRC_DIR}/H5PB.c
+)
+
+set (H5PB_HDRS
+ ${HDF5_SRC_DIR}/H5PBpkg.h
+)
+IDE_GENERATED_PROPERTIES ("H5PB" "${H5PB_HDRS}" "${H5PB_SOURCES}" )
+
set (H5PL_SOURCES
${HDF5_SRC_DIR}/H5PL.c
)
@@ -699,6 +709,7 @@ set (common_SRCS
${H5MP_SOURCES}
${H5O_SOURCES}
${H5P_SOURCES}
+ ${H5PB_SOURCES}
${H5PL_SOURCES}
${H5R_SOURCES}
${H5UC_SOURCES}
@@ -739,6 +750,7 @@ set (H5_PUBLIC_HEADERS
${H5MP_HDRS}
${H5O_HDRS}
${H5P_HDRS}
+ ${H5PB_HDRS}
${H5PL_HDRS}
${H5R_HDRS}
${H5S_HDRS}
@@ -777,6 +789,7 @@ set (H5_PRIVATE_HEADERS
${HDF5_SRC_DIR}/H5MPprivate.h
${HDF5_SRC_DIR}/H5Oprivate.h
${HDF5_SRC_DIR}/H5Pprivate.h
+ ${HDF5_SRC_DIR}/H5PBprivate.h
${HDF5_SRC_DIR}/H5PLprivate.h
${HDF5_SRC_DIR}/H5UCprivate.h
${HDF5_SRC_DIR}/H5Rprivate.h
diff --git a/src/H5AC.c b/src/H5AC.c
index ee68a6f..a561852 100644
--- a/src/H5AC.c
+++ b/src/H5AC.c
@@ -353,6 +353,44 @@ H5AC_term_package(void)
/*-------------------------------------------------------------------------
+ *
+ * Function: H5AC_cache_image_pending()
+ *
+ * Purpose: Debugging function that tests to see if the load of a
+ * metadata cache image load is pending (i.e. will be executed
+ * on the next protect or insert)
+ *
+ * Returns TRUE if a cache image load is pending, and FALSE
+ * if not. Throws an assertion failure on error.
+ *
+ * Return: TRUE if a cache image load is pending, and FALSE otherwise.
+ *
+ * Programmer: John Mainzer, 1/10/17
+ *
+ * Changes: None.
+ *
+ *-------------------------------------------------------------------------
+ */
+hbool_t
+H5AC_cache_image_pending(const H5F_t *f)
+{
+ H5C_t *cache_ptr;
+ hbool_t ret_value = FALSE; /* Return value */
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ /* Sanity checks */
+ HDassert(f);
+ HDassert(f->shared);
+ cache_ptr = f->shared->cache;
+
+ ret_value = H5C_cache_image_pending(cache_ptr);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5AC_cache_image_pending() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5AC_create
*
* Purpose: Initialize the cache just after a file is opened. The
@@ -501,7 +539,7 @@ H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr, H5AC_cache_image_co
/* Turn on metadata cache logging, if being used */
if(H5F_USE_MDC_LOGGING(f)) {
if(H5C_set_up_logging(f->shared->cache, H5F_MDC_LOG_LOCATION(f), H5F_START_MDC_LOG_ON_ACCESS(f)) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "mdc logging setup failed")
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTINIT, FAIL, "mdc logging setup failed")
/* Write the log header regardless of current logging status */
if(H5AC__write_create_cache_log_msg(f->shared->cache) < 0)
@@ -510,9 +548,9 @@ H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr, H5AC_cache_image_co
/* Set the cache parameters */
if(H5AC_set_cache_auto_resize_config(f->shared->cache, config_ptr) < 0)
- HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "auto resize configuration failed")
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTSET, FAIL, "auto resize configuration failed")
- /* don't need to get the current H5C image config here since the
+ /* Don't need to get the current H5C image config here since the
* cache has just been created, and thus f->shared->cache->image_ctl
* must still set to its initial value (H5C__DEFAULT_CACHE_IMAGE_CTL).
* Note that this not true as soon as control returns to the application
@@ -522,9 +560,8 @@ H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr, H5AC_cache_image_co
int_ci_config.generate_image = image_config_ptr->generate_image;
int_ci_config.save_resize_status = image_config_ptr->save_resize_status;
int_ci_config.entry_ageout = image_config_ptr->entry_ageout;
-
if(H5C_set_cache_image_config(f, f->shared->cache, &int_ci_config) < 0)
- HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "auto resize configuration failed")
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTSET, FAIL, "auto resize configuration failed")
done:
#ifdef H5_HAVE_PARALLEL
@@ -836,6 +873,46 @@ done:
/*-------------------------------------------------------------------------
+ * Function: H5AC_force_cache_image_load()
+ *
+ * Purpose: On rare occasions, it is necessary to run
+ * H5MF_tidy_self_referential_fsm_hack() prior to the first
+ * metadata cache access. This is a problem as if there is a
+ * cache image at the end of the file, that routine will
+ * discard it.
+ *
+ * We solve this issue by calling this function, which will
+ * load the cache image and then call
+ * H5MF_tidy_self_referential_fsm_hack() to discard it.
+ *
+ * Return: SUCCEED on success, and FAIL on failure.
+ *
+ * Programmer: John Mainzer
+ * 1/11/17
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5AC_force_cache_image_load(H5F_t *f, hid_t dxpl_id)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* Sanity checks */
+ HDassert(f);
+ HDassert(f->shared);
+ HDassert(f->shared->cache);
+
+ if(H5C_force_cache_image_load(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, FAIL, "Can't load cache image")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5AC_force_cache_image_load() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5AC_get_entry_status
*
* Purpose: Given a file address, determine whether the metadata
@@ -3120,6 +3197,48 @@ done:
/*-------------------------------------------------------------------------
+ * Function: H5AC_unsettle_ring()
+ *
+ * Purpose: Advise the metadata cache that the specified free space
+ * manager ring is no longer settled (if it was on entry).
+ *
+ * If the target free space manager ring is already
+ * unsettled, do nothing, and return SUCCEED.
+ *
+ * If the target free space manager ring is settled, and
+ * we are not in the process of a file shutdown, mark
+ * the ring as unsettled, and return SUCCEED.
+ *
+ * If the target free space manager is settled, and we
+ * are in the process of a file shutdown, post an error
+ * message, and return FAIL.
+ *
+ * Note that this function simply passes the call on to
+ * the metadata cache proper, and returns the result.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: John Mainzer
+ * 10/15/16
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5AC_unsettle_ring(H5F_t * f, H5C_ring_t ring)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ if(FAIL == (ret_value = H5C_unsettle_ring(f, ring)))
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_unsettle_ring() failed")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5AC_unsettle_ring() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5AC_remove_entry()
*
* Purpose: Remove an entry from the cache. Must be not protected, pinned,
diff --git a/src/H5ACdbg.c b/src/H5ACdbg.c
index 8ca5102..6073288 100644
--- a/src/H5ACdbg.c
+++ b/src/H5ACdbg.c
@@ -101,6 +101,7 @@ H5AC_stats(const H5F_t *f)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* H5AC_stats() */
+#ifndef NDEBUG
/*-------------------------------------------------------------------------
* Function: H5AC_dump_cache
@@ -133,6 +134,7 @@ H5AC_dump_cache(const H5F_t *f)
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5AC_dump_cache() */
+#endif /* NDEBUG */
/*-------------------------------------------------------------------------
diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h
index 1fe6456..1f8299b 100644
--- a/src/H5ACprivate.h
+++ b/src/H5ACprivate.h
@@ -451,6 +451,8 @@ H5_DLL herr_t H5AC_validate_config(H5AC_cache_config_t *config_ptr);
H5_DLL herr_t H5AC_load_cache_image_on_next_protect(H5F_t *f, haddr_t addr,
hsize_t len, hbool_t rw);
H5_DLL herr_t H5AC_validate_cache_image_config(H5AC_cache_image_config_t *config_ptr);
+H5_DLL hbool_t H5AC_cache_image_pending(const H5F_t *f);
+H5_DLL herr_t H5AC_force_cache_image_load(H5F_t * f, hid_t dxpl_id);
/* Tag & Ring routines */
H5_DLL herr_t H5AC_tag(hid_t dxpl_id, haddr_t metadata_tag, haddr_t *prev_tag);
@@ -464,6 +466,7 @@ H5_DLL herr_t H5AC_set_ring(hid_t dxpl_id, H5AC_ring_t ring, H5P_genplist_t **dx
H5AC_ring_t *orig_ring);
H5_DLL herr_t H5AC_reset_ring(H5P_genplist_t *dxpl, H5AC_ring_t orig_ring);
H5_DLL herr_t H5AC_unsettle_entry_ring(void *entry);
+H5_DLL herr_t H5AC_unsettle_ring(H5F_t * f, H5AC_ring_t ring);
H5_DLL herr_t H5AC_expunge_tag_type_metadata(H5F_t *f, hid_t dxpl_id, haddr_t tag, int type_id, unsigned flags);
H5_DLL herr_t H5AC_get_tag(const void *thing, /*OUT*/ haddr_t *tag);
@@ -482,8 +485,8 @@ H5_DLL herr_t H5AC_add_candidate(H5AC_t * cache_ptr, haddr_t addr);
/* Debugging functions */
H5_DLL herr_t H5AC_stats(const H5F_t *f);
-H5_DLL herr_t H5AC_dump_cache(const H5F_t *f);
#ifndef NDEBUG
+H5_DLL herr_t H5AC_dump_cache(const H5F_t *f);
H5_DLL herr_t H5AC_get_entry_ptr_from_addr(const H5F_t *f, haddr_t addr,
void **entry_ptr_ptr);
H5_DLL herr_t H5AC_flush_dependency_exists(H5F_t *f, haddr_t parent_addr,
diff --git a/src/H5C.c b/src/H5C.c
index 805b4f5..120abb8 100644
--- a/src/H5C.c
+++ b/src/H5C.c
@@ -386,6 +386,7 @@ H5C_create(size_t max_cache_size,
cache_ptr->cache_full = FALSE;
cache_ptr->size_decreased = FALSE;
cache_ptr->resize_in_progress = FALSE;
+ cache_ptr->msic_in_progress = FALSE;
(cache_ptr->resize_ctl).version = H5C__CURR_AUTO_SIZE_CTL_VER;
(cache_ptr->resize_ctl).rpt_fcn = NULL;
@@ -731,7 +732,8 @@ herr_t
H5C_prep_for_file_close(H5F_t *f, hid_t dxpl_id)
{
H5C_t * cache_ptr;
- herr_t ret_value = SUCCEED; /* Return value */
+ hbool_t image_generated = FALSE; /* Whether a cache image was generated */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
@@ -755,9 +757,48 @@ H5C_prep_for_file_close(H5F_t *f, hid_t dxpl_id)
HDassert(cache_ptr->pl_len == 0);
/* Prepare cache image */
- if(H5C__prep_image_for_file_close(f, dxpl_id) < 0)
+ if(H5C__prep_image_for_file_close(f, dxpl_id, &image_generated) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, FAIL, "can't create cache image")
+#ifdef H5_HAVE_PARALLEL
+ if(!image_generated && cache_ptr->aux_ptr != NULL && f->shared->fs_persist) {
+ /* If persistent free space managers are enabled, flushing the
+ * metadata cache may result in the deletion, insertion, and/or
+ * dirtying of entries.
+ *
+ * This is a problem in PHDF5, as it breaks two invariants of
+ * our management of the metadata cache across all processes:
+ *
+ * 1) Entries will not be dirtied, deleted, inserted, or moved
+ * during flush in the parallel case.
+ *
+ * 2) All processes contain the same set of dirty metadata
+ * entries on entry to a sync point.
+ *
+ * To solve this problem for the persistent free space managers,
+ * serialize the metadata cache on all processes prior to the
+ * first sync point on file shutdown. The shutdown warning is
+ * a convenient location for this call.
+ *
+ * This is sufficient since:
+ *
+ * 1) FSM settle routines are only invoked on file close. Since
+ * serialization make the same settle calls as flush on file
+ * close, and since the close warning is issued after all
+ * non FSM related space allocations and just before the
+ * first sync point on close, this call will leave the caches
+ * in a consistant state across the processes if they were
+ * consistant before.
+ *
+ * 2) Since the FSM settle routines are only invoked once during
+ * file close, invoking them now will prevent their invocation
+ * during a flush, and thus avoid any resulting entrie dirties,
+ * deletions, insertion, or moves during the flush.
+ */
+ if(H5C__serialize_cache(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTSERIALIZE, FAIL, "serialization of the cache failed")
+ } /* end if */
+#endif /* H5_HAVE_PARALLEL */
done:
FUNC_LEAVE_NOAPI(ret_value)
@@ -925,12 +966,6 @@ H5C_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type,
HGOTO_ERROR(H5E_CACHE, H5E_CANTEXPUNGE, FAIL, "Target entry is protected")
if(entry_ptr->is_pinned)
HGOTO_ERROR(H5E_CACHE, H5E_CANTEXPUNGE, FAIL, "Target entry is pinned")
-#ifdef H5_HAVE_PARALLEL
- if(entry_ptr->coll_access) {
- entry_ptr->coll_access = FALSE;
- H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL)
- } /* end if */
-#endif /* H5_HAVE_PARALLEL */
/* If we get this far, call H5C__flush_single_entry() with the
* H5C__FLUSH_INVALIDATE_FLAG and the H5C__FLUSH_CLEAR_ONLY_FLAG.
@@ -1465,10 +1500,15 @@ H5C_insert_entry(H5F_t * f,
entry_ptr->prefetched = FALSE;
entry_ptr->prefetch_type_id = 0;
entry_ptr->age = 0;
+ entry_ptr->prefetched_dirty = FALSE;
#ifndef NDEBUG /* debugging field */
entry_ptr->serialization_count = 0;
#endif /* NDEBUG */
+ entry_ptr->tl_next = NULL;
+ entry_ptr->tl_prev = NULL;
+ entry_ptr->tag_info = NULL;
+
/* Apply tag to newly inserted entry */
if(H5C__tag_entry(cache_ptr, entry_ptr, dxpl_id) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "Cannot tag metadata entry")
@@ -2437,7 +2477,7 @@ H5C_protect(H5F_t * f,
marked as collective, and is clean, it is possible that
other processes will not have it in its cache and will
expect a bcast of the entry from process 0. So process 0
- will bcast the entry to all other ranks. Ranks that do have
+ will bcast the entry to all other ranks. Ranks that _do_ have
the entry in their cache still have to participate in the
bcast. */
#ifdef H5_HAVE_PARALLEL
@@ -3537,6 +3577,73 @@ done:
/*-------------------------------------------------------------------------
+ * Function: H5C_unsettle_ring()
+ *
+ * Purpose: Advise the metadata cache that the specified free space
+ * manager ring is no longer settled (if it was on entry).
+ *
+ * If the target free space manager ring is already
+ * unsettled, do nothing, and return SUCCEED.
+ *
+ * If the target free space manager ring is settled, and
+ * we are not in the process of a file shutdown, mark
+ * the ring as unsettled, and return SUCCEED.
+ *
+ * If the target free space manager is settled, and we
+ * are in the process of a file shutdown, post an error
+ * message, and return FAIL.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: John Mainzer
+ * 10/15/16
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5C_unsettle_ring(H5F_t * f, H5C_ring_t ring)
+{
+ H5C_t * cache_ptr;
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* Sanity checks */
+ HDassert(f);
+ HDassert(f->shared);
+ HDassert(f->shared->cache);
+ HDassert((H5C_RING_RDFSM == ring) || (H5C_RING_MDFSM == ring));
+ cache_ptr = f->shared->cache;
+ HDassert(H5C__H5C_T_MAGIC == cache_ptr->magic);
+
+ switch(ring) {
+ case H5C_RING_RDFSM:
+ if(cache_ptr->rdfsm_settled) {
+ if(cache_ptr->close_warning_received)
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unexpected rdfsm ring unsettle")
+ cache_ptr->rdfsm_settled = FALSE;
+ } /* end if */
+ break;
+
+ case H5C_RING_MDFSM:
+ if(cache_ptr->mdfsm_settled) {
+ if(cache_ptr->close_warning_received)
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unexpected mdfsm ring unsettle")
+ cache_ptr->mdfsm_settled = FALSE;
+ } /* end if */
+ break;
+
+ default:
+ HDassert(FALSE); /* this should be un-reachable */
+ break;
+ } /* end switch */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5C_unsettle_ring() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5C_validate_resize_config()
*
* Purpose: Run a sanity check on the specified sections of the
@@ -4586,7 +4693,7 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f,
( (entry_ptr->type)->id != H5AC_EPOCH_MARKER_ID ) &&
( bytes_evicted < eviction_size_limit ) )
{
- hbool_t corked = FALSE;
+ hbool_t skipping_entry = FALSE;
HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC);
HDassert( ! (entry_ptr->is_protected) );
@@ -4600,9 +4707,11 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f,
prev_is_dirty = prev_ptr->is_dirty;
if(entry_ptr->is_dirty ) {
+ HDassert(!entry_ptr->prefetched_dirty);
+
/* dirty corked entry is skipped */
if(entry_ptr->tag_info && entry_ptr->tag_info->corked)
- corked = TRUE;
+ skipping_entry = TRUE;
else {
/* reset entries_removed_counter and
* last_entry_removed_ptr prior to the call to
@@ -4622,16 +4731,22 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f,
restart_scan = TRUE;
} /* end else */
} /* end if */
- else {
+ else if(!entry_ptr->prefetched_dirty) {
bytes_evicted += entry_ptr->size;
if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0 )
HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry")
- }
+ } /* end else-if */
+ else {
+ HDassert(!entry_ptr->is_dirty);
+ HDassert(entry_ptr->prefetched_dirty);
+
+ skipping_entry = TRUE;
+ } /* end else */
if(prev_ptr != NULL) {
- if(corked) /* dirty corked entry is skipped */
+ if(skipping_entry)
entry_ptr = prev_ptr;
else if(restart_scan || (prev_ptr->is_dirty != prev_is_dirty)
|| (prev_ptr->next != next_ptr)
@@ -4691,10 +4806,10 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f,
prev_ptr = entry_ptr->prev;
- if(!(entry_ptr->is_dirty)) {
+ if(!(entry_ptr->is_dirty) && !(entry_ptr->prefetched_dirty))
if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush clean entry")
- } /* end if */
+
/* just skip the entry if it is dirty, as we can't do
* anything with it now since we can't write.
*
@@ -5999,6 +6114,7 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr,
hbool_t write_entry; /* internal flag */
hbool_t destroy_entry; /* internal flag */
hbool_t generate_image; /* internal flag */
+ hbool_t update_page_buffer; /* internal flag */
hbool_t was_dirty;
hbool_t suppress_image_entry_writes = FALSE;
hbool_t suppress_image_entry_frees = FALSE;
@@ -6024,6 +6140,7 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr,
del_from_slist_on_destroy = ((flags & H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) != 0);
during_flush = ((flags & H5C__DURING_FLUSH_FLAG) != 0);
generate_image = ((flags & H5C__GENERATE_IMAGE_FLAG) != 0);
+ update_page_buffer = ((flags & H5C__UPDATE_PAGE_BUFFER_FLAG) != 0);
/* Set the flag for destroying the entry, based on the 'take ownership'
* and 'destroy' flags
@@ -6033,10 +6150,6 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr,
else
destroy_entry = destroy;
-#ifdef H5_HAVE_PARALLEL
- HDassert(FALSE == entry_ptr->coll_access);
-#endif
-
/* we will write the entry to disk if it exists, is dirty, and if the
* clear only flag is not set.
*/
@@ -6232,9 +6345,11 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr,
*
* 2) Delete it from the skip list if requested.
*
- * 3) Update the replacement policy for eviction
+ * 3) Delete it from the collective read access list.
*
- * 4) Remove it from the tag list for this object
+ * 4) Update the replacement policy for eviction
+ *
+ * 5) Remove it from the tag list for this object
*
* Finally, if the destroy_entry flag is set, discard the
* entry.
@@ -6244,6 +6359,14 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr,
if(entry_ptr->in_slist && del_from_slist_on_destroy)
H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush)
+#ifdef H5_HAVE_PARALLEL
+ /* Check for collective read access flag */
+ if(entry_ptr->coll_access) {
+ entry_ptr->coll_access = FALSE;
+ H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL)
+ } /* end if */
+#endif /* H5_HAVE_PARALLEL */
+
H5C__UPDATE_RP_FOR_EVICTION(cache_ptr, entry_ptr, FAIL)
/* Remove entry from tag list */
@@ -6290,7 +6413,8 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr,
HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag cleared")
/* Propagate the clean flag up the flush dependency chain if appropriate */
- HDassert(entry_ptr->flush_dep_ndirty_children == 0);
+ if(entry_ptr->flush_dep_ndirty_children != 0)
+ HDassert(entry_ptr->flush_dep_ndirty_children == 0);
if(entry_ptr->flush_dep_nparents > 0)
if(H5C__mark_flush_dep_clean(entry_ptr) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep clean flag")
@@ -6432,6 +6556,19 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr,
} /* end else */
} /* if (destroy) */
+ /* Check if we have to update the page buffer with cleared entries
+ * so it doesn't go out of date
+ */
+ if(update_page_buffer) {
+ /* Sanity check */
+ HDassert(!destroy);
+ HDassert(entry_ptr->image_ptr);
+
+ if(f->shared->page_buf && f->shared->page_buf->page_size >= entry_ptr->size)
+ if(H5PB_update_entry(f->shared->page_buf, entry_ptr->addr, entry_ptr->size, entry_ptr->image_ptr) > 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Failed to update PB with metadata cache")
+ } /* end if */
+
if(cache_ptr->log_flush)
if((cache_ptr->log_flush)(cache_ptr, entry_addr, was_dirty, flags) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "log_flush callback failed")
@@ -6822,10 +6959,15 @@ H5C_load_entry(H5F_t * f,
entry->prefetched = FALSE;
entry->prefetch_type_id = 0;
entry->age = 0;
+ entry->prefetched_dirty = FALSE;
#ifndef NDEBUG /* debugging field */
entry->serialization_count = 0;
#endif /* NDEBUG */
+ entry->tl_next = NULL;
+ entry->tl_prev = NULL;
+ entry->tag_info = NULL;
+
H5C__RESET_CACHE_ENTRY_STATS(entry);
ret_value = thing;
@@ -6864,13 +7006,6 @@ done:
* Thus the function simply does its best, returning success
* unless an error is encountered.
*
- * The primary_dxpl_id and secondary_dxpl_id parameters
- * specify the dxpl_ids used on the first write occasioned
- * by the call (primary_dxpl_id), and on all subsequent
- * writes (secondary_dxpl_id). This is useful in the metadata
- * cache, but may not be needed elsewhere. If so, just use the
- * same dxpl_id for both parameters.
- *
* Observe that this function cannot occasion a read.
*
* Return: Non-negative on success/Negative on failure.
@@ -6886,11 +7021,13 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed,
H5C_t * cache_ptr = f->shared->cache;
#if H5C_COLLECT_CACHE_STATS
int32_t clean_entries_skipped = 0;
+ int32_t dirty_pf_entries_skipped = 0;
int32_t total_entries_scanned = 0;
#endif /* H5C_COLLECT_CACHE_STATS */
uint32_t entries_examined = 0;
uint32_t initial_list_len;
size_t empty_space;
+ hbool_t reentrant_call = FALSE;
hbool_t prev_is_dirty = FALSE;
hbool_t didnt_flush_entry = FALSE;
hbool_t restart_scan;
@@ -6908,6 +7045,18 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed,
HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC);
HDassert(cache_ptr->index_size == (cache_ptr->clean_index_size + cache_ptr->dirty_index_size));
+ /* check to see if cache_ptr->msic_in_progress is TRUE. If it, this
+ * is a re-entrant call via a client callback called in the make
+ * space in cache process. To avoid an infinite recursion, set
+ * reentrant_call to TRUE, and goto done.
+ */
+ if(cache_ptr->msic_in_progress) {
+ reentrant_call = TRUE;
+ HGOTO_DONE(SUCCEED);
+ } /* end if */
+
+ cache_ptr->msic_in_progress = TRUE;
+
if ( write_permitted ) {
restart_scan = FALSE;
initial_list_len = cache_ptr->LRU_list_len;
@@ -6954,7 +7103,8 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed,
didnt_flush_entry = TRUE;
} else if ( ( (entry_ptr->type)->id != H5AC_EPOCH_MARKER_ID ) &&
- ( ! entry_ptr->flush_in_progress ) ) {
+ ( ! entry_ptr->flush_in_progress ) &&
+ ( ! entry_ptr->prefetched_dirty ) ) {
didnt_flush_entry = FALSE;
@@ -6980,13 +7130,6 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed,
cache_ptr->entries_removed_counter = 0;
cache_ptr->last_entry_removed_ptr = NULL;
-#ifdef H5_HAVE_PARALLEL
- if(TRUE == entry_ptr->coll_access) {
- entry_ptr->coll_access = FALSE;
- H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL)
- } /* end if */
-#endif
-
if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__NO_FLAGS_SET) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry")
@@ -7020,10 +7163,16 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed,
} else {
- /* Skip epoch markers and entries that are in the process
- * of being flushed.
+ /* Skip epoch markers, entries that are in the process
+ * of being flushed, and entries marked as prefetched_dirty
+ * (occurs in the R/O case only).
*/
didnt_flush_entry = TRUE;
+
+#if H5C_COLLECT_CACHE_STATS
+ if(entry_ptr->prefetched_dirty)
+ dirty_pf_entries_skipped++;
+#endif /* H5C_COLLECT_CACHE_STATS */
}
if ( prev_ptr != NULL ) {
@@ -7088,6 +7237,7 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed,
cache_ptr->calls_to_msic++;
cache_ptr->total_entries_skipped_in_msic += clean_entries_skipped;
+ cache_ptr->total_dirty_pf_entries_skipped_in_msic += dirty_pf_entries_skipped;
cache_ptr->total_entries_scanned_in_msic += total_entries_scanned;
if ( clean_entries_skipped > cache_ptr->max_entries_skipped_in_msic ) {
@@ -7095,6 +7245,9 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed,
cache_ptr->max_entries_skipped_in_msic = clean_entries_skipped;
}
+ if(dirty_pf_entries_skipped > cache_ptr->max_dirty_pf_entries_skipped_in_msic)
+ cache_ptr->max_dirty_pf_entries_skipped_in_msic = dirty_pf_entries_skipped;
+
if ( total_entries_scanned > cache_ptr->max_entries_scanned_in_msic ) {
cache_ptr->max_entries_scanned_in_msic = total_entries_scanned;
@@ -7163,6 +7316,12 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed,
}
done:
+ /* Sanity checks */
+ HDassert(cache_ptr->msic_in_progress);
+ if(!reentrant_call)
+ cache_ptr->msic_in_progress = FALSE;
+ HDassert((!reentrant_call) || (cache_ptr->msic_in_progress));
+
FUNC_LEAVE_NOAPI(ret_value)
} /* H5C__make_space_in_cache() */
@@ -8723,12 +8882,21 @@ H5C_remove_entry(void *_entry)
/* Update the cache internal data structures as appropriate for a destroy.
* Specifically:
* 1) Delete it from the index
- * 2) Update the replacement policy for eviction
- * 3) Remove it from the tag list for this object
+ * 2) Delete it from the collective read access list
+ * 3) Update the replacement policy for eviction
+ * 4) Remove it from the tag list for this object
*/
H5C__DELETE_FROM_INDEX(cache, entry, FAIL)
+#ifdef H5_HAVE_PARALLEL
+ /* Check for collective read access flag */
+ if(entry->coll_access) {
+ entry->coll_access = FALSE;
+ H5C__REMOVE_FROM_COLL_LIST(cache, entry, FAIL)
+ } /* end if */
+#endif /* H5_HAVE_PARALLEL */
+
H5C__UPDATE_RP_FOR_EVICTION(cache, entry, FAIL)
/* Remove entry from tag list */
diff --git a/src/H5Cdbg.c b/src/H5Cdbg.c
index eb5f123..a955eaf 100644
--- a/src/H5Cdbg.c
+++ b/src/H5Cdbg.c
@@ -70,6 +70,7 @@
/*******************/
+#ifndef NDEBUG
/*-------------------------------------------------------------------------
* Function: H5C_dump_cache
@@ -175,6 +176,85 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5C_dump_cache() */
+#endif /* NDEBUG */
+
+#ifndef NDEBUG
+
+/*-------------------------------------------------------------------------
+ * Function: H5C_dump_cache_LRU
+ *
+ * Purpose: Print a summary of the contents of the metadata cache
+ * LRU for debugging purposes.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: John Mainzer
+ * 10/10/10
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5C_dump_cache_LRU(H5C_t *cache_ptr, const char *cache_name)
+{
+ H5C_cache_entry_t * entry_ptr;
+ int i = 0;
+
+ FUNC_ENTER_NOAPI_NOERR
+
+ /* Sanity check */
+ HDassert(cache_ptr != NULL);
+ HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC);
+ HDassert(cache_name != NULL );
+
+ HDfprintf(stdout, "\n\nDump of metadata cache LRU \"%s\"\n", cache_name);
+ HDfprintf(stdout, "LRU len = %d, LRU size = %d\n",
+ cache_ptr->LRU_list_len, (int)(cache_ptr->LRU_list_size));
+ HDfprintf(stdout, "index_size = %d, max_cache_size = %d, delta = %d\n\n",
+ (int)(cache_ptr->index_size), (int)(cache_ptr->max_cache_size),
+ (int)(cache_ptr->max_cache_size) - (int)(cache_ptr->index_size));
+
+ /* Print header */
+ HDfprintf(stdout, "Entry ");
+ HDfprintf(stdout, "| Address ");
+ HDfprintf(stdout, "| Tag ");
+ HDfprintf(stdout, "| Size ");
+ HDfprintf(stdout, "| Ring ");
+ HDfprintf(stdout, "| Type ");
+ HDfprintf(stdout, "| Dirty");
+ HDfprintf(stdout, "\n");
+
+ HDfprintf(stdout, "----------------------------------------------------------------------------------------------------------------\n");
+
+ entry_ptr = cache_ptr->LRU_head_ptr;
+ while(entry_ptr != NULL) {
+ HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC);
+
+ /* Print entry */
+ HDfprintf(stdout, "%s%5d ", cache_ptr->prefix, i);
+ HDfprintf(stdout, " 0x%16llx ", (long long)(entry_ptr->addr));
+
+ if(NULL == entry_ptr->tag_info)
+ HDfprintf(stdout, " %16s ", "N/A");
+ else
+ HDfprintf(stdout, " 0x%16llx ",
+ (long long)(entry_ptr->tag_info->tag));
+
+ HDfprintf(stdout, " %5lld ", (long long)(entry_ptr->size));
+ HDfprintf(stdout, " %d ", (int)(entry_ptr->ring));
+ HDfprintf(stdout, " %2d %-32s ", (int)(entry_ptr->type->id),
+ (entry_ptr->type->name));
+ HDfprintf(stdout, " %d", (int)(entry_ptr->is_dirty));
+ HDfprintf(stdout, "\n");
+
+ i++;
+ entry_ptr = entry_ptr->next;
+ } /* end while */
+
+ HDfprintf(stdout, "----------------------------------------------------------------------------------------------------------------\n");
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5C_dump_cache_LRU() */
+#endif /* NDEBUG */
/*-------------------------------------------------------------------------
@@ -385,6 +465,7 @@ H5C_stats(H5C_t * cache_ptr,
double average_successful_search_depth = 0.0f;
double average_failed_search_depth = 0.0f;
double average_entries_skipped_per_calls_to_msic = 0.0f;
+ double average_dirty_pf_entries_skipped_per_call_to_msic = 0.0f;
double average_entries_scanned_per_calls_to_msic = 0.0f;
#endif /* H5C_COLLECT_CACHE_STATS */
herr_t ret_value = SUCCEED; /* Return value */
@@ -620,6 +701,17 @@ H5C_stats(H5C_t * cache_ptr,
(long)(cache_ptr->max_entries_skipped_in_msic));
if(cache_ptr->calls_to_msic > 0)
+ average_dirty_pf_entries_skipped_per_call_to_msic =
+ (((double)(cache_ptr->total_dirty_pf_entries_skipped_in_msic)) /
+ ((double)(cache_ptr->calls_to_msic)));
+
+ HDfprintf(stdout,
+ "%s MSIC: Average/max dirty pf entries skipped = %lf / %ld\n",
+ cache_ptr->prefix,
+ average_dirty_pf_entries_skipped_per_call_to_msic,
+ (long)(cache_ptr->max_dirty_pf_entries_skipped_in_msic));
+
+ if(cache_ptr->calls_to_msic > 0)
average_entries_scanned_per_calls_to_msic =
(((double)(cache_ptr->total_entries_scanned_in_msic)) /
((double)(cache_ptr->calls_to_msic)));
@@ -887,12 +979,14 @@ H5C_stats__reset(H5C_t H5_ATTR_UNUSED * cache_ptr)
cache_ptr->max_pel_len = 0;
cache_ptr->max_pel_size = (size_t)0;
- cache_ptr->calls_to_msic = 0;
- cache_ptr->total_entries_skipped_in_msic = 0;
- cache_ptr->total_entries_scanned_in_msic = 0;
- cache_ptr->max_entries_skipped_in_msic = 0;
- cache_ptr->max_entries_scanned_in_msic = 0;
- cache_ptr->entries_scanned_to_make_space = 0;
+ cache_ptr->calls_to_msic = 0;
+ cache_ptr->total_entries_skipped_in_msic = 0;
+ cache_ptr->total_dirty_pf_entries_skipped_in_msic = 0;
+ cache_ptr->total_entries_scanned_in_msic = 0;
+ cache_ptr->max_entries_skipped_in_msic = 0;
+ cache_ptr->max_dirty_pf_entries_skipped_in_msic = 0;
+ cache_ptr->max_entries_scanned_in_msic = 0;
+ cache_ptr->entries_scanned_to_make_space = 0;
cache_ptr->slist_scan_restarts = 0;
cache_ptr->LRU_scan_restarts = 0;
diff --git a/src/H5Cimage.c b/src/H5Cimage.c
index 1da2545..fc58dac 100644
--- a/src/H5Cimage.c
+++ b/src/H5Cimage.c
@@ -147,6 +147,40 @@ H5FL_DEFINE(H5C_cache_entry_t);
/*-------------------------------------------------------------------------
+ *
+ * Function: H5C_cache_image_pending()
+ *
+ * Purpose: Tests to see if the load of a metadata cache image
+ * load is pending (i.e. will be executed on the next
+ * protect or insert)
+ *
+ * Returns TRUE if a cache image load is pending, and FALSE
+ * if not. Throws an assertion failure on error.
+ *
+ * Return: TRUE if a cache image load is pending, and FALSE otherwise.
+ *
+ * Programmer: John Mainzer, 6/18/16
+ *
+ *-------------------------------------------------------------------------
+ */
+hbool_t
+H5C_cache_image_pending(const H5C_t *cache_ptr)
+{
+ hbool_t ret_value = TRUE; /* Return value */
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ /* Sanity checks */
+ HDassert(cache_ptr);
+ HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC);
+
+ ret_value = (cache_ptr->load_image && !cache_ptr->image_loaded);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5C_cache_image_pending() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5C_cache_image_status()
*
* Purpose: Examine the metadata cache associated with the supplied
@@ -489,7 +523,6 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr,
*/
HDassert(!((type->flags & H5C__CLASS_SKIP_READS) &&
(type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG)));
-
HDassert(H5F_addr_defined(addr));
HDassert(type->get_initial_load_size);
HDassert(type->deserialize);
@@ -498,6 +531,7 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr,
* relationships now. The client will restore the relationship(s) with
* the deserialized entry if appropriate.
*/
+ HDassert(pf_entry_ptr->fd_parent_count == pf_entry_ptr->flush_dep_nparents);
for(i = (int)(pf_entry_ptr->fd_parent_count) - 1; i >= 0; i--) {
HDassert(pf_entry_ptr->flush_dep_parent);
HDassert(pf_entry_ptr->flush_dep_parent[i]);
@@ -525,7 +559,7 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr,
*/
if(pf_entry_ptr->fd_child_count > 0) {
if(NULL == (fd_children = (H5C_cache_entry_t **)H5MM_calloc(sizeof(H5C_cache_entry_t **) * (size_t)(pf_entry_ptr->fd_child_count + 1))))
- HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for fd child ptr array")
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for fd child ptr array")
if(H5C__destroy_pf_entry_child_flush_deps(cache_ptr, pf_entry_ptr, fd_children) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTUNDEPEND, FAIL, "can't destroy pf entry child flush dependency(s).")
@@ -544,7 +578,6 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr,
*/
if(NULL == (thing = type->deserialize(pf_entry_ptr->image_ptr, len, udata, &dirty)))
HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, FAIL, "Can't deserialize image")
-
ds_entry_ptr = (H5C_cache_entry_t *)thing;
/* In general, an entry should be clean just after it is loaded.
@@ -583,8 +616,7 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr,
ds_entry_ptr->image_ptr = pf_entry_ptr->image_ptr;
ds_entry_ptr->image_up_to_date = !dirty;
ds_entry_ptr->type = type;
- ds_entry_ptr->is_dirty = dirty |
- pf_entry_ptr->is_dirty;
+ ds_entry_ptr->is_dirty = dirty | pf_entry_ptr->is_dirty;
ds_entry_ptr->dirtied = FALSE;
ds_entry_ptr->is_protected = FALSE;
ds_entry_ptr->is_read_only = FALSE;
@@ -626,7 +658,7 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr,
pf_entry_ptr->coll_prev = NULL;
#endif /* H5_HAVE_PARALLEL */
- /* initialize cache image related fields */
+ /* Initialize cache image related fields */
ds_entry_ptr->include_in_image = FALSE;
ds_entry_ptr->lru_rank = 0;
ds_entry_ptr->image_dirty = FALSE;
@@ -638,6 +670,10 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr,
ds_entry_ptr->prefetched = FALSE;
ds_entry_ptr->prefetch_type_id = 0;
ds_entry_ptr->age = 0;
+ ds_entry_ptr->prefetched_dirty = pf_entry_ptr->prefetched_dirty;
+#ifndef NDEBUG /* debugging field */
+ ds_entry_ptr->serialization_count = 0;
+#endif /* NDEBUG */
H5C__RESET_CACHE_ENTRY_STATS(ds_entry_ptr);
@@ -666,7 +702,7 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr,
* and H5C__FLUSH_CLEAR_ONLY_FLAG flags set.
*/
pf_entry_ptr->image_ptr = NULL;
- if ( pf_entry_ptr->is_dirty ) {
+ if(pf_entry_ptr->is_dirty) {
HDassert(pf_entry_ptr->in_slist);
flush_flags |= H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG;
} /* end if */
@@ -684,7 +720,6 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr,
H5C__INSERT_IN_INDEX(cache_ptr, ds_entry_ptr, FAIL)
HDassert(!ds_entry_ptr->in_slist);
-
if(ds_entry_ptr->is_dirty)
H5C__INSERT_ENTRY_IN_SLIST(cache_ptr, ds_entry_ptr, FAIL)
@@ -699,16 +734,13 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr,
(ds_entry_ptr->type->notify)(H5C_NOTIFY_ACTION_AFTER_LOAD, ds_entry_ptr) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry loaded into cache")
- /* restore flush dependencies with the flush dependency children of
+ /* Restore flush dependencies with the flush dependency children of
* of the prefetched entry. Note that we must protect *ds_entry_ptr
* before the call to avoid triggering sanity check failures, and
* then unprotect it afterwards.
*/
i = 0;
if(fd_children != NULL) {
- int j;
- hbool_t found;
-
H5C__UPDATE_RP_FOR_PROTECT(cache_ptr, ds_entry_ptr, FAIL)
ds_entry_ptr->is_protected = TRUE;
while(fd_children[i] != NULL) {
@@ -718,15 +750,22 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr,
HDassert((fd_children[i])->fd_parent_count > 0);
HDassert((fd_children[i])->fd_parent_addrs);
- j = 0;
- found = FALSE;
- while((j < (int)((fd_children[i])->fd_parent_count)) && (!found)) {
- if((fd_children[i])->fd_parent_addrs[j] == ds_entry_ptr->addr)
- found = TRUE;
+#ifndef NDEBUG
+ {
+ int j;
+ hbool_t found;
+
+ j = 0;
+ found = FALSE;
+ while((j < (int)((fd_children[i])->fd_parent_count)) && (!found)) {
+ if((fd_children[i])->fd_parent_addrs[j] == ds_entry_ptr->addr)
+ found = TRUE;
- j++;
- } /* end while */
- HDassert(found);
+ j++;
+ } /* end while */
+ HDassert(found);
+ }
+#endif /* NDEBUG */
if(H5C_create_flush_dependency(ds_entry_ptr, fd_children[i]) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTDEPEND, FAIL, "Can't restore child flush dependency")
@@ -735,7 +774,6 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr,
} /* end while */
H5C__UPDATE_RP_FOR_UNPROTECT(cache_ptr, ds_entry_ptr, FAIL);
-
ds_entry_ptr->is_protected = FALSE;
} /* end if ( fd_children != NULL ) */
HDassert((unsigned)i == ds_entry_ptr->fd_child_count);
@@ -828,6 +866,54 @@ H5C__free_image_entries_array(H5C_t * cache_ptr)
/*-------------------------------------------------------------------------
+ * Function: H5C_force_cache_image_load()
+ *
+ * Purpose: On rare occasions, it is necessary to run
+ * H5MF_tidy_self_referential_fsm_hack() prior to the first
+ * metadata cache access. This is a problem as if there is a
+ * cache image at the end of the file, that routine will
+ * discard it.
+ *
+ * We solve this issue by calling this function, which will
+ * load the cache image and then call
+ * H5MF_tidy_self_referential_fsm_hack() to discard it.
+ *
+ * Return: SUCCEED on success, and FAIL on failure.
+ *
+ * Programmer: John Mainzer
+ * 1/11/17
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5C_force_cache_image_load(H5F_t *f, hid_t dxpl_id)
+{
+ H5C_t *cache_ptr;
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* Sanity checks */
+ HDassert(f);
+ HDassert(f->shared);
+ cache_ptr = f->shared->cache;
+ HDassert(cache_ptr);
+ HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC);
+ HDassert(cache_ptr->load_image);
+
+ /* Load the cache image, if requested */
+ if(cache_ptr->load_image) {
+ cache_ptr->load_image = FALSE;
+ if(H5C__load_cache_image(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, FAIL, "can't load cache image")
+ } /* end if */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5C_force_cache_image_load() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5C_get_cache_image_config
*
* Purpose: Copy the current configuration for cache image generation
@@ -1235,7 +1321,7 @@ H5C__image_entry_cmp(const void *_entry1, const void *_entry2)
*-------------------------------------------------------------------------
*/
herr_t
-H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id)
+H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id, hbool_t *image_generated)
{
H5C_t * cache_ptr = NULL;
haddr_t eoa_frag_addr = HADDR_UNDEF;
@@ -1251,6 +1337,7 @@ H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id)
cache_ptr = f->shared->cache;
HDassert(cache_ptr);
HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC);
+ HDassert(image_generated);
/* If the file is opened and closed without any access to
* any group or data set, it is possible that the cache image (if
@@ -1262,6 +1349,26 @@ H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id)
HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, FAIL, "can't load cache image")
} /* end if */
+ /* Before we start to generate the cache image (if requested), verify
+ * that the superblock supports superblock extension messages, and
+ * silently cancel any request for a cache image if it does not.
+ *
+ * Ideally, we would do this when the cache image is requested,
+ * but the necessary information is not necessary available at that
+ * time -- hence this last minute check.
+ *
+ * Note that under some error conditions, the superblock will be
+ * undefined in this case as well -- if so, assume that the
+ * superblock does not support superblock extension messages.
+ */
+ if((NULL == f->shared->sblock) ||
+ (f->shared->sblock->super_vers < HDF5_SUPERBLOCK_VERSION_2)) {
+ H5C_cache_image_ctl_t default_image_ctl = H5C__DEFAULT_CACHE_IMAGE_CTL;
+
+ cache_ptr->image_ctl = default_image_ctl;
+ HDassert(!(cache_ptr->image_ctl.generate_image));
+ } /* end if */
+
/* Generate the cache image, if requested */
if(cache_ptr->image_ctl.generate_image) {
/* Create the cache image super block extension message.
@@ -1355,6 +1462,15 @@ H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id)
(hsize_t)(cache_ptr->image_data_len), &eoa_frag_addr, &eoa_frag_size)))
HGOTO_ERROR(H5E_CACHE, H5E_NOSPACE, FAIL, "can't allocate file space for metadata cache image")
+ /* Make note of the eoa after allocation of the cache image
+ * block. This value is used for sanity checking when we
+ * shutdown the self referential free space managers after
+ * we destroy the metadata cache.
+ */
+ HDassert(HADDR_UNDEF == f->shared->eoa_post_mdci_fsalloc);
+ if(HADDR_UNDEF == (f->shared->eoa_post_mdci_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)) )
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file size")
+
/* For now, drop any fragment left over from the allocation of the
* image block on the ground. A fragment should only be returned
* if the underlying file alignment is greater than 1.
@@ -1386,7 +1502,7 @@ H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id)
*/
if(cache_ptr->image_ctl.flags & H5C_CI__GEN_MDC_IMAGE_BLK)
if(H5C__write_cache_image_superblock_msg(f, dxpl_id, FALSE) < 0)
- HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "update of cache image SB mesg failed.")
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "update of cache image SB mesg failed")
/* At this point:
*
@@ -1440,10 +1556,13 @@ H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id)
*/
if(cache_ptr->image_ctl.flags & H5C_CI__GEN_MDC_IMAGE_BLK)
if(H5F_super_ext_remove_msg(f, dxpl_id, H5O_MDCI_MSG_ID) < 0)
- HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove MDC image msg from superblock ext.")
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove MDC image msg from superblock ext")
cache_ptr->image_ctl.generate_image = FALSE;
} /* end else */
+
+ /* Indicate that a cache image was generated */
+ *image_generated = TRUE;
} /* end if */
done:
@@ -1458,6 +1577,18 @@ done:
* image_ctl field of *cache_ptr. Make adjustments for
* changes in configuration as required.
*
+ * If the file is open read only, silently
+ * force the cache image configuration to its default
+ * (which disables construction of a cache image).
+ *
+ * Note that in addition to being inapplicable in the
+ * read only case, cache image is also inapplicable if
+ * the superblock does not support superblock extension
+ * messages. Unfortunately, this information need not
+ * be available at this point. Thus we check for this
+ * later, in H5C_prep_for_file_close() and cancel the
+ * cache image request if appropriate.
+ *
* Fail if the new configuration is invalid.
*
* Return: SUCCEED on success, and FAIL on failure.
@@ -1483,26 +1614,13 @@ H5C_set_cache_image_config(const H5F_t *f, H5C_t *cache_ptr,
/* Check arguments */
if((cache_ptr == NULL) || (cache_ptr->magic != H5C__H5C_T_MAGIC))
HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "Bad cache_ptr on entry")
- if(config_ptr == NULL)
- HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "NULL config_ptr on entry")
- if(config_ptr->version != H5C__CURR_CACHE_IMAGE_CTL_VER)
- HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "Unknown config version")
- /* check general configuration section of the config: */
+ /* Validate the config: */
if(H5C_validate_cache_image_config(config_ptr) < 0)
HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "invalid cache image configuration")
- if(H5F_INTENT(f) & H5F_ACC_RDWR) /* file has been opened R/W */
- cache_ptr->image_ctl = *config_ptr;
- else { /* file opened R/O -- suppress cache image silently */
- H5C_cache_image_ctl_t default_image_ctl = H5C__DEFAULT_CACHE_IMAGE_CTL;
-
- cache_ptr->image_ctl = default_image_ctl;
- HDassert(!(cache_ptr->image_ctl.generate_image));
- } /* end else */
-
#ifdef H5_HAVE_PARALLEL
- /* the collective metadata write code is not currently compatible
+ /* The collective metadata write code is not currently compatible
* with cache image. Until this is fixed, suppress cache image silently
* if there is more than one process.
* JRM -- 11/8/16
@@ -1513,6 +1631,30 @@ H5C_set_cache_image_config(const H5F_t *f, H5C_t *cache_ptr,
cache_ptr->image_ctl = default_image_ctl;
HDassert(!(cache_ptr->image_ctl.generate_image));
} /* end if */
+ else {
+#endif /* H5_HAVE_PARALLEL */
+ /* A cache image can only be generated if the file is opened read / write
+ * and the superblock supports superblock extension messages.
+ *
+ * However, the superblock version is not available at this point --
+ * hence we can only check the former requirement now. Do the latter
+ * check just before we construct the image..
+ *
+ * If the file is opened read / write, apply the supplied configuration.
+ *
+ * If it is not, set the image configuration to the default, which has
+ * the effect of silently disabling the cache image if it was requested.
+ */
+ if(H5F_INTENT(f) & H5F_ACC_RDWR)
+ cache_ptr->image_ctl = *config_ptr;
+ else {
+ H5C_cache_image_ctl_t default_image_ctl = H5C__DEFAULT_CACHE_IMAGE_CTL;
+
+ cache_ptr->image_ctl = default_image_ctl;
+ HDassert(!(cache_ptr->image_ctl.generate_image));
+ } /* end else */
+#ifdef H5_HAVE_PARALLEL
+ } /* end else */
#endif /* H5_HAVE_PARALLEL */
done:
@@ -3186,7 +3328,13 @@ H5C__reconstruct_cache_entry(const H5F_t *f, H5C_t *cache_ptr,
* this as otherwise the cache will attempt to write them on file
* close. Since the file is R/O, the metadata cache image superblock
* extension message and the cache image block will not be removed.
- * Hence no danger in this.
+ * Hence no danger in this for subsequent opens.
+ *
+ * However, if the dirty entry (marked clean for purposes of the R/O
+ * file open) is evicted and then referred to, the cache will read
+ * either invalid or obsolete data from the file. Handle this by
+ * setting the prefetched_dirty field, and hiding such entries from
+ * the eviction candidate selection algorithm.
*/
pf_entry_ptr->is_dirty = (is_dirty && file_is_rw);
@@ -3204,6 +3352,8 @@ H5C__reconstruct_cache_entry(const H5F_t *f, H5C_t *cache_ptr,
/* Decode dirty dependency child count */
UINT16DECODE(p, pf_entry_ptr->fd_dirty_child_count);
+ if(!file_is_rw)
+ pf_entry_ptr->fd_dirty_child_count = 0;
if(pf_entry_ptr->fd_dirty_child_count > pf_entry_ptr->fd_child_count)
HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "invalid dirty flush dependency child count")
@@ -3263,6 +3413,7 @@ H5C__reconstruct_cache_entry(const H5F_t *f, H5C_t *cache_ptr,
pf_entry_ptr->image_up_to_date = TRUE;
pf_entry_ptr->type = H5AC_PREFETCHED_ENTRY;
pf_entry_ptr->prefetched = TRUE;
+ pf_entry_ptr->prefetched_dirty = is_dirty && (!file_is_rw);
/* Sanity checks */
HDassert(pf_entry_ptr->size > 0 && pf_entry_ptr->size < H5C_MAX_ENTRY_SIZE);
diff --git a/src/H5Cmpio.c b/src/H5Cmpio.c
index ebb98b3..06ce714 100644
--- a/src/H5Cmpio.c
+++ b/src/H5Cmpio.c
@@ -65,6 +65,11 @@
/* Local Prototypes */
/********************/
static herr_t H5C__collective_write(H5F_t *f, hid_t dxpl_id);
+static herr_t H5C__flush_candidate_entries(H5F_t *f, hid_t dxpl_id,
+ unsigned entries_to_flush[H5C_RING_NTYPES],
+ unsigned entries_to_clear[H5C_RING_NTYPES]);
+static herr_t H5C__flush_candidates_in_ring(H5F_t *f, hid_t dxpl_id,
+ H5C_ring_t ring, unsigned entries_to_flush, unsigned entries_to_clear);
/*********************/
@@ -175,31 +180,18 @@ H5C_apply_candidate_list(H5F_t * f,
int mpi_rank,
int mpi_size)
{
- hbool_t restart_scan;
- hbool_t prev_is_dirty;
int i;
int m;
int n;
unsigned first_entry_to_flush;
unsigned last_entry_to_flush;
- unsigned entries_to_clear = 0;
- unsigned entries_to_flush = 0;
- unsigned entries_to_flush_or_clear_last = 0;
- unsigned entries_to_flush_collectively = 0;
- unsigned entries_cleared = 0;
- unsigned entries_flushed = 0;
- unsigned entries_delayed = 0;
- unsigned entries_flushed_or_cleared_last = 0;
- unsigned entries_flushed_collectively = 0;
- unsigned entries_examined = 0;
- unsigned initial_list_len;
+ unsigned total_entries_to_clear = 0;
+ unsigned total_entries_to_flush = 0;
int * candidate_assignment_table = NULL;
+ unsigned entries_to_flush[H5C_RING_NTYPES];
+ unsigned entries_to_clear[H5C_RING_NTYPES];
haddr_t addr;
- H5C_cache_entry_t * clear_ptr = NULL;
- H5C_cache_entry_t * next_ptr = NULL;
H5C_cache_entry_t * entry_ptr = NULL;
- H5C_cache_entry_t * flush_ptr = NULL;
- H5C_cache_entry_t * delayed_ptr = NULL;
#if H5C_DO_SANITY_CHECKS
haddr_t last_addr;
#endif /* H5C_DO_SANITY_CHECKS */
@@ -220,6 +212,10 @@ H5C_apply_candidate_list(H5F_t * f,
HDassert(0 <= mpi_rank);
HDassert(mpi_rank < mpi_size);
+ /* Initialize the entries_to_flush and entries_to_clear arrays */
+ HDmemset(entries_to_flush, 0, sizeof(entries_to_flush));
+ HDmemset(entries_to_clear, 0, sizeof(entries_to_clear));
+
#if H5C_APPLY_CANDIDATE_LIST__DEBUG
HDfprintf(stdout, "%s:%d: setting up candidate assignment table.\n", FUNC, mpi_rank);
@@ -330,17 +326,26 @@ H5C_apply_candidate_list(H5F_t * f,
*/
HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Listed entry is protected?!?!?")
+ /* Sanity checks */
+ HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC);
+ HDassert(entry_ptr->ring >= H5C_RING_USER);
+ HDassert(entry_ptr->ring <= H5C_RING_SB);
+ HDassert(!entry_ptr->flush_immediately);
+ HDassert(!entry_ptr->clear_on_unprotect);
+
/* Determine whether the entry is to be cleared or flushed,
* and mark it accordingly. We will scan the protected and
* pinned list shortly, and clear or flush according to these
* markings.
*/
if(u >= first_entry_to_flush && u <= last_entry_to_flush) {
- entries_to_flush++;
+ total_entries_to_flush++;
+ entries_to_flush[entry_ptr->ring]++;
entry_ptr->flush_immediately = TRUE;
} /* end if */
else {
- entries_to_clear++;
+ total_entries_to_clear++;
+ entries_to_clear[entry_ptr->ring]++;
entry_ptr->clear_on_unprotect = TRUE;
} /* end else */
@@ -356,372 +361,36 @@ H5C_apply_candidate_list(H5F_t * f,
} /* end if */
} /* end for */
+#if H5C_DO_SANITY_CHECKS
+ m = 0;
+ n = 0;
+ for(i = 0; i < H5C_RING_NTYPES; i++) {
+ m += (int)entries_to_flush[i];
+ n += (int)entries_to_clear[i];
+ } /* end if */
+
+ HDassert((unsigned)m == total_entries_to_flush);
+ HDassert((unsigned)n == total_entries_to_clear);
+#endif /* H5C_DO_SANITY_CHECKS */
+
#if H5C_APPLY_CANDIDATE_LIST__DEBUG
HDfprintf(stdout, "%s:%d: num candidates/to clear/to flush = %u/%u/%u.\n",
- FUNC, mpi_rank, num_candidates, entries_to_clear,
- entries_to_flush);
+ FUNC, mpi_rank, num_candidates, total_entries_to_clear,
+ total_entries_to_flush);
#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
/* We have now marked all the entries on the candidate list for
* either flush or clear -- now scan the LRU and the pinned list
- * for these entries and do the deed.
+ * for these entries and do the deed. Do this via a call to
+ * H5C__flush_candidate_entries().
*
* Note that we are doing things in this round about manner so as
* to preserve the order of the LRU list to the best of our ability.
* If we don't do this, my experiments indicate that we will have a
* noticably poorer hit ratio as a result.
*/
-
-#if H5C_APPLY_CANDIDATE_LIST__DEBUG
- HDfprintf(stdout, "%s:%d: scanning LRU list. len = %d.\n", FUNC, mpi_rank,
- (int)(cache_ptr->LRU_list_len));
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
- /* ===================================================================== *
- * Now scan the LRU and PEL lists, flushing or clearing entries as
- * needed.
- *
- * The flush_me_last flag may dictate how or
- * when some entries can be flushed, and should be addressed here.
- * However, in their initial implementation, these flags only apply to the
- * superblock, so there's only a relatively small change to this function
- * to account for this one case where they come into play. If these flags
- * are ever expanded upon, this function and the following flushing steps
- * should be reworked to account for additional cases.
- * ===================================================================== */
-
- HDassert(entries_to_flush >= 0);
-
- restart_scan = FALSE;
- entries_examined = 0;
- initial_list_len = cache_ptr->LRU_list_len;
- entry_ptr = cache_ptr->LRU_tail_ptr;
-
- /* Examine each entry in the LRU list */
- while ( ( entry_ptr != NULL )
- &&
- ( entries_examined <= (entries_to_flush + 1) * initial_list_len )
- &&
- ( (entries_cleared + entries_flushed) < num_candidates ) ) {
-
- if ( entry_ptr->prev != NULL )
- prev_is_dirty = entry_ptr->prev->is_dirty;
-
- /* If this process needs to clear this entry. */
- if(entry_ptr->clear_on_unprotect) {
-
- HDassert(entry_ptr->is_dirty);
-
- next_ptr = entry_ptr->next;
- entry_ptr->clear_on_unprotect = FALSE;
- clear_ptr = entry_ptr;
- entry_ptr = entry_ptr->prev;
- entries_cleared++;
-
-#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 )
- HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank,
- (long long)clear_ptr->addr);
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
- /* reset entries_removed_counter and
- * last_entry_removed_ptr prior to the call to
- * H5C__flush_single_entry() so that we can spot
- * unexpected removals of entries from the cache,
- * and set the restart_scan flag if proceeding
- * would be likely to cause us to scan an entry
- * that is no longer in the cache.
- *
- * Note that as of this writing (April 2015) this
- * case cannot occur in the parallel case. However
- * Quincey is making noises about changing this, hence
- * the insertion of this test.
- *
- * Note also that there is no test code to verify
- * that this code actually works (although similar code
- * in the serial version exists and is tested).
- *
- * Implementing a test will likely require implementing
- * flush op like facilities in the parallel tests. At
- * a guess this will not be terribly painful, but it
- * will take a bit of time.
- */
- cache_ptr->entries_removed_counter = 0;
- cache_ptr->last_entry_removed_ptr = NULL;
-
- if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0)
- HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't clear entry.")
-
- if((cache_ptr->entries_removed_counter > 1) ||
- (cache_ptr->last_entry_removed_ptr == entry_ptr))
- restart_scan = TRUE;
- } /* end if */
-
- /* Else, if this process needs to flush this entry. */
- else if (entry_ptr->flush_immediately) {
-
- HDassert(entry_ptr->is_dirty);
-
- next_ptr = entry_ptr->next;
- entry_ptr->flush_immediately = FALSE;
- flush_ptr = entry_ptr;
- entry_ptr = entry_ptr->prev;
- entries_flushed++;
-
-#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 )
- HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank,
- (long long)flush_ptr->addr);
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
- /* reset entries_removed_counter and
- * last_entry_removed_ptr prior to the call to
- * H5C__flush_single_entry() so that we can spot
- * unexpected removals of entries from the cache,
- * and set the restart_scan flag if proceeding
- * would be likely to cause us to scan an entry
- * that is no longer in the cache.
- *
- * Note that as of this writing (April 2015) this
- * case cannot occur in the parallel case. However
- * Quincey is making noises about changing this, hence
- * the insertion of this test.
- *
- * Note also that there is no test code to verify
- * that this code actually works (although similar code
- * in the serial version exists and is tested).
- *
- * Implementing a test will likely require implementing
- * flush op like facilities in the parallel tests. At
- * a guess this will not be terribly painful, but it
- * will take a bit of time.
- */
- cache_ptr->entries_removed_counter = 0;
- cache_ptr->last_entry_removed_ptr = NULL;
-
- /* Add this entry to the list of entries to collectively write */
- if(H5C__flush_single_entry(f, dxpl_id, flush_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0)
- HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush entry.")
-
- if((cache_ptr->entries_removed_counter > 1) ||
- (cache_ptr->last_entry_removed_ptr == entry_ptr))
- restart_scan = TRUE;
- } /* end else-if */
-
- /* Otherwise, no action to be taken on this entry. Grab the next. */
- else {
- entry_ptr = entry_ptr->prev;
-
- if ( entry_ptr != NULL )
- next_ptr = entry_ptr->next;
-
- } /* end else */
-
- if ( ( entry_ptr != NULL )
- &&
- ( ( restart_scan )
- ||
- ( entry_ptr->is_dirty != prev_is_dirty )
- ||
- ( entry_ptr->next != next_ptr )
- ||
- ( entry_ptr->is_protected )
- ||
- ( entry_ptr->is_pinned )
- )
- ) {
-
- /* something has happened to the LRU -- start over
- * from the tail.
- *
- * Recall that this code should be un-reachable at present,
- * as all the operations by entries on flush that could cause
- * it to be reachable are disallowed in the parallel case at
- * present. Hence the following assertion which should be
- * removed if the above changes.
- */
-
- HDassert( ! restart_scan );
- HDassert( entry_ptr->is_dirty == prev_is_dirty );
- HDassert( entry_ptr->next == next_ptr );
- HDassert( ! entry_ptr->is_protected );
- HDassert( ! entry_ptr->is_pinned );
-
- HDassert(FALSE); /* see comment above */
-
- restart_scan = FALSE;
- entry_ptr = cache_ptr->LRU_tail_ptr;
-/*
- H5C__UPDATE_STATS_FOR_LRU_SCAN_RESTART(cache_ptr)
-*/
- }
-
- entries_examined++;
- } /* end while */
-
-#if H5C_APPLY_CANDIDATE_LIST__DEBUG
- HDfprintf(stdout, "%s:%d: entries examined/cleared/flushed = %u/%u/%u.\n",
- FUNC, mpi_rank, entries_examined,
- entries_cleared, entries_flushed);
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
- /* It is also possible that some of the cleared entries are on the
- * pinned list. Must scan that also.
- *
- * WARNING:
- *
- * As we now allow unpinning, and removal of other entries as a side
- * effect of flushing an entry, it is possible that the next entry
- * in a PEL scan could either be no longer pinned, or no longer in
- * the cache by the time we get to it.
- *
- * At present, this is not possible in this case, as we disallow such
- * operations in the parallel version of the library. However, Quincey
- * has been making noises about relaxing this. If and when he does,
- * we have a potential problem here.
- *
- * The same issue exists in the serial cache, and there are tests
- * to detect this problem when it occurs, and adjust to it. As seen
- * above in the LRU scan, I have ported such tests to the parallel
- * code where a close cognate exists in the serial code.
- *
- * I haven't done so here, as there are no PEL scans where the problem
- * can occur in the serial code. Needless to say, this will have to
- * be repaired if the constraints on pre_serialize and serialize
- * callbacks are relaxed in the parallel version of the metadata cache.
- *
- * JRM -- 4/1/15
- */
-
-#if H5C_APPLY_CANDIDATE_LIST__DEBUG
- HDfprintf(stdout, "%s:%d: scanning pinned entry list. len = %d\n",
- FUNC, mpi_rank, (int)(cache_ptr->pel_len));
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
- entry_ptr = cache_ptr->pel_head_ptr;
- while((entry_ptr != NULL) &&
- ((entries_cleared + entries_flushed + entries_delayed)
- < num_candidates)) {
-
- /* If entry is marked for flush or for clear */
- if((entry_ptr->clear_on_unprotect||entry_ptr->flush_immediately)) {
-
- /* If this entry needs to be flushed last */
- if (entry_ptr->flush_me_last) {
-
- /* At this time, only the superblock supports being
- flushed last. Conveniently, it also happens to be the only
- entry that supports being flushed collectively, as well. Also
- conveniently, it's always pinned, so we only need to check
- for it while scanning the PEL here. Finally, it's never
- included in a candidate list that excludes other dirty
- entries in a cache, so we can handle this relatively simple
- case here.
-
- For now, this function asserts this and saves the entry
- to flush it after scanning the rest of the PEL list.
-
- If there are ever more entries that either need to be
- flushed last and/or flushed collectively, this whole routine
- will need to be reworked to handle all additional cases. As
- it is the simple case of a single pinned entry needing
- flushed last and collectively is just a minor addition to
- this routine, but signficantly buffing up the usage of
- flush_me_last will require a more
- intense rework of this function and potentially the function
- of candidate lists as a whole. */
-
- entries_to_flush_or_clear_last++;
- entries_to_flush_collectively++;
- HDassert(entries_to_flush_or_clear_last == 1);
- HDassert(entries_to_flush_collectively == 1);
-
- /* Delay the entry. It will be flushed later. */
- delayed_ptr = entry_ptr;
- entries_delayed++;
- HDassert(entries_delayed == 1);
-
- } /* end if */
-
- /* Else, this process needs to clear this entry. */
- else if (entry_ptr->clear_on_unprotect) {
- HDassert(!entry_ptr->flush_immediately);
- entry_ptr->clear_on_unprotect = FALSE;
- clear_ptr = entry_ptr;
- entry_ptr = entry_ptr->next;
- entries_cleared++;
-
-#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 )
- HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank,
- (long long)clear_ptr->addr);
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
- if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0)
- HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't clear entry.")
- } /* end else-if */
-
- /* Else, if this process needs to independently flush this entry. */
- else if (entry_ptr->flush_immediately) {
- entry_ptr->flush_immediately = FALSE;
- flush_ptr = entry_ptr;
- entry_ptr = entry_ptr->next;
- entries_flushed++;
-
-#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 )
- HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank,
- (long long)flush_ptr->addr);
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
- /* Add this entry to the list of entries to collectively write */
- if(H5C__flush_single_entry(f, dxpl_id, flush_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0)
- HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't clear entry.")
- } /* end else-if */
- } /* end if */
-
- /* Otherwise, this entry is not marked for flush or clear. Grab the next. */
- else {
- entry_ptr = entry_ptr->next;
- } /* end else */
-
- } /* end while */
-
-#if H5C_APPLY_CANDIDATE_LIST__DEBUG
- HDfprintf(stdout,
- "%s:%d: pel entries examined/cleared/flushed = %u/%u/%u.\n",
- FUNC, mpi_rank, entries_examined,
- entries_cleared, entries_flushed);
- HDfprintf(stdout, "%s:%d: done.\n", FUNC, mpi_rank);
-
- HDfsync(stdout);
-#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */
-
- /* ====================================================================== *
- * Now, handle all delayed entries. *
- * *
- * This can *only* be the superblock at this time, so it's relatively *
- * easy to deal with. We're collectively flushing the entry saved from *
- * above. This will need to be handled differently if there are ever more *
- * than one entry needing this special treatment.) *
- * ====================================================================== */
-
- if (delayed_ptr) {
-
- if (delayed_ptr->clear_on_unprotect) {
- if(H5C__flush_single_entry(f, dxpl_id, delayed_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG) < 0)
- HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush entry.")
-
- entry_ptr->clear_on_unprotect = FALSE;
- entries_cleared++;
- } else if (delayed_ptr->flush_immediately) {
- /* Add this entry to the list of entries to collectively write */
- if(H5C__flush_single_entry(f, dxpl_id, delayed_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0)
- HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush entry collectively.")
-
- entry_ptr->flush_immediately = FALSE;
- entries_flushed++;
- } /* end if */
-
- entries_flushed_collectively++;
- entries_flushed_or_cleared_last++;
- } /* end if */
+ if(H5C__flush_candidate_entries(f, dxpl_id, entries_to_flush, entries_to_clear) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "flush candidates failed")
/* If we've deferred writing to do it collectively, take care of that now */
if(f->coll_md_write) {
@@ -733,21 +402,6 @@ H5C_apply_candidate_list(H5F_t * f,
HGOTO_ERROR(H5E_CACHE, H5E_WRITEERROR, FAIL, "can't write metadata collectively")
} /* end if */
- /* ====================================================================== *
- * Finished flushing everything. *
- * ====================================================================== */
-
- HDassert((entries_flushed == entries_to_flush));
- HDassert((entries_cleared == entries_to_clear));
- HDassert((entries_flushed_or_cleared_last == entries_to_flush_or_clear_last));
- HDassert((entries_flushed_collectively == entries_to_flush_collectively));
-
- if((entries_flushed != entries_to_flush) ||
- (entries_cleared != entries_to_clear) ||
- (entries_flushed_or_cleared_last != entries_to_flush_or_clear_last) ||
- (entries_flushed_collectively != entries_to_flush_collectively))
- HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry count mismatch")
-
done:
if(candidate_assignment_table != NULL)
candidate_assignment_table = (int *)H5MM_xfree((void *)candidate_assignment_table);
@@ -998,11 +652,13 @@ H5C_mark_entries_as_clean(H5F_t * f,
{
H5C_t * cache_ptr;
unsigned entries_cleared;
+ unsigned pinned_entries_cleared;
+ hbool_t progress;
unsigned entries_examined;
unsigned initial_list_len;
haddr_t addr;
-#if H5C_DO_SANITY_CHECKS
unsigned pinned_entries_marked = 0;
+#if H5C_DO_SANITY_CHECKS
unsigned protected_entries_marked = 0;
unsigned other_entries_marked = 0;
haddr_t last_addr;
@@ -1086,19 +742,13 @@ H5C_mark_entries_as_clean(H5F_t * f,
} /* end if */
entry_ptr->clear_on_unprotect = TRUE;
+ if(entry_ptr->is_pinned)
+ pinned_entries_marked++;
#if H5C_DO_SANITY_CHECKS
- if ( entry_ptr->is_protected ) {
-
+ else if(entry_ptr->is_protected)
protected_entries_marked++;
-
- } else if ( entry_ptr->is_pinned ) {
-
- pinned_entries_marked++;
-
- } else {
-
+ else
other_entries_marked++;
- }
#endif /* H5C_DO_SANITY_CHECKS */
}
}
@@ -1140,7 +790,8 @@ H5C_mark_entries_as_clean(H5F_t * f,
entry_ptr = entry_ptr->prev;
entries_cleared++;
- if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0)
+ if(H5C__flush_single_entry(f, dxpl_id, clear_ptr,
+ (H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG | H5C__UPDATE_PAGE_BUFFER_FLAG)) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't clear entry")
} /* end if */
else
@@ -1155,19 +806,27 @@ H5C_mark_entries_as_clean(H5F_t * f,
/* It is also possible that some of the cleared entries are on the
* pinned list. Must scan that also.
*/
- entry_ptr = cache_ptr->pel_head_ptr;
- while(entry_ptr != NULL) {
- if(entry_ptr->clear_on_unprotect) {
- entry_ptr->clear_on_unprotect = FALSE;
- clear_ptr = entry_ptr;
- entry_ptr = entry_ptr->next;
- entries_cleared++;
+ pinned_entries_cleared = 0;
+ progress = TRUE;
+ while((pinned_entries_cleared < pinned_entries_marked) && progress) {
+ progress = FALSE;
+ entry_ptr = cache_ptr->pel_head_ptr;
+ while(entry_ptr != NULL) {
+ if(entry_ptr->clear_on_unprotect && entry_ptr->flush_dep_ndirty_children == 0) {
+ entry_ptr->clear_on_unprotect = FALSE;
+ clear_ptr = entry_ptr;
+ entry_ptr = entry_ptr->next;
+ entries_cleared++;
+ pinned_entries_cleared++;
+ progress = TRUE;
- if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0)
- HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't clear entry")
- } /* end if */
- else
- entry_ptr = entry_ptr->next;
+ if(H5C__flush_single_entry(f, dxpl_id, clear_ptr,
+ (H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG | H5C__UPDATE_PAGE_BUFFER_FLAG)) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't clear entry")
+ } /* end if */
+ else
+ entry_ptr = entry_ptr->next;
+ } /* end while */
} /* end while */
#if H5C_DO_SANITY_CHECKS
@@ -1297,7 +956,6 @@ H5C__collective_write(H5F_t *f, hid_t dxpl_id)
/* Get number of entries in collective write list */
count = (int)H5SL_count(cache_ptr->coll_write_list);
-
if(count > 0) {
H5FD_mpio_xfer_t xfer_mode = H5FD_MPIO_COLLECTIVE;
H5SL_node_t *node;
@@ -1410,4 +1068,511 @@ done:
FUNC_LEAVE_NOAPI(ret_value);
} /* end H5C__collective_write() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5C__flush_candidate_entries
+ *
+ * Purpose: Flush or clear (as indicated) the candidate entries that
+ * have been marked in the metadata cache. In so doing,
+ * observe rings and flush dependencies.
+ *
+ * Note that this function presumes that:
+ *
+ * 1) no candidate entries are protected,
+ *
+ * 2) all candidate entries are dirty, and
+ *
+ * 3) if a candidate entry has a dirty flush dependency
+ * child, that child is also a candidate entry.
+ *
+ * The function will fail if any of these preconditions are
+ * not met.
+ *
+ * Candidate entries are marked by setting either the
+ * flush_immediately or the clear_on_unprotect flags in the
+ * cache entry (but not both). Entries marked flush_immediately
+ * will be flushed, those marked clear_on_unprotect will be
+ * cleared.
+ *
+ * Note that this function is a modified version of
+ * H5C_flush_cache() -- any changes there may need to be
+ * reflected here and vise versa.
+ *
+ * Return: Non-negative on success/Negative on failure.
+ *
+ * Programmer: John Mainzer
+ * 2/10/17
+ *
+ * Changes: None.
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5C__flush_candidate_entries(H5F_t *f, hid_t dxpl_id,
+ unsigned entries_to_flush[H5C_RING_NTYPES],
+ unsigned entries_to_clear[H5C_RING_NTYPES])
+{
+#if H5C_DO_SANITY_CHECKS
+ int i;
+ uint32_t index_len = 0;
+ size_t index_size = (size_t)0;
+ size_t clean_index_size = (size_t)0;
+ size_t dirty_index_size = (size_t)0;
+ size_t slist_size = (size_t)0;
+ uint32_t slist_len = 0;
+#endif /* H5C_DO_SANITY_CHECKS */
+ H5C_ring_t ring;
+ H5C_t * cache_ptr;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ HDassert(f);
+ HDassert(f->shared);
+
+ cache_ptr = f->shared->cache;
+
+ HDassert(cache_ptr);
+ HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC);
+ HDassert(cache_ptr->slist_ptr);
+
+ HDassert(entries_to_flush[H5C_RING_UNDEFINED] == 0);
+ HDassert(entries_to_clear[H5C_RING_UNDEFINED] == 0);
+
+#if H5C_DO_SANITY_CHECKS
+ HDassert(cache_ptr->index_ring_len[H5C_RING_UNDEFINED] == 0);
+ HDassert(cache_ptr->index_ring_size[H5C_RING_UNDEFINED] == (size_t)0);
+ HDassert(cache_ptr->clean_index_ring_size[H5C_RING_UNDEFINED] == (size_t)0);
+ HDassert(cache_ptr->dirty_index_ring_size[H5C_RING_UNDEFINED] == (size_t)0);
+ HDassert(cache_ptr->slist_ring_len[H5C_RING_UNDEFINED] == 0);
+ HDassert(cache_ptr->slist_ring_size[H5C_RING_UNDEFINED] == (size_t)0);
+
+ for(i = H5C_RING_USER; i < H5C_RING_NTYPES; i++) {
+ index_len += cache_ptr->index_ring_len[i];
+ index_size += cache_ptr->index_ring_size[i];
+ clean_index_size += cache_ptr->clean_index_ring_size[i];
+ dirty_index_size += cache_ptr->dirty_index_ring_size[i];
+
+ slist_len += cache_ptr->slist_ring_len[i];
+ slist_size += cache_ptr->slist_ring_size[i];
+ } /* end for */
+
+ HDassert(cache_ptr->index_len == index_len);
+ HDassert(cache_ptr->index_size == index_size);
+ HDassert(cache_ptr->clean_index_size == clean_index_size);
+ HDassert(cache_ptr->dirty_index_size == dirty_index_size);
+ HDassert(cache_ptr->slist_len == slist_len);
+ HDassert(cache_ptr->slist_size == slist_size);
+#endif /* H5C_DO_SANITY_CHECKS */
+
+#if H5C_DO_EXTREME_SANITY_CHECKS
+ if(H5C_validate_protected_entry_list(cache_ptr) < 0
+ || H5C_validate_pinned_entry_list(cache_ptr) < 0
+ || H5C_validate_lru_list(cache_ptr) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry")
+#endif /* H5C_DO_EXTREME_SANITY_CHECKS */
+
+ cache_ptr->flush_in_progress = TRUE;
+
+ /* flush each ring, starting from the outermost ring and
+ * working inward.
+ */
+ ring = H5C_RING_USER;
+ while(ring < H5C_RING_NTYPES) {
+ if(H5C__flush_candidates_in_ring(f, dxpl_id, ring, entries_to_flush[ring], entries_to_clear[ring]) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "flush candidates in ring failed")
+
+ ring++;
+ } /* end while */
+
+done:
+ cache_ptr->flush_in_progress = FALSE;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5C__flush_candidate_entries() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5C__flush_candidates_in_ring
+ *
+ * Purpose: Flush or clear (as indicated) the candidate entries
+ * contained in the specified cache and ring. All candidate
+ * entries in rings outside the specified ring must have been
+ * flushed (or cleared) on entry.
+ *
+ * Note that this function presumes that:
+ *
+ * 1) no candidate entries are protected,
+ *
+ * 2) all candidate entries are dirty, and
+ *
+ * 3) if a candidate entry has a dirty flush dependency
+ * child, that child is also a candidate entry.
+ *
+ * The function will fail if any of these preconditions are
+ * not met.
+ *
+ * Candidate entries are marked by setting either the
+ * flush_immediately or the clear_on_unprotect flags in the
+ * cache entry (but not both). Entries marked flush_immediately
+ * will be flushed, those marked clear_on_unprotect will be
+ * cleared.
+ *
+ * Candidate entries residing in the LRU must be flushed
+ * (or cleared) in LRU order to avoid performance issues.
+ *
+ * Return: Non-negative on success/Negative on failure.
+ *
+ * Programmer: John Mainzer
+ * 2/10/17
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5C__flush_candidates_in_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring,
+ unsigned entries_to_flush, unsigned entries_to_clear)
+{
+ H5C_t * cache_ptr;
+ hbool_t progress;
+ hbool_t restart_scan = FALSE;
+ unsigned entries_flushed = 0;
+ unsigned entries_cleared = 0;
+#if H5C_DO_SANITY_CHECKS
+ unsigned init_index_len;
+#endif /* H5C_DO_SANITY_CHECKS */
+ unsigned clear_flags = H5C__FLUSH_CLEAR_ONLY_FLAG |
+ H5C__GENERATE_IMAGE_FLAG |
+ H5C__UPDATE_PAGE_BUFFER_FLAG;
+ unsigned flush_flags = H5C__NO_FLAGS_SET;
+ unsigned op_flags;
+ H5C_cache_entry_t *op_ptr;
+ H5C_cache_entry_t *entry_ptr;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_STATIC
+
+ /* Sanity checks */
+ HDassert(f);
+ HDassert(f->shared);
+ cache_ptr = f->shared->cache;
+ HDassert(cache_ptr);
+ HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC);
+ HDassert(cache_ptr->slist_ptr);
+ HDassert(ring > H5C_RING_UNDEFINED);
+ HDassert(ring < H5C_RING_NTYPES);
+
+#if H5C_DO_EXTREME_SANITY_CHECKS
+ if((H5C_validate_protected_entry_list(cache_ptr) < 0) ||
+ (H5C_validate_pinned_entry_list(cache_ptr) < 0) ||
+ (H5C_validate_lru_list(cache_ptr) < 0))
+ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry")
+#endif /* H5C_DO_EXTREME_SANITY_CHECKS */
+
+#if H5C_DO_SANITY_CHECKS
+ /* index len should not change */
+ init_index_len = cache_ptr->index_len;
+#endif /* H5C_DO_SANITY_CHECKS */
+
+ /* Examine entries in the LRU list, and flush or clear all entries
+ * so marked in the target ring.
+ *
+ * With the current implementation of flush dependencies, no entry
+ * in the LRU can have flush dependency children -- thus one pass
+ * through the LRU will be sufficient.
+ *
+ * It is possible that this will change -- hence the assertion.
+ */
+ restart_scan = FALSE;
+ entry_ptr = cache_ptr->LRU_tail_ptr;
+ while(((entries_flushed < entries_to_flush) || (entries_cleared < entries_to_clear))
+ && (entry_ptr != NULL)) {
+ hbool_t prev_is_dirty = FALSE;
+ H5C_cache_entry_t *next_ptr;
+
+ /* Entries in the LRU must not have flush dependency children */
+ HDassert(entry_ptr->flush_dep_nchildren == 0);
+
+ /* Remember dirty state of entry to advance to */
+ if(entry_ptr->prev != NULL)
+ prev_is_dirty = entry_ptr->prev->is_dirty;
+
+ /* If the entry is in the ring */
+ if(entry_ptr->ring == ring) {
+ /* If this process needs to clear this entry. */
+ if(entry_ptr->clear_on_unprotect) {
+ HDassert(entry_ptr->is_dirty);
+
+ /* Set entry and flags for operation */
+ op_ptr = entry_ptr;
+ op_flags = clear_flags;
+
+ /* Set next entry appropriately */
+ next_ptr = entry_ptr->next;
+
+ /* Reset entry flag */
+ entry_ptr->clear_on_unprotect = FALSE;
+ entries_cleared++;
+ } /* end if */
+ else if(entry_ptr->flush_immediately) {
+ HDassert(entry_ptr->is_dirty);
+
+ /* Set entry and flags for operation */
+ op_ptr = entry_ptr;
+ op_flags = flush_flags;
+
+ /* Set next entry appropriately */
+ next_ptr = entry_ptr->next;
+
+ /* Reset entry flag */
+ entry_ptr->flush_immediately = FALSE;
+ entries_flushed++;
+ } /* end else-if */
+ else {
+ /* No operation for this entry */
+ op_ptr = NULL;
+
+ /* Set next entry appropriately */
+ next_ptr = entry_ptr;
+ } /* end else */
+
+ /* Advance to next entry */
+ entry_ptr = entry_ptr->prev;
+
+ /* Check for operation */
+ if(op_ptr) {
+ /* reset entries_removed_counter and
+ * last_entry_removed_ptr prior to the call to
+ * H5C__flush_single_entry() so that we can spot
+ * unexpected removals of entries from the cache,
+ * and set the restart_scan flag if proceeding
+ * would be likely to cause us to scan an entry
+ * that is no longer in the cache.
+ *
+ * Note that as of this writing, this
+ * case cannot occur in the parallel case.
+ *
+ * Note also that there is no test code to verify
+ * that this code actually works (although similar code
+ * in the serial version exists and is tested).
+ */
+ cache_ptr->entries_removed_counter = 0;
+ cache_ptr->last_entry_removed_ptr = NULL;
+
+ if(H5C__flush_single_entry(f, dxpl_id, op_ptr, op_flags) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't flush entry")
+
+ if(cache_ptr->entries_removed_counter != 0
+ || cache_ptr->last_entry_removed_ptr != NULL)
+ restart_scan = TRUE;
+ } /* end if */
+ } /* end if */
+ else {
+ /* Remember "next" pointer (after advancing entries) */
+ next_ptr = entry_ptr;
+
+ /* Advance to next entry */
+ entry_ptr = entry_ptr->prev;
+ } /* end else */
+
+ /* Check for restarts, etc. */
+ if((entry_ptr != NULL) &&
+ (restart_scan || (entry_ptr->is_dirty != prev_is_dirty)
+ || (entry_ptr->next != next_ptr) || entry_ptr->is_protected
+ || entry_ptr->is_pinned)) {
+
+ /* Something has happened to the LRU -- start over
+ * from the tail.
+ *
+ * Recall that this code should be un-reachable at present,
+ * as all the operations by entries on flush that could cause
+ * it to be reachable are disallowed in the parallel case at
+ * present. Hence the following assertion which should be
+ * removed if the above changes.
+ */
+ HDassert(!restart_scan);
+ HDassert(entry_ptr->is_dirty == prev_is_dirty);
+ HDassert(entry_ptr->next == next_ptr);
+ HDassert(!entry_ptr->is_protected);
+ HDassert(!entry_ptr->is_pinned);
+
+ HDassert(FALSE); /* see comment above */
+
+ restart_scan = FALSE;
+ entry_ptr = cache_ptr->LRU_tail_ptr;
+
+ H5C__UPDATE_STATS_FOR_LRU_SCAN_RESTART(cache_ptr)
+ } /* end if */
+ } /* end while */
+
+ /* It is also possible that some of the cleared entries are on the
+ * pinned list. Must scan that also.
+ *
+ * Observe that in the case of the pinned entry list, most of the
+ * entries will have flush dependency children. As entries with
+ * flush dependency children may not be flushed until all of their
+ * children are clean, multiple passes throguh the pinned entry list
+ * may be required.
+ *
+ * WARNING:
+ *
+ * As we now allow unpinning, and removal of other entries as a side
+ * effect of flushing an entry, it is possible that the next entry
+ * in a PEL scan could either be no longer pinned, or no longer in
+ * the cache by the time we get to it.
+ *
+ * At present, this should not be possible in this case, as we disallow
+ * such operations in the parallel version of the library. However,
+ * this may change, and to that end, I have included code to detect
+ * such changes and cause this function to fail if they are detected.
+ */
+ progress = TRUE;
+ while(progress && ((entries_flushed < entries_to_flush) || (entries_cleared < entries_to_clear))) {
+ progress = FALSE;
+ entry_ptr = cache_ptr->pel_head_ptr;
+ while((entry_ptr != NULL) &&
+ ((entries_flushed < entries_to_flush) || (entries_cleared < entries_to_clear))) {
+ H5C_cache_entry_t *prev_ptr;
+ hbool_t next_is_dirty = FALSE;
+
+ HDassert(entry_ptr->is_pinned);
+
+ /* Remember dirty state of entry to advance to */
+ if(entry_ptr->next != NULL)
+ next_is_dirty = entry_ptr->next->is_dirty;
+
+ if(entry_ptr->ring == ring && entry_ptr->flush_dep_ndirty_children == 0) {
+ if(entry_ptr->clear_on_unprotect) {
+ HDassert(entry_ptr->is_dirty);
+
+ /* Set entry and flags for operation */
+ op_ptr = entry_ptr;
+ op_flags = clear_flags;
+
+ /* Reset entry flag */
+ entry_ptr->clear_on_unprotect = FALSE;
+ entries_cleared++;
+ progress = TRUE;
+ } /* end if */
+ else if(entry_ptr->flush_immediately) {
+ HDassert(entry_ptr->is_dirty);
+
+ /* Set entry and flags for operation */
+ op_ptr = entry_ptr;
+ op_flags = flush_flags;
+
+ /* Reset entry flag */
+ entry_ptr->flush_immediately = FALSE;
+ entries_flushed++;
+ progress = TRUE;
+ } /* end else-if */
+ else
+ /* No operation for this entry */
+ op_ptr = NULL;
+
+ /* Check for operation */
+ if(op_ptr) {
+ /* reset entries_removed_counter and
+ * last_entry_removed_ptr prior to the call to
+ * H5C__flush_single_entry() so that we can spot
+ * unexpected removals of entries from the cache,
+ * and set the restart_scan flag if proceeding
+ * would be likely to cause us to scan an entry
+ * that is no longer in the cache.
+ *
+ * Note that as of this writing, this
+ * case cannot occur in the parallel case.
+ *
+ * Note also that there is no test code to verify
+ * that this code actually works (although similar code
+ * in the serial version exists and is tested).
+ */
+ cache_ptr->entries_removed_counter = 0;
+ cache_ptr->last_entry_removed_ptr = NULL;
+
+ /* Add this entry to the list of entries to collectively write
+ *
+ * This comment is misleading -- the entry will be added to the
+ * collective write list only if said list exists.
+ *
+ * JRM -- 2/9/17
+ */
+ if(H5C__flush_single_entry(f, dxpl_id, op_ptr, op_flags) < 0)
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't flush entry")
+
+ if(cache_ptr->entries_removed_counter != 0
+ || cache_ptr->last_entry_removed_ptr != NULL)
+ restart_scan = TRUE;
+ } /* end if */
+ } /* end if */
+
+ /* Remember "previous" pointer (after advancing entries) */
+ prev_ptr = entry_ptr;
+
+ /* Advance to next entry */
+ entry_ptr = entry_ptr->next;
+
+ /* Check for restarts, etc. */
+ if((entry_ptr != NULL) &&
+ (restart_scan || (entry_ptr->is_dirty != next_is_dirty)
+ || (entry_ptr->prev != prev_ptr) || entry_ptr->is_protected
+ || !entry_ptr->is_pinned)) {
+ /* Something has happened to the pinned entry list -- start
+ * over from the head.
+ *
+ * Recall that this code should be un-reachable at present,
+ * as all the operations by entries on flush that could cause
+ * it to be reachable are disallowed in the parallel case at
+ * present. Hence the following assertion which should be
+ * removed if the above changes.
+ */
+
+ HDassert(!restart_scan);
+ HDassert(entry_ptr->is_dirty == next_is_dirty);
+ HDassert(entry_ptr->prev == prev_ptr);
+ HDassert(!entry_ptr->is_protected);
+ HDassert(entry_ptr->is_pinned);
+
+ HDassert(FALSE); /* see comment above */
+
+ restart_scan = FALSE;
+
+ entry_ptr = cache_ptr->pel_head_ptr;
+
+ /* we don't keeps stats for pinned entry list scan
+ * restarts. If this code ever becomes reachable,
+ * define the necessary field, and implement the
+ * the following macro:
+ *
+ * H5C__UPDATE_STATS_FOR_PEL_SCAN_RESTART(cache_ptr)
+ */
+ } /* end if */
+ } /* end while ( ( entry_ptr != NULL ) &&
+ * ( ( entries_flushed > entries_to_flush ) ||
+ * ( entries_cleared > entries_to_clear ) ) )
+ */
+ } /* end while ( ( ( entries_flushed > entries_to_flush ) ||
+ * ( entries_cleared > entries_to_clear ) ) &&
+ * ( progress ) )
+ */
+
+#if H5C_DO_SANITY_CHECKS
+ HDassert(init_index_len == cache_ptr->index_len);
+#endif /* H5C_DO_SANITY_CHECKS */
+
+ if(entries_flushed != entries_to_flush || entries_cleared != entries_to_clear) {
+ entry_ptr = cache_ptr->il_head;
+ while(entry_ptr != NULL) {
+ HDassert(!entry_ptr->clear_on_unprotect || (entry_ptr->ring > ring));
+ HDassert(!entry_ptr->flush_immediately || (entry_ptr->ring > ring));
+ entry_ptr = entry_ptr->il_next;
+ } /* end while */
+
+ HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't flush/clear all entries")
+ } /* end if */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5C__flush_candidates_in_ring() */
#endif /* H5_HAVE_PARALLEL */
+
diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h
index 5b923e9..69e8145 100644
--- a/src/H5Cpkg.h
+++ b/src/H5Cpkg.h
@@ -4104,6 +4104,23 @@ typedef struct H5C_tag_info_t {
* of changes to the file driver info superblock extension
* management code needed to support rings.
*
+ * msic_in_progress: As the metadata cache has become re-entrant, and as
+ * the free space manager code has become more tightly
+ * integrated with the metadata cache, it is possible that
+ * a call to H5C_insert_entry() may trigger a call to
+ * H5C_make_space_in_cache(), which, via H5C__flush_single_entry()
+ * and client callbacks, may trigger an infinite regression
+ * of calls to H5C_make_space_in_cache().
+ *
+ * The msic_in_progress boolean flag is used to detect this,
+ * and prevent the infinite regression that would otherwise
+ * occur.
+ *
+ * Note that this is issue is not hypothetical -- this field
+ * was added 2/16/17 to address this issue when it was
+ * exposed by modifications to test/fheap.c to cause it to
+ * use paged allocation.
+ *
* resize_ctl: Instance of H5C_auto_size_ctl_t containing configuration
* data for automatic cache resizing.
*
@@ -4500,12 +4517,22 @@ typedef struct H5C_tag_info_t {
* total_entries_skipped_in_msic: Number of clean entries skipped while
* enforcing the min_clean_fraction in H5C__make_space_in_cache().
*
+ * total_dirty_pf_entries_skipped_in_msic: Number of dirty prefetched entries
+ * skipped in H5C__make_space_in_cache(). Note that this can
+ * only occur when a file is opened R/O with a cache image
+ * containing dirty entries.
+ *
* total_entries_scanned_in_msic: Number of clean entries skipped while
* enforcing the min_clean_fraction in H5C__make_space_in_cache().
*
* max_entries_skipped_in_msic: Maximum number of clean entries skipped
* in any one call to H5C__make_space_in_cache().
*
+ * max_dirty_pf_entries_skipped_in_msic: Maximum number of dirty prefetched
+ * entries skipped in any one call to H5C__make_space_in_cache().
+ * Note that this can only occur when the file is opened
+ * R/O with a cache image containing dirty entries.
+ *
* max_entries_scanned_in_msic: Maximum number of entries scanned over
* in any one call to H5C__make_space_in_cache().
*
@@ -4733,6 +4760,7 @@ struct H5C_t {
hbool_t cache_full;
hbool_t size_decreased;
hbool_t resize_in_progress;
+ hbool_t msic_in_progress;
H5C_auto_size_ctl_t resize_ctl;
/* Fields for epoch markers used in automatic cache size adjustment */
@@ -4822,8 +4850,10 @@ struct H5C_t {
/* Fields for tracking 'make space in cache' (msic) operations */
int64_t calls_to_msic;
int64_t total_entries_skipped_in_msic;
+ int64_t total_dirty_pf_entries_skipped_in_msic;
int64_t total_entries_scanned_in_msic;
int32_t max_entries_skipped_in_msic;
+ int32_t max_dirty_pf_entries_skipped_in_msic;
int32_t max_entries_scanned_in_msic;
int64_t entries_scanned_to_make_space;
@@ -4871,7 +4901,8 @@ typedef int (*H5C_tag_iter_cb_t)(H5C_cache_entry_t *entry, void *ctx);
/******************************/
/* Package Private Prototypes */
/******************************/
-H5_DLL herr_t H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id);
+H5_DLL herr_t H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id,
+ hbool_t *image_generated);
H5_DLL herr_t H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id,
H5C_t * cache_ptr, H5C_cache_entry_t** entry_ptr_ptr,
const H5C_class_t * type, haddr_t addr, void * udata);
diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h
index 28eacf2..539dece 100644
--- a/src/H5Cprivate.h
+++ b/src/H5Cprivate.h
@@ -185,6 +185,7 @@
* H5C__TAKE_OWNERSHIP_FLAG
* H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG
* H5C__GENERATE_IMAGE_FLAG
+ * H5C__UPDATE_PAGE_BUFFER_FLAG
*/
#define H5C__NO_FLAGS_SET 0x00000
#define H5C__SET_FLUSH_MARKER_FLAG 0x00001
@@ -205,6 +206,7 @@
#define H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG 0x08000
#define H5C__DURING_FLUSH_FLAG 0x10000 /* Set when the entire cache is being flushed */
#define H5C__GENERATE_IMAGE_FLAG 0x20000 /* Set during parallel I/O */
+#define H5C__UPDATE_PAGE_BUFFER_FLAG 0x40000 /* Set during parallel I/O */
/* Debugging/sanity checking/statistics settings */
#ifndef NDEBUG
@@ -1066,9 +1068,9 @@ typedef int H5C_ring_t;
*
* is_read_only: Boolean flag that is only meaningful if is_protected is
* TRUE. In this circumstance, it indicates whether the
- * entry has been protected read only, or read/write.
+ * entry has been protected read-only, or read/write.
*
- * If the entry has been protected read only (i.e. is_protected
+ * If the entry has been protected read-only (i.e. is_protected
* and is_read_only are both TRUE), we allow the entry to be
* protected more than once.
*
@@ -1078,7 +1080,7 @@ typedef int H5C_ring_t;
* the entry is actually unprotected.
*
* ro_ref_count: Integer field used to maintain a count of the number of
- * outstanding read only protects on this entry. This field
+ * outstanding read-only protects on this entry. This field
* must be zero whenever either is_protected or is_read_only
* are TRUE.
*
@@ -1458,13 +1460,13 @@ typedef int H5C_ring_t;
* the load of a cache image block, although other scenarios
* are contemplated for the use of this feature. Note that
* unlike the usual prefetch situation, this means that a
- * pre fetched entry can be dirty, and/or can be a party to
+ * prefetched entry can be dirty, and/or can be a party to
* flush dependency relationship(s). This complicates matters
* somewhat.
*
- * The essential feature of a pre-fetched entry is that it
+ * The essential feature of a prefetched entry is that it
* consists only of a buffer containing the on disk image of
- * the entry. Thus it must be deserialized before it can
+ * the entry. Thus it must be deserialized before it can
* be passed back to the library on a protect call. This
* task is handled by H5C_deserialized_prefetched_entry().
* In essence, this routine calls the deserialize callback
@@ -1475,7 +1477,7 @@ typedef int H5C_ring_t;
*
* Further, if the prefetched entry is a flush dependency parent,
* all its flush dependency children (which must also be
- * pre-fetched entries), must be tranfered to the new cache
+ * prefetched entries), must be tranfered to the new cache
* entry returned by the deserailization callback.
*
* Finally, if the prefetched entry is a flush dependency child,
@@ -1511,6 +1513,46 @@ typedef int H5C_ring_t;
*
* This field must be zero if prefetched is FALSE.
*
+ * prefetched_dirty: Boolean field that must be set to FALSE unless the
+ * following conditions hold:
+ *
+ * 1) The file has been opened R/O.
+ *
+ * 2) The entry is either a prefetched entry, or was
+ * re-constructed from a prefetched entry.
+ *
+ * 3) The base prefetched entry was marked dirty.
+ *
+ * This field exists to solve the following problem with
+ * files containing cache images that are opened R/O.
+ *
+ * If the cache image contains a dirty entry, that entry
+ * must be marked clean when it is inserted into the cache
+ * in the read-only case, as otherwise the metadata cache
+ * will attempt to flush it on file close -- which is poor
+ * form in the read-only case.
+ *
+ * However, since the entry is marked clean, it is possible
+ * that the metadata cache will evict it if the size of the
+ * metadata in the file exceeds the size of the metadata cache,
+ * and the application visits much of this data.
+ *
+ * If this happens, and the metadata cache is then asked for
+ * this entry, it will attempt to read it from file, and will
+ * obtain either obsolete or invalid data depending on whether
+ * the entry has ever been written to it assigned location in
+ * the file.
+ *
+ * With this background, the purpose of this field should be
+ * obvious -- when set, it allows the eviction candidate
+ * selection code to skip over the entry, thus avoiding the
+ * issue.
+ *
+ * Since the issue only arises in the R/O case, there is
+ * no possible interaction with SWMR. There are also
+ * potential interactions with Evict On Close -- at present,
+ * we deal with this by disabling EOC in the R/O case.
+ *
* serialization_count: Integer field used to maintain a count of the
* number of times each entry is serialized during cache
* serialization. While no entry should be serialized more than
@@ -1627,6 +1669,7 @@ typedef struct H5C_cache_entry_t {
hbool_t prefetched;
int prefetch_type_id;
int32_t age;
+ hbool_t prefetched_dirty;
#ifndef NDEBUG /* debugging field */
int serialization_count;
@@ -2208,6 +2251,7 @@ H5_DLL herr_t H5C_expunge_entry(H5F_t *f, hid_t dxpl_id,
const H5C_class_t *type, haddr_t addr, unsigned flags);
H5_DLL herr_t H5C_flush_cache(H5F_t *f, hid_t dxpl_id, unsigned flags);
H5_DLL herr_t H5C_flush_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag);
+H5_DLL herr_t H5C_force_cache_image_load(H5F_t * f, hid_t dxpl_id);
H5_DLL herr_t H5C_evict_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag, hbool_t match_global);
H5_DLL herr_t H5C_expunge_tag_type_metadata(H5F_t *f, hid_t dxpl_id, haddr_t tag, int type_id, unsigned flags);
H5_DLL herr_t H5C_get_tag(const void *thing, /*OUT*/ haddr_t *tag);
@@ -2259,7 +2303,6 @@ H5_DLL herr_t H5C_set_trace_file_ptr(H5C_t *cache_ptr, FILE *trace_file_ptr);
H5_DLL herr_t H5C_stats(H5C_t *cache_ptr, const char *cache_name,
hbool_t display_detailed_stats);
H5_DLL void H5C_stats__reset(H5C_t *cache_ptr);
-H5_DLL herr_t H5C_dump_cache(H5C_t *cache_ptr, const char *cache_name);
H5_DLL herr_t H5C_unpin_entry(void *thing);
H5_DLL herr_t H5C_destroy_flush_dependency(void *parent_thing, void *child_thing);
H5_DLL herr_t H5C_unprotect(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *thing,
@@ -2273,9 +2316,11 @@ H5_DLL herr_t H5C_retag_entries(H5C_t * cache_ptr, haddr_t src_tag, haddr_t dest
H5_DLL herr_t H5C_cork(H5C_t *cache_ptr, haddr_t obj_addr, unsigned action, hbool_t *corked);
H5_DLL herr_t H5C_get_entry_ring(const H5F_t *f, haddr_t addr, H5C_ring_t *ring);
H5_DLL herr_t H5C_unsettle_entry_ring(void *thing);
+H5_DLL herr_t H5C_unsettle_ring(H5F_t * f, H5C_ring_t ring);
H5_DLL herr_t H5C_remove_entry(void *thing);
H5_DLL herr_t H5C_cache_image_status(H5F_t * f, hbool_t *load_ci_ptr,
hbool_t *write_ci_ptr);
+H5_DLL hbool_t H5C_cache_image_pending(const H5C_t *cache_ptr);
#ifdef H5_HAVE_PARALLEL
H5_DLL herr_t H5C_apply_candidate_list(H5F_t *f, hid_t dxpl_id,
@@ -2289,6 +2334,8 @@ H5_DLL herr_t H5C_mark_entries_as_clean(H5F_t *f, hid_t dxpl_id, unsigned ce_arr
#endif /* H5_HAVE_PARALLEL */
#ifndef NDEBUG /* debugging functions */
+H5_DLL herr_t H5C_dump_cache(H5C_t *cache_ptr, const char *cache_name);
+H5_DLL herr_t H5C_dump_cache_LRU(H5C_t *cache_ptr, const char *cache_name);
H5_DLL hbool_t H5C_get_serialization_in_progress(const H5C_t *cache_ptr);
H5_DLL hbool_t H5C_cache_is_clean(const H5C_t *cache_ptr, H5C_ring_t inner_ring);
H5_DLL herr_t H5C_dump_cache_skip_list(H5C_t *cache_ptr, char *calling_fcn);
diff --git a/src/H5Ctag.c b/src/H5Ctag.c
index 157a838..a9bcca1 100644
--- a/src/H5Ctag.c
+++ b/src/H5Ctag.c
@@ -465,7 +465,7 @@ H5C__evict_tagged_entries_cb(H5C_cache_entry_t *entry, void *_ctx)
entry and we'll loop back around again (as evicting other
entries will hopefully unpin this entry) */
ctx->pinned_entries_need_evicted = TRUE;
- else {
+ else if(!entry->prefetched_dirty) {
/* Evict the Entry */
if(H5C__flush_single_entry(ctx->f, ctx->dxpl_id, entry, H5C__FLUSH_INVALIDATE_FLAG | H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, H5_ITER_ERROR, "Entry eviction failed.")
diff --git a/src/H5F.c b/src/H5F.c
index 5fd3a7d..a0f7599 100644
--- a/src/H5F.c
+++ b/src/H5F.c
@@ -406,7 +406,7 @@ H5Fis_hdf5(const char *name)
HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "no file name specified")
/* call the private is_HDF5 function */
- if((ret_value = H5F_is_hdf5(name, H5AC_ind_read_dxpl_id)) < 0)
+ if((ret_value = H5F__is_hdf5(name, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable open file")
done:
@@ -535,7 +535,7 @@ done:
* Modifications:
* Robb Matzke, 1997-07-18
* File struct creation and destruction is through H5F_new() and
- * H5F_dest(). Reading the root symbol table entry is done with
+ * H5F__dest(). Reading the root symbol table entry is done with
* H5G_decode().
*
* Robb Matzke, 1997-09-23
@@ -713,12 +713,12 @@ H5Fflush(hid_t object_id, H5F_scope_t scope)
/* Flush other files, depending on scope */
if(H5F_SCOPE_GLOBAL == scope) {
/* Call the flush routine for mounted file hierarchies */
- if(H5F_flush_mounts(f, H5AC_ind_read_dxpl_id) < 0)
+ if(H5F_flush_mounts(f, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush mounted file hierarchy")
} /* end if */
else {
/* Call the flush routine, for this file */
- if(H5F_flush(f, H5AC_ind_read_dxpl_id, FALSE) < 0)
+ if(H5F__flush(f, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, FALSE) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file's cached information")
} /* end else */
} /* end if */
@@ -773,7 +773,7 @@ H5Fclose(hid_t file_id)
if((nref = H5I_get_ref(file_id, FALSE)) < 0)
HGOTO_ERROR(H5E_ATOM, H5E_CANTGET, FAIL, "can't get ID ref count")
if(nref == 1)
- if(H5F_flush(f, H5AC_ind_read_dxpl_id, FALSE) < 0)
+ if(H5F__flush(f, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, FALSE) < 0)
HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache")
} /* end if */
@@ -838,7 +838,7 @@ H5Freopen(hid_t file_id)
done:
if(ret_value < 0 && new_file)
- if(H5F_dest(new_file, H5AC_ind_read_dxpl_id, FALSE) < 0)
+ if(H5F__dest(new_file, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, FALSE) < 0)
HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "can't close file")
FUNC_LEAVE_API(ret_value)
@@ -1042,7 +1042,7 @@ H5Fget_file_image(hid_t file_id, void *buf_ptr, size_t buf_len)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a file ID")
/* call private get_file_image function */
- if((ret_value = H5F_get_file_image(file, buf_ptr, buf_len, H5AC_ind_read_dxpl_id)) < 0)
+ if((ret_value = H5F_get_file_image(file, buf_ptr, buf_len, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file image")
done:
@@ -1619,7 +1619,7 @@ H5Fstart_swmr_write(hid_t file_id)
H5G_name_t *obj_paths=NULL; /* Group hierarchy path */
size_t u; /* Local index variable */
hbool_t setup = FALSE; /* Boolean flag to indicate whether SWMR setting is enabled */
- H5F_io_info_t fio_info; /* I/O info for operation */
+ H5F_io_info2_t fio_info; /* I/O info for operation */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
@@ -1650,7 +1650,7 @@ H5Fstart_swmr_write(hid_t file_id)
HGOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, FAIL, "can't have both SWMR and MDC cache image")
/* Flush data buffers */
- if(H5F_flush(file, H5AC_ind_read_dxpl_id, FALSE) < 0)
+ if(H5F__flush(file, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, FALSE) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file's cached information")
/* Get the # of opened named datatypes and attributes */
@@ -1704,7 +1704,9 @@ H5Fstart_swmr_write(hid_t file_id)
/* Set up I/O info for operation */
fio_info.f = file;
- if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id)))
+ if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
/* Flush and reset the accumulator */
@@ -1923,7 +1925,7 @@ H5Fset_latest_format(hid_t file_id, hbool_t latest_format)
latest_flags = H5F_USE_LATEST_FLAGS(f, H5F_LATEST_ALL_FLAGS);
if(latest_format != (H5F_LATEST_ALL_FLAGS == latest_flags)) {
/* Call the flush routine, for this file */
- if(H5F_flush(f, H5AC_ind_read_dxpl_id, FALSE) < 0)
+ if(H5F__flush(f, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, FALSE) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file's cached information")
/* Toggle the 'latest format' flag */
@@ -1973,7 +1975,9 @@ H5Fformat_convert(hid_t fid)
/* Check for persistent freespace manager, which needs to be downgraded */
if(!(f->shared->fs_strategy == H5F_FILE_SPACE_STRATEGY_DEF &&
- f->shared->fs_threshold == H5F_FREE_SPACE_THRESHOLD_DEF)) {
+ f->shared->fs_persist == H5F_FREE_SPACE_PERSIST_DEF &&
+ f->shared->fs_threshold == H5F_FREE_SPACE_THRESHOLD_DEF &&
+ f->shared->fs_page_size == H5F_FILE_SPACE_PAGE_SIZE_DEF)) {
/* Check to remove free-space manager info message from superblock extension */
if(H5F_addr_defined(f->shared->sblock->ext_addr))
if(H5F_super_ext_remove_msg(f, H5AC_ind_read_dxpl_id, H5O_FSINFO_ID) < 0)
@@ -1985,7 +1989,9 @@ H5Fformat_convert(hid_t fid)
/* Set non-persistent freespace manager */
f->shared->fs_strategy = H5F_FILE_SPACE_STRATEGY_DEF;
+ f->shared->fs_persist = H5F_FREE_SPACE_PERSIST_DEF;
f->shared->fs_threshold = H5F_FREE_SPACE_THRESHOLD_DEF;
+ f->shared->fs_page_size = H5F_FILE_SPACE_PAGE_SIZE_DEF;
/* Indicate that the superblock should be marked dirty */
mark_dirty = TRUE;
@@ -2004,3 +2010,79 @@ done:
FUNC_LEAVE_API(ret_value)
} /* end H5Fformat_convert() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5Freset_page_buffering_stats
+ *
+ * Purpose: Resets statistics for the page buffer layer.
+ *
+ * Return: Success: SUCCEED
+ * Failure: FAIL
+ *
+ * Programmer: Mohamad Chaarawi
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Freset_page_buffering_stats(hid_t file_id)
+{
+ H5F_t *file; /* File to reset stats on */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE1("e", "i", file_id);
+
+ /* Check args */
+ if(NULL == (file = (H5F_t *)H5I_object(file_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid file identifier")
+ if(NULL == file->shared->page_buf)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "page buffering not enabled on file")
+
+ /* Reset the statistics */
+ if(H5PB_reset_stats(file->shared->page_buf) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't reset stats for page buffering")
+
+done:
+ FUNC_LEAVE_API(ret_value)
+} /* H5Freset_page_buffering_stats() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Fget_page_buffering_stats
+ *
+ * Purpose: Retrieves statistics for the page buffer layer.
+ *
+ * Return: Success: SUCCEED
+ * Failure: FAIL
+ *
+ * Programmer: Mohamad Chaarawi
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Fget_page_buffering_stats(hid_t file_id, unsigned accesses[2], unsigned hits[2],
+ unsigned misses[2], unsigned evictions[2], unsigned bypasses[2])
+{
+ H5F_t *file; /* File object for file ID */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE6("e", "i*Iu*Iu*Iu*Iu*Iu", file_id, accesses, hits, misses, evictions,
+ bypasses);
+
+ /* Check args */
+ if(NULL == (file = (H5F_t *)H5I_object_verify(file_id, H5I_FILE)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a file ID")
+ if(NULL == file->shared->page_buf)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "page buffering not enabled on file")
+ if(NULL == accesses || NULL == hits || NULL == misses || NULL == evictions || NULL == bypasses)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL input parameters for stats")
+
+ /* Get the statistics */
+ if(H5PB_get_stats(file->shared->page_buf, accesses, hits, misses, evictions, bypasses) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't retrieve stats for page buffering")
+
+done:
+ FUNC_LEAVE_API(ret_value)
+} /* H5Fget_page_buffering_stats() */
+
diff --git a/src/H5FD.c b/src/H5FD.c
index 660f496..96d1230 100644
--- a/src/H5FD.c
+++ b/src/H5FD.c
@@ -1530,7 +1530,7 @@ herr_t
H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size,
void *buf/*out*/)
{
- H5P_genplist_t *dxpl; /* DXPL object */
+ H5FD_io_info_t fdio_info; /* File driver I/O object */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
@@ -1549,13 +1549,24 @@ H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size
if(!buf)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null result buffer")
- /* Get the DXPL plist object for DXPL ID */
- if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
- HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ /* Set up the file driver I/O info object */
+ fdio_info.file = file;
+ if(H5FD_MEM_DRAW == type) {
+ if(NULL == (fdio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ } /* end if */
+ else {
+ if(NULL == (fdio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ } /* end else */
/* Do the real work */
/* (Note compensating for base address addition in internal routine) */
- if(H5FD_read(file, dxpl, type, addr - file->base_addr, size, buf) < 0)
+ if(H5FD_read(&fdio_info, type, addr - file->base_addr, size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "file read request failed")
done:
@@ -1584,7 +1595,7 @@ herr_t
H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size,
const void *buf)
{
- H5P_genplist_t *dxpl; /* DXPL object */
+ H5FD_io_info_t fdio_info; /* File driver I/O object */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
@@ -1602,13 +1613,24 @@ H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t siz
if(!buf)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null buffer")
- /* Get the DXPL plist object for DXPL ID */
- if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
- HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ /* Set up the file driver I/O info object */
+ fdio_info.file = file;
+ if(H5FD_MEM_DRAW == type) {
+ if(NULL == (fdio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ } /* end if */
+ else {
+ if(NULL == (fdio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ } /* end else */
/* The real work */
/* (Note compensating for base address addition in internal routine) */
- if(H5FD_write(file, dxpl, type, addr - file->base_addr, size, buf) < 0)
+ if(H5FD_write(&fdio_info, type, addr - file->base_addr, size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "file write request failed")
done:
@@ -2032,3 +2054,27 @@ H5FD_get_base_addr(const H5FD_t *file)
FUNC_LEAVE_NOAPI(file->base_addr)
} /* end H5FD_get_base_addr() */
+
+/*--------------------------------------------------------------------------
+ * Function: H5FD_set_paged_aggr
+ *
+ * Purpose: Set "paged_aggr" for the file.
+ *
+ * Return: Non-negative if succeed; negative if fails.
+ *
+ * Programmer: Vailin Choi; April 2013
+ *
+ *--------------------------------------------------------------------------
+ */
+herr_t
+H5FD_set_paged_aggr(H5FD_t *file, hbool_t paged)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ HDassert(file);
+
+ /* Indicate whether paged aggregation for handling file space is enabled or not */
+ file->paged_aggr = paged;
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5FD_set_paged_aggr() */
diff --git a/src/H5FDint.c b/src/H5FDint.c
index 744c3d1..0809ac8 100644
--- a/src/H5FDint.c
+++ b/src/H5FDint.c
@@ -93,12 +93,9 @@
*-------------------------------------------------------------------------
*/
herr_t
-H5FD_locate_signature(H5FD_t *file,
-#ifndef H5_DEBUG_BUILD
-const
-#endif /* H5_DEBUG_BUILD */
-H5P_genplist_t *dxpl, haddr_t *sig_addr)
+H5FD_locate_signature(H5FD_io_info_t *fdio_info, haddr_t *sig_addr)
{
+ H5FD_t *file;
haddr_t addr, eoa, eof;
uint8_t buf[H5F_SIGNATURE_LEN];
unsigned n, maxpow;
@@ -106,6 +103,10 @@ H5P_genplist_t *dxpl, haddr_t *sig_addr)
FUNC_ENTER_NOAPI_NOINIT
+ HDassert(fdio_info);
+ file = fdio_info->file;
+ HDassert(file);
+
/* Find the least N such that 2^N is larger than the file size */
eof = H5FD_get_eof(file, H5FD_MEM_SUPER);
eoa = H5FD_get_eoa(file, H5FD_MEM_SUPER);
@@ -124,7 +125,7 @@ H5P_genplist_t *dxpl, haddr_t *sig_addr)
addr = (8 == n) ? 0 : (haddr_t)1 << n;
if(H5FD_set_eoa(file, H5FD_MEM_SUPER, addr + H5F_SIGNATURE_LEN) < 0)
HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to set EOA value for file signature")
- if(H5FD_read(file, dxpl, H5FD_MEM_SUPER, addr, (size_t)H5F_SIGNATURE_LEN, buf) < 0)
+ if(H5FD_read(fdio_info, H5FD_MEM_SUPER, addr, (size_t)H5F_SIGNATURE_LEN, buf) < 0)
HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to read file signature")
if(!HDmemcmp(buf, H5F_SIGNATURE, (size_t)H5F_SIGNATURE_LEN))
break;
@@ -162,29 +163,36 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
-H5FD_read(H5FD_t *file,
-#ifndef H5_DEBUG_BUILD
-const
-#endif /* H5_DEBUG_BUILD */
-H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t addr,
+H5FD_read(H5FD_io_info_t *fdio_info, H5FD_mem_t type, haddr_t addr,
size_t size, void *buf/*out*/)
{
+ H5FD_t *file;
+ H5P_genplist_t *io_dxpl;
haddr_t eoa = HADDR_UNDEF;
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
+ HDassert(fdio_info);
+ file = fdio_info->file;
HDassert(file && file->cls);
- HDassert(TRUE == H5P_class_isa(H5P_CLASS(dxpl), H5P_CLS_DATASET_XFER_g));
+ HDassert(TRUE == H5P_class_isa(H5P_CLASS(fdio_info->meta_dxpl), H5P_CLS_DATASET_XFER_g));
+ HDassert(TRUE == H5P_class_isa(H5P_CLASS(fdio_info->raw_dxpl), H5P_CLS_DATASET_XFER_g));
HDassert(buf);
+ /* Set up proper DXPL for I/O */
+ if(H5FD_MEM_DRAW == type)
+ io_dxpl = fdio_info->raw_dxpl;
+ else
+ io_dxpl = fdio_info->meta_dxpl;
+
/* Sanity check the dxpl type against the mem type */
#ifdef H5_DEBUG_BUILD
{
H5FD_dxpl_type_t dxpl_type; /* Property indicating the type of the internal dxpl */
/* get the dxpl type */
- if(H5P_get(dxpl, H5FD_DXPL_TYPE_NAME, &dxpl_type) < 0)
+ if(H5P_get(io_dxpl, H5FD_DXPL_TYPE_NAME, &dxpl_type) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't retrieve dxpl type")
/* we shouldn't be here if the dxpl is labeled with NO I/O */
@@ -219,7 +227,7 @@ H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t addr,
HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, addr = %llu, size = %llu, eoa = %llu", (unsigned long long)(addr + file->base_addr), (unsigned long long)size, (unsigned long long)eoa)
/* Dispatch to driver */
- if((file->cls->read)(file, type, H5P_PLIST_ID(dxpl), addr + file->base_addr, size, buf) < 0)
+ if((file->cls->read)(file, type, H5P_PLIST_ID(io_dxpl), addr + file->base_addr, size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "driver read request failed")
done:
@@ -241,29 +249,36 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
-H5FD_write(H5FD_t *file,
-#ifndef H5_DEBUG_BUILD
-const
-#endif /* H5_DEBUG_BUILD */
-H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t addr,
+H5FD_write(const H5FD_io_info_t *fdio_info, H5FD_mem_t type, haddr_t addr,
size_t size, const void *buf)
{
+ H5FD_t *file;
+ H5P_genplist_t *io_dxpl;
haddr_t eoa = HADDR_UNDEF;
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
+ HDassert(fdio_info);
+ file = fdio_info->file;
HDassert(file && file->cls);
- HDassert(TRUE == H5P_class_isa(H5P_CLASS(dxpl), H5P_CLS_DATASET_XFER_g));
+ HDassert(TRUE == H5P_class_isa(H5P_CLASS(fdio_info->meta_dxpl), H5P_CLS_DATASET_XFER_g));
+ HDassert(TRUE == H5P_class_isa(H5P_CLASS(fdio_info->raw_dxpl), H5P_CLS_DATASET_XFER_g));
HDassert(buf);
+ /* Set up proper DXPL for I/O */
+ if(H5FD_MEM_DRAW == type)
+ io_dxpl = fdio_info->raw_dxpl;
+ else
+ io_dxpl = fdio_info->meta_dxpl;
+
/* Sanity check the dxpl type against the mem type */
#ifdef H5_DEBUG_BUILD
{
H5FD_dxpl_type_t dxpl_type; /* Property indicating the type of the internal dxpl */
/* get the dxpl type */
- if(H5P_get(dxpl, H5FD_DXPL_TYPE_NAME, &dxpl_type) < 0)
+ if(H5P_get(io_dxpl, H5FD_DXPL_TYPE_NAME, &dxpl_type) < 0)
HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't retrieve dxpl type")
/* we shouldn't be here if the dxpl is labeled with NO I/O */
@@ -291,7 +306,7 @@ H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t addr,
(unsigned long long)(addr+ file->base_addr), (unsigned long long)size, (unsigned long long)eoa)
/* Dispatch to driver */
- if((file->cls->write)(file, type, H5P_PLIST_ID(dxpl), addr + file->base_addr, size, buf) < 0)
+ if((file->cls->write)(file, type, H5P_PLIST_ID(io_dxpl), addr + file->base_addr, size, buf) < 0)
HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "driver write request failed")
done:
diff --git a/src/H5FDlog.c b/src/H5FDlog.c
index 03228d2..f320946 100644
--- a/src/H5FDlog.c
+++ b/src/H5FDlog.c
@@ -934,13 +934,7 @@ H5FD_log_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hsi
/* Compute the address for the block to allocate */
addr = file->eoa;
- /* Check if we need to align this block */
- if(size >= file->pub.threshold) {
- /* Check for an already aligned block */
- if(addr % file->pub.alignment != 0)
- addr = ((addr / file->pub.alignment) + 1) * file->pub.alignment;
- } /* end if */
-
+ /* Extend the end-of-allocated space address */
file->eoa = addr + size;
/* Retain the (first) flavor of the information written to the file */
diff --git a/src/H5FDmulti.c b/src/H5FDmulti.c
index 7e12869..0ab7fc5 100644
--- a/src/H5FDmulti.c
+++ b/src/H5FDmulti.c
@@ -1194,6 +1194,8 @@ H5FD_multi_query(const H5FD_t *_f, unsigned long *flags /* out */)
*flags = 0;
*flags |= H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
*flags |= H5FD_FEAT_AGGREGATE_SMALLDATA; /* OK to aggregate "small" raw data allocations */
+ *flags |= H5FD_FEAT_USE_ALLOC_SIZE; /* OK just pass the allocation size to the alloc callback */
+ *flags |= H5FD_FEAT_PAGED_AGGR; /* OK special file space mapping for paged aggregation */
} /* end if */
return(0);
@@ -1538,6 +1540,14 @@ H5FD_multi_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size)
mmt = file->fa.memb_map[type];
if (H5FD_MEM_DEFAULT==mmt) mmt = type;
+ /* XXX: NEED to work on this again */
+ if(file->pub.paged_aggr) {
+ ALL_MEMBERS(mt) {
+ if(file->memb[mt])
+ file->memb[mt]->paged_aggr = file->pub.paged_aggr;
+ } END_MEMBERS;
+ }
+
if (HADDR_UNDEF==(addr=H5FDalloc(file->memb[mmt], mmt, dxpl_id, size)))
H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "member file can't alloc", HADDR_UNDEF)
addr += file->fa.memb_addr[mmt];
diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h
index 45f0187..c64ec30 100644
--- a/src/H5FDprivate.h
+++ b/src/H5FDprivate.h
@@ -115,6 +115,19 @@ typedef enum {
#define H5FD_DXPL_TYPE_NAME "H5P_dxpl_type"
#endif /* H5_DEBUG_BUILD */
+/* I/O Info for an operation */
+typedef struct H5FD_io_info_t {
+ H5FD_t *file; /* File driver object */
+#ifndef H5_DEBUG_BUILD
+ const
+#endif /* H5_DEBUG_BUILD */
+ H5P_genplist_t *meta_dxpl; /* Metadata DXPL object */
+#ifndef H5_DEBUG_BUILD
+ const
+#endif /* H5_DEBUG_BUILD */
+ H5P_genplist_t *raw_dxpl; /* Raw data DXPL object */
+} H5FD_io_info_t;
+
/*****************************/
/* Library Private Variables */
@@ -126,15 +139,10 @@ typedef enum {
/******************************/
/* Forward declarations for prototype arguments */
-struct H5P_genplist_t;
struct H5F_t;
H5_DLL int H5FD_term_interface(void);
-H5_DLL herr_t H5FD_locate_signature(H5FD_t *file,
-#ifndef H5_DEBUG_BUILD
-const
-#endif /* H5_DEBUG_BUILD */
-H5P_genplist_t *dxpl, haddr_t *sig_addr);
+H5_DLL herr_t H5FD_locate_signature(H5FD_io_info_t *fdio_info, haddr_t *sig_addr);
H5_DLL H5FD_class_t *H5FD_get_class(hid_t id);
H5_DLL hsize_t H5FD_sb_size(H5FD_t *file);
H5_DLL herr_t H5FD_sb_encode(H5FD_t *file, char *name/*out*/, uint8_t *buf);
@@ -146,8 +154,8 @@ H5_DLL H5FD_t *H5FD_open(const char *name, unsigned flags, hid_t fapl_id,
haddr_t maxaddr);
H5_DLL herr_t H5FD_close(H5FD_t *file);
H5_DLL int H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2);
-H5_DLL haddr_t H5FD_alloc(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, struct H5F_t *f,
- hsize_t size, haddr_t *align_addr, hsize_t *align_size);
+H5_DLL haddr_t H5FD_alloc(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type,
+ struct H5F_t *f, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size);
H5_DLL herr_t H5FD_free(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, struct H5F_t *f,
haddr_t addr, hsize_t size);
H5_DLL htri_t H5FD_try_extend(H5FD_t *file, H5FD_mem_t type, struct H5F_t *f,
@@ -159,17 +167,9 @@ H5_DLL haddr_t H5FD_get_maxaddr(const H5FD_t *file);
H5_DLL herr_t H5FD_get_feature_flags(const H5FD_t *file, unsigned long *feature_flags);
H5_DLL herr_t H5FD_set_feature_flags(H5FD_t *file, unsigned long feature_flags);
H5_DLL herr_t H5FD_get_fs_type_map(const H5FD_t *file, H5FD_mem_t *type_map);
-H5_DLL herr_t H5FD_read(H5FD_t *file,
-#ifndef H5_DEBUG_BUILD
-const
-#endif /* H5_DEBUG_BUILD */
-H5P_genplist_t *dxpl, H5FD_mem_t type,
+H5_DLL herr_t H5FD_read(H5FD_io_info_t *fdio_info, H5FD_mem_t type,
haddr_t addr, size_t size, void *buf/*out*/);
-H5_DLL herr_t H5FD_write(H5FD_t *file,
-#ifndef H5_DEBUG_BUILD
-const
-#endif /* H5_DEBUG_BUILD */
-H5P_genplist_t *dxpl, H5FD_mem_t type,
+H5_DLL herr_t H5FD_write(const H5FD_io_info_t *fdio_info, H5FD_mem_t type,
haddr_t addr, size_t size, const void *buf);
H5_DLL herr_t H5FD_flush(H5FD_t *file, hid_t dxpl_id, hbool_t closing);
H5_DLL herr_t H5FD_truncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing);
@@ -179,6 +179,7 @@ H5_DLL herr_t H5FD_get_fileno(const H5FD_t *file, unsigned long *filenum);
H5_DLL herr_t H5FD_get_vfd_handle(H5FD_t *file, hid_t fapl, void** file_handle);
H5_DLL herr_t H5FD_set_base_addr(H5FD_t *file, haddr_t base_addr);
H5_DLL haddr_t H5FD_get_base_addr(const H5FD_t *file);
+H5_DLL herr_t H5FD_set_paged_aggr(H5FD_t *file, hbool_t paged);
/* Function prototypes for MPI based VFDs*/
#ifdef H5_HAVE_PARALLEL
diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h
index 436be10..883b28d 100644
--- a/src/H5FDpublic.h
+++ b/src/H5FDpublic.h
@@ -239,6 +239,19 @@ typedef enum H5F_mem_t H5FD_mem_t;
* driver supports the single-writer/multiple-readers I/O pattern.
*/
#define H5FD_FEAT_SUPPORTS_SWMR_IO 0x00001000
+ /*
+ * Defining H5FD_FEAT_USE_ALLOC_SIZE for a VFL driver
+ * means that the library will just pass the allocation size to the
+ * the driver's allocation callback which will eventually handle alignment.
+ * This is specifically used for the multi/split driver.
+ */
+#define H5FD_FEAT_USE_ALLOC_SIZE 0x00002000
+ /*
+ * Defining H5FD_FEAT_PAGED_AGGR for a VFL driver
+ * means that the driver needs special file space mapping for paged aggregation.
+ * This is specifically used for the multi/split driver.
+ */
+#define H5FD_FEAT_PAGED_AGGR 0x00004000
/* Forward declaration */
typedef struct H5FD_t H5FD_t;
@@ -307,6 +320,7 @@ struct H5FD_t {
/* Space allocation management fields */
hsize_t threshold; /* Threshold for alignment */
hsize_t alignment; /* Allocation alignment */
+ hbool_t paged_aggr; /* Paged aggregation for file space is enabled or not */
};
/* Define enum for the source of file image callbacks */
diff --git a/src/H5FDspace.c b/src/H5FDspace.c
index fcddecf..0ad3cf0 100644
--- a/src/H5FDspace.c
+++ b/src/H5FDspace.c
@@ -99,7 +99,7 @@ H5FL_DEFINE(H5FD_free_t);
*-------------------------------------------------------------------------
*/
static haddr_t
-H5FD_extend(H5FD_t *file, H5FD_mem_t type, hbool_t new_block, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size)
+H5FD_extend(H5FD_t *file, H5FD_mem_t type, hsize_t size)
{
hsize_t orig_size = size; /* Original allocation size */
haddr_t eoa; /* Address of end-of-allocated space */
@@ -117,40 +117,18 @@ H5FD_extend(H5FD_t *file, H5FD_mem_t type, hbool_t new_block, hsize_t size, hadd
/* Get current end-of-allocated space address */
eoa = file->cls->get_eoa(file, type);
- /* Compute extra space to allocate, if this is a new block and should be aligned */
- extra = 0;
- if(new_block && file->alignment > 1 && orig_size >= file->threshold) {
- hsize_t mis_align; /* Amount EOA is misaligned */
-
- /* Check for EOA already aligned */
- if((mis_align = (eoa % file->alignment)) > 0) {
- extra = file->alignment - mis_align;
- if(frag_addr)
- *frag_addr = eoa - file->base_addr; /* adjust for file's base address */
- if(frag_size)
- *frag_size = extra;
- } /* end if */
- } /* end if */
-
- /* Add in extra allocation amount */
- size += extra;
-
/* Check for overflow when extending */
if(H5F_addr_overflow(eoa, size) || (eoa + size) > file->maxaddr)
HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "file allocation request failed")
- /* Set the [possibly aligned] address to return */
- ret_value = eoa + extra;
+ /* Set the [NOT aligned] address to return */
+ ret_value = eoa;
/* Extend the end-of-allocated space address */
eoa += size;
if(file->cls->set_eoa(file, type, eoa) < 0)
HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "file allocation request failed")
- /* Post-condition sanity check */
- if(new_block && file->alignment && orig_size >= file->threshold)
- HDassert(!(ret_value % file->alignment));
-
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_extend() */
@@ -160,6 +138,8 @@ done:
* Function: H5FD_alloc_real
*
* Purpose: Allocate space in the file with the VFD
+ * Note: the handling of alignment is moved up from each driver to
+ * this routine.
*
* Return: Success: The format address of the new file memory.
* Failure: The undefined address HADDR_UNDEF
@@ -170,8 +150,14 @@ done:
*-------------------------------------------------------------------------
*/
haddr_t
-H5FD_alloc_real(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size)
+H5FD_alloc_real(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, hsize_t size,
+ haddr_t *frag_addr, hsize_t *frag_size)
{
+ hsize_t orig_size = size; /* Original allocation size */
+ haddr_t eoa; /* Address of end-of-allocated space */
+ hsize_t extra; /* Extra space to allocate, to align request */
+ unsigned long flags = 0; /* Driver feature flags */
+ hbool_t use_alloc_size; /* Just pass alloc size to the driver */
haddr_t ret_value = HADDR_UNDEF; /* Return value */
FUNC_ENTER_NOAPI(HADDR_UNDEF)
@@ -185,16 +171,53 @@ HDfprintf(stderr, "%s: type = %u, size = %Hu\n", FUNC, (unsigned)type, size);
HDassert(type >= H5FD_MEM_DEFAULT && type < H5FD_MEM_NTYPES);
HDassert(size > 0);
+ /* Check for query driver and call it */
+ if(file->cls->query)
+ (file->cls->query)(file, &flags);
+
+ /* Check for the driver feature flag */
+ use_alloc_size = flags & H5FD_FEAT_USE_ALLOC_SIZE;
+
+ /* Get current end-of-allocated space address */
+ eoa = file->cls->get_eoa(file, type);
+
+ /* Compute extra space to allocate, if this should be aligned */
+ extra = 0;
+ if(!file->paged_aggr && file->alignment > 1 && orig_size >= file->threshold) {
+ hsize_t mis_align; /* Amount EOA is misaligned */
+
+ /* Check for EOA already aligned */
+ if((mis_align = (eoa % file->alignment)) > 0) {
+ extra = file->alignment - mis_align;
+ if(frag_addr)
+ *frag_addr = eoa - file->base_addr; /* adjust for file's base address */
+ if(frag_size)
+ *frag_size = extra;
+ } /* end if */
+ } /* end if */
+
/* Dispatch to driver `alloc' callback or extend the end-of-address marker */
+ /* For the multi/split driver: the size passed down to the alloc callback is the original size from H5FD_alloc() */
+ /* For all other drivers: the size passed down to the alloc callback is the size + [possibly] alignment size */
if(file->cls->alloc) {
- if((ret_value = (file->cls->alloc)(file, type, dxpl_id, size)) == HADDR_UNDEF)
+ ret_value = (file->cls->alloc)(file, type, dxpl_id, use_alloc_size ? size : size + extra);
+ if(!H5F_addr_defined(ret_value))
HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "driver allocation request failed")
} /* end if */
else {
- if((ret_value = H5FD_extend(file, type, TRUE, size, frag_addr, frag_size)) == HADDR_UNDEF)
+ ret_value = H5FD_extend(file, type, size + extra);
+ if(!H5F_addr_defined(ret_value))
HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "driver eoa update request failed")
} /* end else */
+ /* Set the [possibly aligned] address to return */
+ if(!use_alloc_size)
+ ret_value += extra;
+
+ /* Post-condition sanity check */
+ if(!file->paged_aggr && file->alignment > 1 && orig_size >= file->threshold)
+ HDassert(!(ret_value % file->alignment));
+
/* Convert absolute file offset to relative address */
ret_value -= file->base_addr;
@@ -420,7 +443,7 @@ H5FD_try_extend(H5FD_t *file, H5FD_mem_t type, H5F_t *f, hid_t dxpl_id,
/* Check if the block is exactly at the end of the file */
if(H5F_addr_eq(blk_end, eoa)) {
/* Extend the object by extending the underlying file */
- if(HADDR_UNDEF == H5FD_extend(file, type, FALSE, extra_requested, NULL, NULL))
+ if(HADDR_UNDEF == H5FD_extend(file, type, extra_requested))
HGOTO_ERROR(H5E_VFL, H5E_CANTEXTEND, FAIL, "driver extend request failed")
/* Mark EOA info dirty in cache, so change will get encoded */
diff --git a/src/H5FDstdio.c b/src/H5FDstdio.c
index 4c62bcc..13f728e 100644
--- a/src/H5FDstdio.c
+++ b/src/H5FDstdio.c
@@ -601,13 +601,6 @@ H5FD_stdio_alloc(H5FD_t *_file, H5FD_mem_t /*UNUSED*/ type, hid_t /*UNUSED*/ dxp
/* Compute the address for the block to allocate */
addr = file->eoa;
- /* Check if we need to align this block */
- if(size >= file->pub.threshold) {
- /* Check for an already aligned block */
- if((addr % file->pub.alignment) != 0)
- addr = ((addr / file->pub.alignment) + 1) * file->pub.alignment;
- } /* end if */
-
file->eoa = addr + size;
return addr;
diff --git a/src/H5FS.c b/src/H5FS.c
index 8644a6f..a433fbb 100644
--- a/src/H5FS.c
+++ b/src/H5FS.c
@@ -135,7 +135,7 @@ HDfprintf(stderr, "%s: Creating free space manager, nclasses = %Zu\n", FUNC, ncl
fspace->swmr_write = (H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) > 0;
fspace->alignment = alignment;
- fspace->threshold = threshold;
+ fspace->align_thres = threshold;
/* Check if the free space tracker is supposed to be persistant */
if(fs_addr) {
@@ -227,7 +227,7 @@ HDfprintf(stderr, "%s: fspace->rc = %u\n", FUNC, fspace->rc);
HGOTO_ERROR(H5E_FSPACE, H5E_CANTINC, NULL, "unable to increment ref. count on free space header")
fspace->alignment = alignment;
- fspace->threshold = threshold;
+ fspace->align_thres = threshold;
/* Unlock free space header */
if(H5AC_unprotect(f, dxpl_id, H5AC_FSPACE_HDR, fs_addr, fspace, H5AC__NO_FLAGS_SET) < 0)
@@ -873,10 +873,7 @@ H5FS_alloc_sect(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id)
HDassert(fspace);
if(!H5F_addr_defined(fspace->sect_addr) && fspace->sinfo && fspace->serial_sect_count > 0) {
- /* Allocate space for section info from aggregator/vfd (or temp. address space) */
- /* (The original version called H5MF_alloc(), but that may cause sect_size to change again) */
- /* (This routine is only called during file close operations, so don't allocate from temp. address space) */
- if(HADDR_UNDEF == (fspace->sect_addr = H5MF_aggr_vfd_alloc(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, fspace->sect_size)))
+ if(HADDR_UNDEF == (fspace->sect_addr = H5MF_alloc(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, fspace->sect_size)))
HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "file allocation failed for section info")
fspace->alloc_sect_size = fspace->sect_size;
@@ -901,169 +898,6 @@ done:
/*-------------------------------------------------------------------------
- * Function: H5FS_alloc_vfd_alloc_hdr_and_section_info
- *
- * Purpose: This function is part of a hack to patch over a design
- * flaw in the free space managers for file space allocation.
- * Specifically, if a free space manager allocates space for
- * its own section info, it is possible for it to
- * go into an infinite loop as it:
- *
- * 1) computes the size of the section info
- *
- * 2) allocates file space for the section info
- *
- * 3) notices that the size of the section info
- * has changed
- *
- * 4) deallocates the section info file space and
- * returns to 1) above.
- *
- * Similarly, if it allocates space for its own header, it
- * can go into an infinte loop as it:
- *
- * 1) allocates space for the header
- *
- * 2) notices that the free space manager is empty
- * and thus should not have file space allocated
- * to its header
- *
- * 3) frees the space allocated to the header
- *
- * 4) notices that the free space manager is not
- * empty and thus must have file space allocated
- * to it, and thus returns to 1) above.
- *
- * In a nutshell, the solution applied in this hack is to
- * deallocate file space for the free space manager(s) that
- * handle FSM header and/or section info file space allocations,
- * wait until all other allocation/deallocation requests have
- * been handled, and then test to see if the free space manager(s)
- * in question are empty. If they are, do nothing. If they
- * are not, allocate space for them at end of file bypassing the
- * usual file space allocation calls, and thus avoiding the
- * potential infinite loops.
- *
- * The purpose of this function is to allocate file space for
- * the header and section info of the target free space manager
- * directly from the VFD if needed. In this case the function
- * also re-inserts the header and section info in the metadata
- * cache with this allocation.
- *
- * Note that if f->shared->alignment > 1, and EOA is not a
- * multiple of the alignment, it is possible that performing
- * this allocation may generate a fragment of file space in
- * addition to the space allocated for the section info.
- *
- * At present we deal with this by screaming and dying.
- * Obviously, this is not acceptatable, but it should do
- * for now.
- *
- *
- * Return: Success: non-negative
- * Failure: negative
- *
- * Programmer: John Mainzer
- * 6/6/16
- *
- *-------------------------------------------------------------------------
- */
-herr_t
-H5FS_alloc_vfd_alloc_hdr_and_section_info(H5F_t *f, hid_t dxpl_id,
- H5FS_t *fspace, haddr_t *fs_addr_ptr)
-{
- herr_t ret_value = SUCCEED; /* Return value */
-
- FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
-
- /* Check arguments */
- HDassert(f);
- HDassert(fspace);
- HDassert(fs_addr_ptr);
-
- /* The section info should be unlocked */
- HDassert(fspace->sinfo_lock_count == 0);
-
- /* No space should be allocated */
- HDassert(*fs_addr_ptr == HADDR_UNDEF);
- HDassert(fspace->addr == HADDR_UNDEF);
- HDassert(fspace->sect_addr == HADDR_UNDEF);
- HDassert(fspace->alloc_sect_size == 0);
-
- /* Check if any space will be needed */
- if(fspace->serial_sect_count > 0) {
- haddr_t sect_addr; /* Address of sinfo */
-
- /* The section info is floating, so fspace->sinfo should be defined */
- HDassert(fspace->sinfo);
-
- /* Start by allocating file space for the header */
- if(HADDR_UNDEF == (fspace->addr = H5MF_vfd_alloc(f, dxpl_id, H5FD_MEM_FSPACE_HDR,
- (hsize_t)H5FS_HEADER_SIZE(f), FALSE)))
- HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space for hdr")
-
- /* Cache the new free space header (pinned) */
- if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_HDR, fspace->addr, fspace, H5AC__PIN_ENTRY_FLAG) < 0)
- HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't add free space header to cache")
-
-
- /* Now allocate file space for the section info */
- if(HADDR_UNDEF == (sect_addr = H5MF_vfd_alloc(f, dxpl_id, H5FD_MEM_FSPACE_SINFO,
- fspace->sect_size, FALSE)))
- HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space")
-
- /* Update fspace->alloc_sect_size and fspace->sect_addr to reflect
- * the allocation
- */
- fspace->alloc_sect_size = fspace->sect_size;
- fspace->sect_addr = sect_addr;
-
- /* We have changed the sinfo address -- Mark free space header dirty */
- if(H5AC_mark_entry_dirty(fspace) < 0)
- HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty")
-
- /* Insert the new section info into the metadata cache. */
-
- /* Question: Do we need to worry about this insertion causing an
- * eviction from the metadata cache? Think on this. If so, add a
- * flag to H5AC_insert() to force it to skip the call to make space in
- * cache.
- *
- * On reflection, no.
- *
- * On a regular file close, any eviction will not change the
- * the contents of the free space manger(s), as all entries
- * should have correct file space allocated by the time this
- * function is called.
- *
- * In the cache image case, the selection of entries for inclusion
- * in the cache image will not take place until after this call.
- * (Recall that this call is made during the metadata fsm settle
- * routine, which is called during the serialization routine in
- * the cache image case. Entries are not selected for inclusion
- * in the image until after the cache is serialized.)
- *
- * JRM -- 11/4/16
- */
- if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_SINFO, sect_addr, fspace->sinfo, H5AC__NO_FLAGS_SET) < 0)
- HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't add free space sinfo to cache")
-
- /* Since space has been allocated for the section info and the sinfo
- * has been inserted into the cache, relinquish owership (i.e. float)
- * the section info.
- */
- fspace->sinfo = NULL;
-
- /* Set the address of the free space header, on success */
- *fs_addr_ptr = fspace->addr;
- } /* end if */
-
-done:
- FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
-} /* H5FS_alloc_vfd_alloc_hdr_and_section_info() */
-
-
-/*-------------------------------------------------------------------------
* Function: H5FS_free()
*
* Purpose: Free space for free-space manager header and section info header
@@ -1076,7 +910,7 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
-H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id)
+H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id, hbool_t free_file_space)
{
haddr_t saved_addr; /* Previous address of item */
unsigned cache_flags; /* Flags for unprotecting cache entries */
@@ -1123,7 +957,8 @@ H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id)
/* Free space for the free-space manager section info */
if(!H5F_IS_TMP_ADDR(f, saved_addr)) {
- if(H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, saved_addr, saved_size) < 0)
+ if(free_file_space &&
+ H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, saved_addr, saved_size) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to release free space sections")
} /* end if */
@@ -1165,7 +1000,8 @@ H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id)
fspace->addr = HADDR_UNDEF;
/* Free space for the free-space manager header */
- if(H5MF_xfree(f, H5FD_MEM_FSPACE_HDR, dxpl_id, saved_addr, (hsize_t)H5FS_HEADER_SIZE(f)) < 0)
+ if(free_file_space &&
+ H5MF_xfree(f, H5FD_MEM_FSPACE_HDR, dxpl_id, saved_addr, (hsize_t)H5FS_HEADER_SIZE(f)) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to free free space header")
} /* end if */
@@ -1339,6 +1175,23 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FS_sinfo_dest() */
+herr_t
+H5FS_get_sect_count(const H5FS_t *frsp, hsize_t *tot_sect_count)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ /* Check arguments. */
+ HDassert(frsp);
+ HDassert(tot_sect_count);
+
+ /* Report statistics for free space */
+ *tot_sect_count = frsp->serial_sect_count;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+}
+
#ifdef H5FS_DEBUG_ASSERT
/*-------------------------------------------------------------------------
diff --git a/src/H5FScache.c b/src/H5FScache.c
index dcc8122..ddc66fd 100644
--- a/src/H5FScache.c
+++ b/src/H5FScache.c
@@ -37,6 +37,7 @@
#include "H5private.h" /* Generic Functions */
#include "H5ACprivate.h" /* Metadata cache */
#include "H5Eprivate.h" /* Error handling */
+#include "H5Fprivate.h" /* File */
#include "H5FSpkg.h" /* File free space */
#include "H5MFprivate.h" /* File memory management */
#include "H5VMprivate.h" /* Vectors and arrays */
@@ -296,7 +297,7 @@ H5FS__cache_hdr_deserialize(const void *_image, size_t len, void *_udata,
/* # of section classes */
/* (only check if we actually have some classes) */
UINT16DECODE(image, nclasses);
- if(fspace->nclasses > 0 && fspace->nclasses != nclasses)
+ if(fspace->nclasses > 0 && nclasses > fspace->nclasses)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTLOAD, NULL, "section class count mismatch")
/* Shrink percent */
@@ -596,9 +597,11 @@ H5FS__cache_hdr_pre_serialize(H5F_t *f, hid_t dxpl_id, void *_thing,
* real file space lest the header be written to file with
* a nonsense section info address.
*/
- HDassert(fspace->serial_sect_count > 0);
- HDassert(fspace->sect_size > 0);
- HDassert(fspace->alloc_sect_size == (size_t)fspace->sect_size);
+ if(!H5F_POINT_OF_NO_RETURN(f)) {
+ HDassert(fspace->serial_sect_count > 0);
+ HDassert(fspace->sect_size > 0);
+ HDassert(fspace->alloc_sect_size == (size_t)fspace->sect_size);
+ } /* end if */
if(H5F_IS_TMP_ADDR(f, fspace->sect_addr)) {
unsigned sect_status = 0;
@@ -702,10 +705,12 @@ H5FS__cache_hdr_serialize(const H5F_t *f, void *_image, size_t len,
* The following asserts are a cursory check on this.
*/
HDassert((! H5F_addr_defined(fspace->sect_addr)) || (! H5F_IS_TMP_ADDR(f, fspace->sect_addr)));
- HDassert((! H5F_addr_defined(fspace->sect_addr)) ||
- ((fspace->serial_sect_count > 0) &&
- (fspace->sect_size > 0) &&
- (fspace->alloc_sect_size == (size_t)fspace->sect_size)));
+
+ if(!H5F_POINT_OF_NO_RETURN(f))
+ HDassert((! H5F_addr_defined(fspace->sect_addr)) ||
+ ((fspace->serial_sect_count > 0) &&
+ (fspace->sect_size > 0) &&
+ (fspace->alloc_sect_size == (size_t)fspace->sect_size)));
/* Magic number */
HDmemcpy(image, H5FS_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC);
@@ -1061,7 +1066,7 @@ H5FS__cache_sinfo_deserialize(const void *_image, size_t len, void *_udata,
/* Insert section in free space manager, unless requested not to */
if(!(des_flags & H5FS_DESERIALIZE_NO_ADD))
- if(H5FS_sect_add(udata->f, udata->dxpl_id, udata->fspace, new_sect, H5FS_ADD_DESERIALIZING, NULL) < 0)
+ if(H5FS_sect_add(udata->f, udata->dxpl_id, udata->fspace, new_sect, H5FS_ADD_DESERIALIZING, udata) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, NULL, "can't add section to free space manager")
} /* end for */
} while(image < (((const uint8_t *)_image + old_sect_size) - H5FS_SIZEOF_CHKSUM));
@@ -1177,8 +1182,9 @@ H5FS__cache_sinfo_pre_serialize(H5F_t *f, hid_t dxpl_id, void *_thing,
HDassert(new_len);
HDassert(flags);
- /* we shouldn't be called if the section info is empty */
- HDassert(fspace->serial_sect_count > 0);
+ /* we shouldn't be called if the section info is empty, unless we hit the point of no return. */
+ if(!H5F_POINT_OF_NO_RETURN(f))
+ HDassert(fspace->serial_sect_count > 0);
sinfo_addr = addr; /* this will change if we relocate the section data */
diff --git a/src/H5FSpkg.h b/src/H5FSpkg.h
index 4411236..ab97955 100644
--- a/src/H5FSpkg.h
+++ b/src/H5FSpkg.h
@@ -187,8 +187,9 @@ struct H5FS_t {
/* must be either H5C__NO_FLAGS_SET (i.e r/w) */
/* or H5AC__READ_ONLY_FLAG (i.e. r/o). */
size_t max_cls_serial_size; /* Max. additional size of serialized form of section */
- hsize_t threshold; /* Threshold for alignment */
hsize_t alignment; /* Alignment */
+ hsize_t align_thres; /* Threshold for alignment */
+
/* Memory data structures (not stored directly) */
H5FS_section_class_t *sect_cls; /* Array of section classes for this free list */
diff --git a/src/H5FSprivate.h b/src/H5FSprivate.h
index 1816a6e..74f9dcb 100644
--- a/src/H5FSprivate.h
+++ b/src/H5FSprivate.h
@@ -67,6 +67,12 @@
* managed sections is in flux)
*/
+#define H5FS_PAGE_END_NO_ADD 0x08 /* For "small" page fs:
+ * Don't add section to free space:
+ * when the section is at page end and
+ * when the section size is <= "small"
+ */
+
/* Flags for deserialize callback */
#define H5FS_DESERIALIZE_NO_ADD 0x01 /* Don't add section to free space
* manager after it's deserialized
@@ -98,11 +104,11 @@ typedef struct H5FS_section_class_t {
herr_t (*term_cls)(struct H5FS_section_class_t *); /* Routine to terminate class-specific settings */
/* Object methods */
- herr_t (*add)(H5FS_section_info_t *, unsigned *, void *); /* Routine called when section is about to be added to manager */
+ herr_t (*add)(H5FS_section_info_t **, unsigned *, void *); /* Routine called when section is about to be added to manager */
herr_t (*serialize)(const struct H5FS_section_class_t *, const H5FS_section_info_t *, uint8_t *); /* Routine to serialize a "live" section into a buffer */
H5FS_section_info_t *(*deserialize)(const struct H5FS_section_class_t *, hid_t dxpl_id, const uint8_t *, haddr_t, hsize_t, unsigned *); /* Routine to deserialize a buffer into a "live" section */
htri_t (*can_merge)(const H5FS_section_info_t *, const H5FS_section_info_t *, void *); /* Routine to determine if two nodes are mergable */
- herr_t (*merge)(H5FS_section_info_t *, H5FS_section_info_t *, void *); /* Routine to merge two nodes */
+ herr_t (*merge)(H5FS_section_info_t **, H5FS_section_info_t *, void *); /* Routine to merge two nodes */
htri_t (*can_shrink)(const H5FS_section_info_t *, void *); /* Routine to determine if node can shrink container */
herr_t (*shrink)(H5FS_section_info_t **, void *); /* Routine to shrink container */
herr_t (*free)(H5FS_section_info_t *); /* Routine to free node */
@@ -182,9 +188,8 @@ H5_DLL herr_t H5FS_delete(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr);
H5_DLL herr_t H5FS_close(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace);
H5_DLL herr_t H5FS_alloc_hdr(H5F_t *f, H5FS_t *fspace, haddr_t *fs_addr, hid_t dxpl_id);
H5_DLL herr_t H5FS_alloc_sect(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id);
-H5_DLL herr_t H5FS_alloc_vfd_alloc_hdr_and_section_info(H5F_t *f, hid_t dxpl_id,
- H5FS_t *fspace, haddr_t *fs_addr_ptr);
-H5_DLL herr_t H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id);
+H5_DLL herr_t H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id,
+ hbool_t free_file_space);
/* Free space section routines */
H5_DLL herr_t H5FS_sect_add(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace,
@@ -192,7 +197,7 @@ H5_DLL herr_t H5FS_sect_add(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace,
H5_DLL htri_t H5FS_sect_try_merge(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace,
H5FS_section_info_t *sect, unsigned flags, void *op_data);
H5_DLL htri_t H5FS_sect_try_extend(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace,
- haddr_t addr, hsize_t size, hsize_t extra_requested);
+ haddr_t addr, hsize_t size, hsize_t extra_requested, unsigned flags, void *op_data);
H5_DLL herr_t H5FS_sect_remove(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace,
H5FS_section_info_t *node);
H5_DLL htri_t H5FS_sect_find(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace,
@@ -202,11 +207,18 @@ H5_DLL herr_t H5FS_sect_stats(const H5FS_t *fspace, hsize_t *tot_space,
hsize_t *nsects);
H5_DLL herr_t H5FS_sect_change_class(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace,
H5FS_section_info_t *sect, uint16_t new_class);
-H5_DLL htri_t H5FS_sect_try_shrink_eoa(const H5F_t *f, hid_t dxpl_id, const H5FS_t *fspace, void *op_data);
-H5_DLL herr_t H5FS_sect_query_last_sect(const H5FS_t *fspace, haddr_t *sect_addr, hsize_t *sect_size);
+H5_DLL htri_t H5FS_sect_try_shrink_eoa(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace,
+ void *op_data);
/* Statistics routine */
H5_DLL herr_t H5FS_stat_info(const H5F_t *f, const H5FS_t *frsp, H5FS_stat_t *stats);
+H5_DLL herr_t H5FS_get_sect_count(const H5FS_t *frsp, hsize_t *tot_sect_count);
+
+/* free space manager settling routines */
+H5_DLL herr_t H5FS_vfd_alloc_hdr_and_section_info_if_needed(H5F_t *f,
+ hid_t dxpl_id,
+ H5FS_t *fspace,
+ haddr_t *fs_addr_ptr);
/* Debugging routines for dumping file structures */
H5_DLL herr_t H5FS_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr,
diff --git a/src/H5FSsection.c b/src/H5FSsection.c
index efe0031..49ff003 100644
--- a/src/H5FSsection.c
+++ b/src/H5FSsection.c
@@ -25,6 +25,8 @@
/* Module Setup */
/****************/
+#define H5F_FRIEND /*suppress error about including H5Fpkg */
+
#include "H5FSmodule.h" /* This source code file is part of the H5FS module */
@@ -33,6 +35,7 @@
/***********/
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
+#include "H5Fpkg.h" /* File access */
#include "H5FSpkg.h" /* File free space */
#include "H5MFprivate.h" /* File memory management */
#include "H5VMprivate.h" /* Vectors and arrays */
@@ -1210,11 +1213,13 @@ H5FS_sect_merge(H5FS_t *fspace, H5FS_section_info_t **sect, void *op_data)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't remove section from internal data structures")
/* Merge the two sections together */
- if((*tmp_sect_cls->merge)(tmp_sect, *sect, op_data) < 0)
+ if((*tmp_sect_cls->merge)(&tmp_sect, *sect, op_data) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't merge two sections")
/* Retarget section pointer to 'less than' node that was merged into */
*sect = tmp_sect;
+ if(*sect == NULL)
+ HGOTO_DONE(ret_value);
/* Indicate successful merge occurred */
modified = TRUE;
@@ -1254,9 +1259,13 @@ H5FS_sect_merge(H5FS_t *fspace, H5FS_section_info_t **sect, void *op_data)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't remove section from internal data structures")
/* Merge the two sections together */
- if((*sect_cls->merge)(*sect, tmp_sect, op_data) < 0)
+ if((*sect_cls->merge)(sect, tmp_sect, op_data) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't merge two sections")
+ /* It's possible that the merge caused the section to be deleted (particularly in the paged allocation case) */
+ if(*sect == NULL)
+ HGOTO_DONE(ret_value);
+
/* Indicate successful merge occurred */
modified = TRUE;
} /* end if */
@@ -1383,7 +1392,7 @@ HDfprintf(stderr, "%s: *sect = {%a, %Hu, %u, %s}\n", FUNC, sect->addr, sect->siz
/* Call "add" section class callback, if there is one */
cls = &fspace->sect_cls[sect->type];
if(cls->add) {
- if((*cls->add)(sect, &flags, op_data) < 0)
+ if((*cls->add)(&sect, &flags, op_data) < 0)
HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "'add' section class callback failed")
} /* end if */
@@ -1411,7 +1420,7 @@ HDfprintf(stderr, "%s: fspace->tot_space = %Hu\n", FUNC, fspace->tot_space);
#endif /* H5FS_SINFO_DEBUG */
/* Mark free space sections as changed */
/* (if adding sections while deserializing sections, don't set the flag) */
- if(!(flags & H5FS_ADD_DESERIALIZING))
+ if(!(flags & (H5FS_ADD_DESERIALIZING | H5FS_PAGE_END_NO_ADD)))
sinfo_modified = TRUE;
done:
@@ -1444,7 +1453,7 @@ HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value);
*/
htri_t
H5FS_sect_try_extend(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, haddr_t addr,
- hsize_t size, hsize_t extra_requested)
+ hsize_t size, hsize_t extra_requested, unsigned flags, void *op_data)
{
hbool_t sinfo_valid = FALSE; /* Whether the section info is valid */
hbool_t sinfo_modified = FALSE; /* Whether the section info was modified */
@@ -1528,10 +1537,16 @@ if(_section_)
/* Adjust section by amount requested */
sect->addr += extra_requested;
sect->size -= extra_requested;
-
- /* Re-add adjusted section to free sections data structures */
- if(H5FS_sect_link(fspace, sect, 0) < 0)
- HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't insert free space section into skip list")
+ if(cls->add)
+ if((*cls->add)(&sect, &flags, op_data) < 0)
+ HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "'add' section class callback failed")
+
+ /* Re-adding the section could cause it to disappear (particularly when paging) */
+ if(sect) {
+ /* Re-add adjusted section to free sections data structures */
+ if(H5FS_sect_link(fspace, sect, 0) < 0)
+ HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't insert free space section into skip list")
+ } /* end if */
} /* end if */
else {
/* Sanity check */
@@ -1667,7 +1682,7 @@ HDfprintf(stderr, "%s: fspace->sinfo->nbins = %u\n", FUNC, fspace->sinfo->nbins)
HDfprintf(stderr, "%s: bin = %u\n", FUNC, bin);
#endif /* QAK */
alignment = fspace->alignment;
- if(!((alignment > 1) && (request >= fspace->threshold)))
+ if(!((alignment > 1) && (request >= fspace->align_thres)))
alignment = 0; /* no alignment */
do {
@@ -2349,7 +2364,7 @@ HDfprintf(stderr, "%s: sect->size = %Hu, sect->addr = %a, sect->type = %u\n", "H
*-------------------------------------------------------------------------
*/
htri_t
-H5FS_sect_try_shrink_eoa(const H5F_t *f, hid_t dxpl_id, const H5FS_t *fspace, void *op_data)
+H5FS_sect_try_shrink_eoa(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, void *op_data)
{
hbool_t sinfo_valid = FALSE; /* Whether the section info is valid */
hbool_t section_removed = FALSE; /* Whether a section was removed */
@@ -2406,42 +2421,264 @@ done:
/*-------------------------------------------------------------------------
- * Function: H5FS_sect_query_last_sect
- *
- * Purpose: Retrieve info about the last section on the merge list
- *
- * Return: Success: non-negative
- * Failure: negative
+ * Function: H5FS_vfd_alloc_hdr_and_section_info_if_needed
+ *
+ * Purpose: This function is part of a hack to patch over a design
+ * flaw in the free space managers for file space allocation.
+ * Specifically, if a free space manager allocates space for
+ * its own section info, it is possible for it to
+ * go into an infinite loop as it:
+ *
+ * 1) computes the size of the section info
+ *
+ * 2) allocates file space for the section info
+ *
+ * 3) notices that the size of the section info
+ * has changed
+ *
+ * 4) deallocates the section info file space and
+ * returns to 1) above.
+ *
+ * Similarly, if it allocates space for its own header, it
+ * can go into an infinte loop as it:
+ *
+ * 1) allocates space for the header
+ *
+ * 2) notices that the free space manager is empty
+ * and thus should not have file space allocated
+ * to its header
+ *
+ * 3) frees the space allocated to the header
+ *
+ * 4) notices that the free space manager is not
+ * empty and thus must have file space allocated
+ * to it, and thus returns to 1) above.
+ *
+ * In a nutshell, the solution applied in this hack is to
+ * deallocate file space for the free space manager(s) that
+ * handle FSM header and/or section info file space allocations,
+ * wait until all other allocation/deallocation requests have
+ * been handled, and then test to see if the free space manager(s)
+ * in question are empty. If they are, do nothing. If they
+ * are not, allocate space for them at end of file bypassing the
+ * usual file space allocation calls, and thus avoiding the
+ * potential infinite loops.
+ *
+ * The purpose of this function is to allocate file space for
+ * the header and section info of the target free space manager
+ * directly from the VFD if needed. In this case the function
+ * also re-inserts the header and section info in the metadata
+ * cache with this allocation.
+ *
+ * When paged allocation is not enabled, allocation of space
+ * for the free space manager header and section info is
+ * straight forward -- we simply allocate the space directly
+ * from file driver.
+ *
+ * Note that if f->shared->alignment > 1, and EOA is not a
+ * multiple of the alignment, it is possible that performing
+ * these allocations may generate a fragment of file space in
+ * addition to the space allocated for the section info. This
+ * excess space is dropped on the floor. As shall be seen,
+ * it will usually be reclaimed later.
+ *
+ * When page allocation is enabled, things are more difficult,
+ * as there is the possibility that page buffering will be
+ * enabled when the free space managers are read. To allow
+ * for this, we must ensure that space allocated for the
+ * free space manager header and section info is either larger
+ * than a page, or resides completely withing a page.
+ *
+ * Do this by allocating space for the free space header and
+ * section info starting at page boundaries, and extending
+ * allocation to the next page boundary. This of course wastes
+ * space, but see below.
+ *
+ * On the first free space allocation / deallocation after the
+ * next file open, we will read the self referential free space
+ * managers, float them and reduce the EOA to its value prior
+ * to allocation of file space for the self referential free
+ * space managers on the preceeding file close. This EOA value
+ * is stored in the free space manager superblock extension
+ * message.
+ *
+ * Return: Success: non-negative
+ * Failure: negative
*
- * Programmer: Vailin Choi
+ * Programmer: John Mainzer
+ * 6/6/16
*
*-------------------------------------------------------------------------
*/
herr_t
-H5FS_sect_query_last_sect(const H5FS_t *fspace, haddr_t *sect_addr, hsize_t *sect_size)
+H5FS_vfd_alloc_hdr_and_section_info_if_needed(H5F_t *f, hid_t dxpl_id,
+ H5FS_t *fspace, haddr_t *fs_addr_ptr)
{
- FUNC_ENTER_NOAPI_NOINIT_NOERR
+ hsize_t hdr_alloc_size;
+ hsize_t sinfo_alloc_size;
+ haddr_t sect_addr = HADDR_UNDEF; /* address of sinfo */
+ haddr_t eoa_frag_addr = HADDR_UNDEF; /* Address of fragment at EOA */
+ hsize_t eoa_frag_size = 0; /* Size of fragment at EOA */
+ haddr_t eoa = HADDR_UNDEF; /* Initial EOA for the file */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI_NOINIT
/* Check arguments. */
+ HDassert(f);
+ HDassert(f->shared);
+ HDassert(f->shared->lf);
HDassert(fspace);
+ HDassert(fs_addr_ptr);
- if(fspace->sinfo && fspace->sinfo->merge_list) {
- H5SL_node_t *last_node; /* Last node in merge list */
+ /* the section info should be unlocked */
+ HDassert(fspace->sinfo_lock_count == 0);
- /* Check for last node in the merge list */
- if(NULL != (last_node = H5SL_last(fspace->sinfo->merge_list))) {
- H5FS_section_info_t *tmp_sect; /* Temporary free space section */
+ /* no space should be allocated */
+ HDassert(*fs_addr_ptr == HADDR_UNDEF);
+ HDassert(fspace->addr == HADDR_UNDEF);
+ HDassert(fspace->sect_addr == HADDR_UNDEF);
+ HDassert(fspace->alloc_sect_size == 0);
- /* Get the pointer to the last section, from the last node */
- tmp_sect = (H5FS_section_info_t *)H5SL_item(last_node);
- HDassert(tmp_sect);
- if(sect_addr)
- *sect_addr = tmp_sect->addr;
- if(sect_size)
- *sect_size = tmp_sect->size;
- } /* end if */
+ /* persistant free space managers must be enabled */
+ HDassert(f->shared->fs_persist);
+
+ /* At present, all free space strategies enable the free space managers.
+ * This will probably change -- at which point this assertion should
+ * be revisited.
+ */
+ /* Updated: Only the following two strategies enable the free-space managers */
+ HDassert((f->shared->fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR) ||
+ (f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE));
+
+ if(fspace->serial_sect_count > 0) {
+ /* the section info is floating, so space->sinfo should be defined */
+ HDassert(fspace->sinfo);
+
+ /* start by allocating file space for the header */
+
+ /* Get the EOA for the file -- need for sanity check below */
+ if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_FSPACE_HDR)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "Unable to get eoa")
+
+ /* check for overlap into temporary allocation space */
+ if(H5F_IS_TMP_ADDR(f, (eoa + fspace->sect_size)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, FAIL, "hdr file space alloc will overlap into 'temporary' file space")
+
+ hdr_alloc_size = H5FS_HEADER_SIZE(f);
+
+ /* if page allocation is enabled, extend the hdr_alloc_size to the
+ * next page boundary.
+ */
+ if(H5F_PAGED_AGGR(f)) {
+ HDassert(0 == (eoa % f->shared->fs_page_size));
+
+ hdr_alloc_size = ((hdr_alloc_size / f->shared->fs_page_size) + 1) * f->shared->fs_page_size;
+
+ HDassert(hdr_alloc_size >= H5FS_HEADER_SIZE(f));
+ HDassert((hdr_alloc_size % f->shared->fs_page_size) == 0);
+ } /* end if */
+
+ /* allocate space for the hdr */
+ if(HADDR_UNDEF == (fspace->addr = H5FD_alloc(f->shared->lf, dxpl_id,
+ H5FD_MEM_FSPACE_HDR, f,
+ hdr_alloc_size,
+ &eoa_frag_addr,
+ &eoa_frag_size)))
+ HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space for hdr")
+
+ /* if the file alignement is 1, there should be no
+ * eoa fragment. Otherwise, drop any fragment on the floor.
+ */
+ HDassert((eoa_frag_size == 0) || (f->shared->alignment != 1));
+
+ /* Cache the new free space header (pinned) */
+ if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_HDR, fspace->addr, fspace, H5AC__PIN_ENTRY_FLAG) < 0)
+ HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space header to cache")
+
+ *fs_addr_ptr = fspace->addr;
+
+ /* now allocate file space for the section info */
+
+ /* Get the EOA for the file -- need for sanity check below */
+ if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_FSPACE_SINFO)))
+ HGOTO_ERROR(H5E_FSPACE, H5E_CANTGET, FAIL, "Unable to get eoa")
+
+ /* check for overlap into temporary allocation space */
+ if(H5F_IS_TMP_ADDR(f, (eoa + fspace->sect_size)))
+ HGOTO_ERROR(H5E_FSPACE, H5E_BADRANGE, FAIL, "sinfo file space alloc will overlap into 'temporary' file space")
+
+ sinfo_alloc_size = fspace->sect_size;
+
+ /* if paged allocation is enabled, extend the sinfo_alloc_size to the
+ * next page boundary.
+ */
+ if(H5F_PAGED_AGGR(f)) {
+ HDassert(0 == (eoa % f->shared->fs_page_size));
+
+ sinfo_alloc_size = ((sinfo_alloc_size / f->shared->fs_page_size) + 1) * f->shared->fs_page_size;
+
+ HDassert(sinfo_alloc_size >= fspace->sect_size);
+ HDassert((sinfo_alloc_size % f->shared->fs_page_size) == 0);
+ } /* end if */
+
+ /* allocate space for the section info */
+ if(HADDR_UNDEF == (sect_addr = H5FD_alloc(f->shared->lf, dxpl_id,
+ H5FD_MEM_FSPACE_SINFO, f,
+ sinfo_alloc_size,
+ &eoa_frag_addr,
+ &eoa_frag_size)))
+ HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space")
+
+ /* if the file alignement is 1, there should be no
+ * eoa fragment. Otherwise, drop the fragment on the floor.
+ */
+ HDassert((eoa_frag_size == 0) || (f->shared->alignment != 1));
+
+ /* update fspace->alloc_sect_size and fspace->sect_addr to reflect
+ * the allocation
+ */
+ fspace->alloc_sect_size = fspace->sect_size;
+ fspace->sect_addr = sect_addr;
+
+ /* insert the new section info into the metadata cache. */
+
+ /* Question: Do we need to worry about this insertion causing an
+ * eviction from the metadata cache? Think on this. If so, add a
+ * flag to H5AC_insert() to force it to skip the call to make space in
+ * cache.
+ *
+ * On reflection, no.
+ *
+ * On a regular file close, any eviction will not change the
+ * the contents of the free space manger(s), as all entries
+ * should have correct file space allocated by the time this
+ * function is called.
+ *
+ * In the cache image case, the selection of entries for inclusion
+ * in the cache image will not take place until after this call.
+ * (Recall that this call is made during the metadata fsm settle
+ * routine, which is called during the serialization routine in
+ * the cache image case. Entries are not selected for inclusion
+ * in the image until after the cache is serialized.)
+ *
+ * JRM -- 11/4/16
+ */
+ if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_SINFO, sect_addr, fspace->sinfo, H5AC__NO_FLAGS_SET) < 0)
+ HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space sinfo to cache")
+
+ /* We have changed the sinfo address -- Mark free space header dirty */
+ if(H5AC_mark_entry_dirty(fspace) < 0)
+ HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty")
+
+ /* since space has been allocated for the section info and the sinfo
+ * has been inserted into the cache, relinquish owership (i.e. float)
+ * the section info.
+ */
+ fspace->sinfo = NULL;
} /* end if */
- FUNC_LEAVE_NOAPI(SUCCEED)
-} /* H5FS_sect_query_last_sect() */
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FS_vfd_alloc_hdr_and_section_info_if_needed() */
diff --git a/src/H5Faccum.c b/src/H5Faccum.c
index 48f9bdd..7f3bb39 100644
--- a/src/H5Faccum.c
+++ b/src/H5Faccum.c
@@ -112,18 +112,25 @@ H5FL_BLK_DEFINE_STATIC(meta_accum);
*-------------------------------------------------------------------------
*/
herr_t
-H5F__accum_read(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t addr,
+H5F__accum_read(const H5F_io_info2_t *fio_info, H5FD_mem_t map_type, haddr_t addr,
size_t size, void *buf/*out*/)
{
+ H5FD_io_info_t fdio_info; /* File driver I/O info */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
HDassert(fio_info);
HDassert(fio_info->f);
- HDassert(fio_info->dxpl);
+ HDassert(fio_info->meta_dxpl);
+ HDassert(fio_info->raw_dxpl);
HDassert(buf);
+ /* Translate to file driver I/O info object */
+ fdio_info.file = fio_info->f->shared->lf;
+ fdio_info.meta_dxpl = fio_info->meta_dxpl;
+ fdio_info.raw_dxpl = fio_info->raw_dxpl;
+
/* Check if this information is in the metadata accumulator */
if((fio_info->f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && map_type != H5FD_MEM_DRAW) {
H5F_meta_accum_t *accum; /* Alias for file's metadata accumulator */
@@ -178,7 +185,7 @@ H5F__accum_read(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t addr
accum->dirty_off += amount_before;
/* Dispatch to driver */
- if(H5FD_read(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, amount_before, accum->buf) < 0)
+ if(H5FD_read(&fdio_info, map_type, addr, amount_before, accum->buf) < 0)
HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed")
} /* end if */
else
@@ -192,7 +199,7 @@ H5F__accum_read(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t addr
H5_CHECKED_ASSIGN(amount_after, size_t, ((addr + size) - (accum->loc + accum->size)), hsize_t);
/* Dispatch to driver */
- if(H5FD_read(fio_info->f->shared->lf, fio_info->dxpl, map_type, (accum->loc + accum->size), amount_after, (accum->buf + accum->size + amount_before)) < 0)
+ if(H5FD_read(&fdio_info, map_type, (accum->loc + accum->size), amount_after, (accum->buf + accum->size + amount_before)) < 0)
HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed")
} /* end if */
@@ -206,13 +213,13 @@ H5F__accum_read(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t addr
/* Current read doesn't overlap with metadata accumulator, read it from file */
else {
/* Dispatch to driver */
- if(H5FD_read(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, size, buf) < 0)
+ if(H5FD_read(&fdio_info, map_type, addr, size, buf) < 0)
HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed")
} /* end else */
} /* end if */
else {
/* Read the data */
- if(H5FD_read(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, size, buf) < 0)
+ if(H5FD_read(&fdio_info, map_type, addr, size, buf) < 0)
HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed")
/* Check for overlap w/dirty accumulator */
@@ -255,7 +262,7 @@ H5F__accum_read(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t addr
} /* end if */
else {
/* Read the data */
- if(H5FD_read(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, size, buf) < 0)
+ if(H5FD_read(&fdio_info, map_type, addr, size, buf) < 0)
HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed")
} /* end else */
@@ -278,7 +285,7 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
-H5F__accum_adjust(H5F_meta_accum_t *accum, const H5F_io_info_t *fio_info,
+H5F__accum_adjust(H5F_meta_accum_t *accum, const H5FD_io_info_t *fdio_info,
H5F_accum_adjust_t adjust, size_t size)
{
herr_t ret_value = SUCCEED; /* Return value */
@@ -286,7 +293,7 @@ H5F__accum_adjust(H5F_meta_accum_t *accum, const H5F_io_info_t *fio_info,
FUNC_ENTER_STATIC
HDassert(accum);
- HDassert(fio_info);
+ HDassert(fdio_info);
HDassert(H5F_ACCUM_APPEND == adjust || H5F_ACCUM_PREPEND == adjust);
HDassert(size > 0);
HDassert(size <= H5F_ACCUM_MAX_SIZE);
@@ -344,7 +351,7 @@ H5F__accum_adjust(H5F_meta_accum_t *accum, const H5F_io_info_t *fio_info,
/* Check if the dirty region overlaps the region to eliminate from the accumulator */
if((accum->size - shrink_size) < (accum->dirty_off + accum->dirty_len)) {
/* Write out the dirty region from the metadata accumulator, with dispatch to driver */
- if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, (accum->loc + accum->dirty_off), accum->dirty_len, (accum->buf + accum->dirty_off)) < 0)
+ if(H5FD_write(fdio_info, H5FD_MEM_DEFAULT, (accum->loc + accum->dirty_off), accum->dirty_len, (accum->buf + accum->dirty_off)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "file write failed")
/* Reset accumulator dirty flag */
@@ -355,7 +362,7 @@ H5F__accum_adjust(H5F_meta_accum_t *accum, const H5F_io_info_t *fio_info,
/* Check if the dirty region overlaps the region to eliminate from the accumulator */
if(shrink_size > accum->dirty_off) {
/* Write out the dirty region from the metadata accumulator, with dispatch to driver */
- if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, (accum->loc + accum->dirty_off), accum->dirty_len, (accum->buf + accum->dirty_off)) < 0)
+ if(H5FD_write(fdio_info, H5FD_MEM_DEFAULT, (accum->loc + accum->dirty_off), accum->dirty_len, (accum->buf + accum->dirty_off)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "file write failed")
/* Reset accumulator dirty flag */
@@ -417,9 +424,10 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
-H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t addr,
+H5F__accum_write(const H5F_io_info2_t *fio_info, H5FD_mem_t map_type, haddr_t addr,
size_t size, const void *buf)
{
+ H5FD_io_info_t fdio_info; /* File driver I/O info */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
@@ -427,9 +435,15 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add
HDassert(fio_info);
HDassert(fio_info->f);
HDassert(H5F_INTENT(fio_info->f) & H5F_ACC_RDWR);
- HDassert(fio_info->dxpl);
+ HDassert(fio_info->meta_dxpl);
+ HDassert(fio_info->raw_dxpl);
HDassert(buf);
+ /* Translate to file driver I/O info object */
+ fdio_info.file = fio_info->f->shared->lf;
+ fdio_info.meta_dxpl = fio_info->meta_dxpl;
+ fdio_info.raw_dxpl = fio_info->raw_dxpl;
+
/* Check for accumulating metadata */
if((fio_info->f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && map_type != H5FD_MEM_DRAW) {
H5F_meta_accum_t *accum; /* Alias for file's metadata accumulator */
@@ -446,7 +460,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add
/* Check if the new metadata adjoins the beginning of the current accumulator */
if((addr + size) == accum->loc) {
/* Check if we need to adjust accumulator size */
- if(H5F__accum_adjust(accum, fio_info, H5F_ACCUM_PREPEND, size) < 0)
+ if(H5F__accum_adjust(accum, &fdio_info, H5F_ACCUM_PREPEND, size) < 0)
HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator")
/* Move the existing metadata to the proper location */
@@ -471,7 +485,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add
/* Check if the new metadata adjoins the end of the current accumulator */
else if(addr == (accum->loc + accum->size)) {
/* Check if we need to adjust accumulator size */
- if(H5F__accum_adjust(accum, fio_info, H5F_ACCUM_APPEND, size) < 0)
+ if(H5F__accum_adjust(accum, &fdio_info, H5F_ACCUM_APPEND, size) < 0)
HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator")
/* Copy the new metadata to the end */
@@ -531,7 +545,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add
H5_CHECKED_ASSIGN(add_size, size_t, (accum->loc - addr), hsize_t);
/* Check if we need to adjust accumulator size */
- if(H5F__accum_adjust(accum, fio_info, H5F_ACCUM_PREPEND, add_size) < 0)
+ if(H5F__accum_adjust(accum, &fdio_info, H5F_ACCUM_PREPEND, add_size) < 0)
HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator")
/* Calculate the proper offset of the existing metadata */
@@ -571,7 +585,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add
H5_CHECKED_ASSIGN(add_size, size_t, (addr + size) - (accum->loc + accum->size), hsize_t);
/* Check if we need to adjust accumulator size */
- if(H5F__accum_adjust(accum, fio_info, H5F_ACCUM_APPEND, add_size) < 0)
+ if(H5F__accum_adjust(accum, &fdio_info, H5F_ACCUM_APPEND, add_size) < 0)
HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator")
/* Compute offset of dirty region (after adjusting accumulator) */
@@ -637,7 +651,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add
else {
/* Write out the existing metadata accumulator, with dispatch to driver */
if(accum->dirty) {
- if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, accum->loc + accum->dirty_off, accum->dirty_len, accum->buf + accum->dirty_off) < 0)
+ if(H5FD_write(&fdio_info, H5FD_MEM_DEFAULT, accum->loc + accum->dirty_off, accum->dirty_len, accum->buf + accum->dirty_off) < 0)
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed")
/* Reset accumulator dirty flag */
@@ -733,7 +747,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add
HGOTO_ERROR(H5E_IO, H5E_CANTRESET, FAIL, "can't reset accumulator")
/* Write the data */
- if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, size, buf) < 0)
+ if(H5FD_write(&fdio_info, map_type, addr, size, buf) < 0)
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed")
/* Check for overlap w/accumulator */
@@ -818,7 +832,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add
} /* end if */
else {
/* Write the data */
- if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, size, buf) < 0)
+ if(H5FD_write(&fdio_info, map_type, addr, size, buf) < 0)
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed")
} /* end else */
@@ -842,10 +856,11 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
-H5F__accum_free(const H5F_io_info_t *fio_info, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr,
+H5F__accum_free(const H5F_io_info2_t *fio_info, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr,
hsize_t size)
{
H5F_meta_accum_t *accum; /* Alias for file's metadata accumulator */
+ H5FD_io_info_t fdio_info; /* File driver I/O info */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE
@@ -853,11 +868,17 @@ H5F__accum_free(const H5F_io_info_t *fio_info, H5FD_mem_t H5_ATTR_UNUSED type, h
/* check arguments */
HDassert(fio_info);
HDassert(fio_info->f);
- HDassert(fio_info->dxpl);
+ HDassert(fio_info->meta_dxpl);
+ HDassert(fio_info->raw_dxpl);
/* Set up alias for file's metadata accumulator info */
accum = &fio_info->f->shared->accum;
+ /* Translate to file driver I/O info object */
+ fdio_info.file = fio_info->f->shared->lf;
+ fdio_info.meta_dxpl = fio_info->meta_dxpl;
+ fdio_info.raw_dxpl = fio_info->raw_dxpl;
+
/* Adjust the metadata accumulator to remove the freed block, if it overlaps */
if((fio_info->f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA)
&& H5F_addr_overlap(addr, size, accum->loc, accum->size)) {
@@ -930,7 +951,7 @@ H5F__accum_free(const H5F_io_info_t *fio_info, H5FD_mem_t H5_ATTR_UNUSED type, h
/* Check if block to free is entirely before dirty region */
if(H5F_addr_le(tail_addr, dirty_start)) {
/* Write out the entire dirty region of the accumulator */
- if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, dirty_start, accum->dirty_len, accum->buf + accum->dirty_off) < 0)
+ if(H5FD_write(&fdio_info, H5FD_MEM_DEFAULT, dirty_start, accum->dirty_len, accum->buf + accum->dirty_off) < 0)
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed")
} /* end if */
/* Block to free overlaps with some/all of dirty region */
@@ -945,7 +966,7 @@ H5F__accum_free(const H5F_io_info_t *fio_info, H5FD_mem_t H5_ATTR_UNUSED type, h
HDassert(write_size > 0);
/* Write out the unfreed dirty region of the accumulator */
- if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, dirty_start + dirty_delta, write_size, accum->buf + accum->dirty_off + dirty_delta) < 0)
+ if(H5FD_write(&fdio_info, H5FD_MEM_DEFAULT, dirty_start + dirty_delta, write_size, accum->buf + accum->dirty_off + dirty_delta) < 0)
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed")
} /* end if */
@@ -965,7 +986,7 @@ H5F__accum_free(const H5F_io_info_t *fio_info, H5FD_mem_t H5_ATTR_UNUSED type, h
HDassert(write_size > 0);
/* Write out the unfreed end of the dirty region of the accumulator */
- if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, dirty_start + dirty_delta, write_size, accum->buf + accum->dirty_off + dirty_delta) < 0)
+ if(H5FD_write(&fdio_info, H5FD_MEM_DEFAULT, dirty_start + dirty_delta, write_size, accum->buf + accum->dirty_off + dirty_delta) < 0)
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed")
} /* end if */
@@ -1006,7 +1027,7 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
-H5F__accum_flush(const H5F_io_info_t *fio_info)
+H5F__accum_flush(const H5F_io_info2_t *fio_info)
{
herr_t ret_value = SUCCEED; /* Return value */
@@ -1014,12 +1035,20 @@ H5F__accum_flush(const H5F_io_info_t *fio_info)
HDassert(fio_info);
HDassert(fio_info->f);
- HDassert(fio_info->dxpl);
+ HDassert(fio_info->meta_dxpl);
+ HDassert(fio_info->raw_dxpl);
/* Check if we need to flush out the metadata accumulator */
if((fio_info->f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && fio_info->f->shared->accum.dirty) {
+ H5FD_io_info_t fdio_info; /* File driver I/O info */
+
+ /* Translate to file driver I/O info object */
+ fdio_info.file = fio_info->f->shared->lf;
+ fdio_info.meta_dxpl = fio_info->meta_dxpl;
+ fdio_info.raw_dxpl = fio_info->raw_dxpl;
+
/* Flush the metadata contents */
- if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, fio_info->f->shared->accum.loc + fio_info->f->shared->accum.dirty_off, fio_info->f->shared->accum.dirty_len, fio_info->f->shared->accum.buf + fio_info->f->shared->accum.dirty_off) < 0)
+ if(H5FD_write(&fdio_info, H5FD_MEM_DEFAULT, fio_info->f->shared->accum.loc + fio_info->f->shared->accum.dirty_off, fio_info->f->shared->accum.dirty_len, fio_info->f->shared->accum.buf + fio_info->f->shared->accum.dirty_off) < 0)
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed")
/* Reset the dirty flag */
@@ -1045,7 +1074,7 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
-H5F__accum_reset(const H5F_io_info_t *fio_info, hbool_t flush)
+H5F__accum_reset(const H5F_io_info2_t *fio_info, hbool_t flush)
{
herr_t ret_value = SUCCEED; /* Return value */
@@ -1053,7 +1082,6 @@ H5F__accum_reset(const H5F_io_info_t *fio_info, hbool_t flush)
HDassert(fio_info);
HDassert(fio_info->f);
- HDassert(fio_info->dxpl);
/* Flush any dirty data in accumulator, if requested */
if(flush)
diff --git a/src/H5Fint.c b/src/H5Fint.c
index d122357..794be50 100644
--- a/src/H5Fint.c
+++ b/src/H5Fint.c
@@ -76,8 +76,8 @@ typedef struct H5F_olist_t {
static int H5F_get_objects_cb(void *obj_ptr, hid_t obj_id, void *key);
static herr_t H5F_build_actual_name(const H5F_t *f, const H5P_genplist_t *fapl,
const char *name, char ** /*out*/ actual_name);/* Declare a free list to manage the H5F_t struct */
-static herr_t H5F__flush_phase1(H5F_t *f, hid_t dxpl_id);
-static herr_t H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing);
+static herr_t H5F__flush_phase1(H5F_t *f, hid_t meta_dxpl_id);
+static herr_t H5F__flush_phase2(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t closing);
/*********************/
@@ -182,6 +182,14 @@ H5F_get_access_plist(H5F_t *f, hbool_t app_ref)
efc_size = H5F_efc_max_nfiles(f->shared->efc);
if(H5P_set(new_plist, H5F_ACS_EFC_SIZE_NAME, &efc_size) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set elink file cache size")
+ if(f->shared->page_buf != NULL) {
+ if(H5P_set(new_plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &(f->shared->page_buf->max_size)) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set page buffer size")
+ if(H5P_set(new_plist, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, &(f->shared->page_buf->min_meta_perc)) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set minimum metadata fraction of page buffer")
+ if(H5P_set(new_plist, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, &(f->shared->page_buf->min_raw_perc)) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set minimum raw data fraction of page buffer")
+ } /* end if */
#ifdef H5_HAVE_PARALLEL
if(H5P_set(new_plist, H5_COLL_MD_READ_FLAG_NAME, &(f->coll_md_read)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set collective metadata read flag")
@@ -495,7 +503,7 @@ done:
/*-------------------------------------------------------------------------
- * Function: H5F_is_hdf5
+ * Function: H5F__is_hdf5
*
* Purpose: Check the file signature to detect an HDF5 file.
*
@@ -509,17 +517,14 @@ done:
*
* Programmer: Unknown
*
- * Modifications:
- * Robb Matzke, 1999-08-02
- * Rewritten to use the virtual file layer.
*-------------------------------------------------------------------------
*/
htri_t
-H5F_is_hdf5(const char *name, hid_t dxpl_id)
+H5F__is_hdf5(const char *name, hid_t meta_dxpl_id, hid_t raw_dxpl_id)
{
H5FD_t *file = NULL; /* Low-level file struct */
+ H5FD_io_info_t fdio_info; /* File driver I/O info */
haddr_t sig_addr; /* Addess of hdf5 file signature */
- H5P_genplist_t *xfer_plist= NULL; /* Dataset transfer property list object */
htri_t ret_value = FAIL; /* Return value */
FUNC_ENTER_NOAPI_NOINIT
@@ -528,12 +533,15 @@ H5F_is_hdf5(const char *name, hid_t dxpl_id)
if(NULL == (file = H5FD_open(name, H5F_ACC_RDONLY, H5P_FILE_ACCESS_DEFAULT, HADDR_UNDEF)))
HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to open file")
- /* Get the property list object */
- if(NULL == (xfer_plist = (H5P_genplist_t *)H5I_object(dxpl_id)))
+ /* Set up the file driver info */
+ fdio_info.file = file;
+ if(NULL == (fdio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id)))
+ HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get new property list object")
+ if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(raw_dxpl_id)))
HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get new property list object")
/* The file is an hdf5 file if the hdf5 file signature can be found */
- if(H5FD_locate_signature(file, xfer_plist, &sig_addr) < 0)
+ if(H5FD_locate_signature(&fdio_info, &sig_addr) < 0)
HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable to locate file signature")
ret_value = (HADDR_UNDEF != sig_addr);
@@ -544,7 +552,7 @@ done:
HDONE_ERROR(H5E_IO, H5E_CANTCLOSEFILE, FAIL, "unable to close file")
FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5F_is_hdf5() */
+} /* end H5F__is_hdf5() */
/*-------------------------------------------------------------------------
@@ -597,11 +605,26 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t
f->shared->flags = flags;
f->shared->sohm_addr = HADDR_UNDEF;
f->shared->sohm_vers = HDF5_SHAREDHEADER_VERSION;
- for(u = 0; u < NELMTS(f->shared->fs_addr); u++)
- f->shared->fs_addr[u] = HADDR_UNDEF;
f->shared->accum.loc = HADDR_UNDEF;
f->shared->lf = lf;
+ /* Initialization for handling file space */
+ for(u = 0; u < NELMTS(f->shared->fs_addr); u++) {
+ f->shared->fs_state[u] = H5F_FS_STATE_CLOSED;
+ f->shared->fs_addr[u] = HADDR_UNDEF;
+ f->shared->fs_man[u] = NULL;
+ } /* end for */
+ f->shared->first_alloc_dealloc = FALSE;
+ f->shared->eoa_pre_fsm_fsalloc = HADDR_UNDEF;
+ f->shared->eoa_post_fsm_fsalloc = HADDR_UNDEF;
+ f->shared->eoa_post_mdci_fsalloc = HADDR_UNDEF;
+
+ /* Initialization for handling file space (for paged aggregation) */
+ f->shared->pgend_meta_thres = H5F_FILE_SPACE_PGEND_META_THRES;
+
+ /* intialize point of no return */
+ f->shared->point_of_no_return = FALSE;
+
/*
* Copy the file creation and file access property lists into the
* new file handle. We do this early because some values might need
@@ -621,8 +644,19 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t
HDassert(f->shared->sohm_nindexes < 255);
if(H5P_get(plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &f->shared->fs_strategy) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get file space strategy")
+ if(H5P_get(plist, H5F_CRT_FREE_SPACE_PERSIST_NAME, &f->shared->fs_persist) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get file space persisting status")
if(H5P_get(plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &f->shared->fs_threshold) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get free-space section threshold")
+ if(H5P_get(plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, &f->shared->fs_page_size) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get file space page size")
+ HDassert(f->shared->fs_page_size >= H5F_FILE_SPACE_PAGE_SIZE_MIN);
+
+ /* Temporary for multi/split drivers: fail file creation
+ when persisting free-space or using paged aggregation strategy */
+ if(H5F_HAS_FEATURE(f, H5FD_FEAT_PAGED_AGGR))
+ if(f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE || f->shared->fs_persist)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't open with this strategy or persistent fs")
/* Get the FAPL values to cache */
if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id)))
@@ -796,7 +830,7 @@ done:
/*-------------------------------------------------------------------------
- * Function: H5F_dest
+ * Function: H5F__dest
*
* Purpose: Destroys a file structure. This function flushes the cache
* but doesn't do any other cleanup other than freeing memory
@@ -812,11 +846,11 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
-H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
+H5F__dest(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t flush)
{
herr_t ret_value = SUCCEED; /* Return value */
- FUNC_ENTER_NOAPI_NOINIT
+ FUNC_ENTER_PACKAGE
/* Sanity check */
HDassert(f);
@@ -824,14 +858,14 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
if(1 == f->shared->nrefs) {
int actype; /* metadata cache type (enum value) */
- H5F_io_info_t fio_info; /* I/O info for operation */
+ H5F_io_info2_t fio_info; /* I/O info for operation */
/* Flush at this point since the file will be closed (phase 1).
* Only try to flush the file if it was opened with write access, and if
* the caller requested a flush.
*/
if((H5F_ACC_RDWR & H5F_INTENT(f)) && flush)
- if(H5F__flush_phase1(f, dxpl_id) < 0)
+ if(H5F__flush_phase1(f, meta_dxpl_id) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush cached data (phase 1)")
@@ -839,7 +873,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
* This allows the cache to set up for creating a metadata cache
* image if this has been requested.
*/
- if(H5AC_prep_for_file_close(f, dxpl_id) < 0)
+ if(H5AC_prep_for_file_close(f, meta_dxpl_id) < 0)
/* Push error, but keep going */
HDONE_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "metadata cache prep for close failed")
@@ -848,7 +882,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
* the caller requested a flush.
*/
if((H5F_ACC_RDWR & H5F_INTENT(f)) && flush)
- if(H5F__flush_phase2(f, dxpl_id, TRUE) < 0)
+ if(H5F__flush_phase2(f, meta_dxpl_id, raw_dxpl_id, TRUE) < 0)
/* Push error, but keep going */
HDONE_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush cached data (phase 2)")
@@ -895,7 +929,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
* -- JRM
*/
if(H5F_ACC_RDWR & H5F_INTENT(f)) {
- if(H5MF_close(f, dxpl_id) < 0)
+ if(H5MF_close(f, meta_dxpl_id) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't release file free space info")
@@ -913,7 +947,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
f->shared->sblock->status_flags &= (uint8_t)(~H5F_SUPER_SWMR_WRITE_ACCESS);
/* Mark EOA info dirty in cache, so change will get encoded */
- if(H5F_eoa_dirty(f, dxpl_id) < 0)
+ if(H5F_eoa_dirty(f, meta_dxpl_id) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty")
@@ -924,12 +958,12 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
* At most, this should change the superblock or the
* superblock extension messages.
*/
- if(H5MF_free_aggrs(f, dxpl_id) < 0)
+ if(H5MF_free_aggrs(f, meta_dxpl_id) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't release file space")
/* Truncate the file to the current allocated size */
- if(H5FD_truncate(f->shared->lf, dxpl_id, TRUE) < 0)
+ if(H5FD_truncate(f->shared->lf, meta_dxpl_id, TRUE) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "low level truncate failed")
@@ -968,10 +1002,15 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing file")
/* Shutdown the metadata cache */
- if(H5AC_dest(f, dxpl_id))
+ if(H5AC_dest(f, meta_dxpl_id))
/* Push error, but keep going*/
HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing file")
+ /* Shutdown the page buffer cache */
+ if(H5PB_dest(f) < 0)
+ /* Push error, but keep going*/
+ HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing page buffer cache")
+
/* Clean up the metadata cache log location string */
if(f->shared->mdc_log_location)
f->shared->mdc_log_location = (char *)H5MM_xfree(f->shared->mdc_log_location);
@@ -990,7 +1029,9 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
/* Set up I/O info for operation */
fio_info.f = f;
- if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
+ if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id)))
+ HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id)))
HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
/* Destroy other components of the file */
@@ -1130,7 +1171,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush)
*/
H5F_t *
H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id,
- hid_t dxpl_id)
+ hid_t meta_dxpl_id)
{
H5F_t *file = NULL; /*the success return value */
H5F_file_t *shared = NULL; /*shared part of `file' */
@@ -1139,6 +1180,10 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id,
H5FD_class_t *drvr; /*file driver class info */
H5P_genplist_t *a_plist; /*file access property list */
H5F_close_degree_t fc_degree; /*file close degree */
+ hid_t raw_dxpl_id = H5AC_rawdata_dxpl_id; /* Raw data dxpl used by library */
+ size_t page_buf_size;
+ unsigned page_buf_min_meta_perc;
+ unsigned page_buf_min_raw_perc;
hbool_t set_flag = FALSE; /*set the status_flags in the superblock */
hbool_t clear = FALSE; /*clear the status_flags */
hbool_t evict_on_close; /* evict on close value from plist */
@@ -1289,6 +1334,25 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id,
if(NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list")
+ /* Check if page buffering is enabled */
+ if(H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &page_buf_size) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get page buffer size")
+ if(page_buf_size) {
+#ifdef H5_HAVE_PARALLEL
+ /* Collective metadata writes are not supported with page buffering */
+ if(file->coll_md_write)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "collective metadata writes are not supported with page buffering")
+
+ /* Temporary: fail file create when page buffering feature is enabled for parallel */
+ HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "page buffering is disabled for parallel")
+#endif /* H5_HAVE_PARALLEL */
+ /* Query for other page buffer cache properties */
+ if(H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, &page_buf_min_meta_perc) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get minimum metadata fraction of page buffer")
+ if(H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, &page_buf_min_raw_perc) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get minimum raw data fraction of page buffer")
+ } /* end if */
+
/*
* Read or write the file superblock, depending on whether the file is
* empty or not.
@@ -1299,26 +1363,35 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id,
* to create & write the superblock.
*/
+ /* Create the page buffer before initializing the superblock */
+ if(page_buf_size)
+ if(H5PB_create(file, page_buf_size, page_buf_min_meta_perc, page_buf_min_raw_perc) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create page buffer")
+
/* Initialize information about the superblock and allocate space for it */
/* (Writes superblock extension messages, if there are any) */
- if(H5F__super_init(file, dxpl_id) < 0)
+ if(H5F__super_init(file, meta_dxpl_id) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to allocate file superblock")
/* Create and open the root group */
/* (This must be after the space for the superblock is allocated in
* the file, since the superblock must be at offset 0)
*/
- if(H5G_mkroot(file, dxpl_id, TRUE) < 0)
+ if(H5G_mkroot(file, meta_dxpl_id, TRUE) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create/open root group")
} /* end if */
else if (1 == shared->nrefs) {
-
/* Read the superblock if it hasn't been read before. */
- if(H5F__super_read(file, dxpl_id, TRUE) < 0)
+ if(H5F__super_read(file, meta_dxpl_id, raw_dxpl_id, TRUE) < 0)
HGOTO_ERROR(H5E_FILE, H5E_READERROR, NULL, "unable to read superblock")
+ /* Create the page buffer before initializing the superblock */
+ if(page_buf_size)
+ if(H5PB_create(file, page_buf_size, page_buf_min_meta_perc, page_buf_min_raw_perc) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create page buffer")
+
/* Open the root group */
- if(H5G_mkroot(file, dxpl_id, FALSE) < 0)
+ if(H5G_mkroot(file, meta_dxpl_id, FALSE) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to read root group")
} /* end if */
@@ -1360,12 +1433,11 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id,
*/
if(H5P_get(a_plist, H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME, &evict_on_close) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get evict on close value")
-
- if(shared->nrefs == 1) {
+ if(shared->nrefs == 1)
shared->evict_on_close = evict_on_close;
- } else if(shared->nrefs > 1) {
+ else if(shared->nrefs > 1) {
if(shared->evict_on_close != evict_on_close)
- HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "file evict-on-close value doesn't match")
+ HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "file evict-on-close value doesn't match")
} /* end if */
/* Formulate the absolute path for later search of target file for external links */
@@ -1393,7 +1465,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id,
/* Flush the superblock */
if(H5F_super_dirty(file) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, NULL, "unable to mark superblock as dirty")
- if(H5F_flush_tagged_metadata(file, H5AC__SUPERBLOCK_TAG, H5AC_ind_read_dxpl_id) < 0)
+ if(H5F_flush_tagged_metadata(file, H5AC__SUPERBLOCK_TAG, meta_dxpl_id) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, NULL, "unable to flush superblock")
/* Remove the file lock for SWMR_WRITE */
@@ -1405,7 +1477,6 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id,
else { /* H5F_ACC_RDONLY: check consistency of status_flags */
/* Skip check of status_flags for file with < superblock version 3 */
if(file->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_3) {
-
if(H5F_INTENT(file) & H5F_ACC_SWMR_READ) {
if((file->shared->sblock->status_flags & H5F_SUPER_WRITE_ACCESS &&
!(file->shared->sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS))
@@ -1413,12 +1484,10 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id,
(!(file->shared->sblock->status_flags & H5F_SUPER_WRITE_ACCESS) &&
file->shared->sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS))
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "file is not already open for SWMR writing")
-
} /* end if */
else if((file->shared->sblock->status_flags & H5F_SUPER_WRITE_ACCESS) ||
(file->shared->sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS))
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "file is already open for write (may use <h5clear file> to clear file consistency flags)")
-
} /* version 3 superblock */
} /* end else */
} /* end if set_flag */
@@ -1428,7 +1497,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id,
done:
if((NULL == ret_value) && file)
- if(H5F_dest(file, dxpl_id, FALSE) < 0)
+ if(H5F__dest(file, meta_dxpl_id, raw_dxpl_id, FALSE) < 0)
HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, NULL, "problems closing file")
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F_open() */
@@ -1448,7 +1517,7 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
-H5F__flush_phase1(H5F_t *f, hid_t dxpl_id)
+H5F__flush_phase1(H5F_t *f, hid_t meta_dxpl_id)
{
herr_t ret_value = SUCCEED; /* Return value */
@@ -1458,7 +1527,7 @@ H5F__flush_phase1(H5F_t *f, hid_t dxpl_id)
HDassert(f);
/* Flush any cached dataset storage raw data */
- if(H5D_flush(f, dxpl_id) < 0)
+ if(H5D_flush(f, meta_dxpl_id) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush dataset cache")
@@ -1468,7 +1537,7 @@ H5F__flush_phase1(H5F_t *f, hid_t dxpl_id)
/* (needs to happen before cache flush, with superblock write, since the
* 'eoa' value is written in superblock -QAK)
*/
- if(H5MF_free_aggrs(f, dxpl_id) < 0)
+ if(H5MF_free_aggrs(f, meta_dxpl_id) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't release file space")
@@ -1490,9 +1559,9 @@ H5F__flush_phase1(H5F_t *f, hid_t dxpl_id)
*-------------------------------------------------------------------------
*/
static herr_t
-H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing)
+H5F__flush_phase2(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t closing)
{
- H5F_io_info_t fio_info; /* I/O info for operation */
+ H5F_io_info2_t fio_info; /* I/O info for operation */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
@@ -1501,23 +1570,26 @@ H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing)
HDassert(f);
/* Flush the entire metadata cache */
- if(H5AC_flush(f, dxpl_id) < 0)
+ if(H5AC_flush(f, meta_dxpl_id) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush metadata cache")
/* Truncate the file to the current allocated size */
- if(H5FD_truncate(f->shared->lf, dxpl_id, closing) < 0)
+ if(H5FD_truncate(f->shared->lf, meta_dxpl_id, closing) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "low level truncate failed")
/* Flush the entire metadata cache again since the EOA could have changed in the truncate call. */
- if(H5AC_flush(f, dxpl_id) < 0)
+ if(H5AC_flush(f, meta_dxpl_id) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush metadata cache")
/* Set up I/O info for operation */
fio_info.f = f;
- if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
+ if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id)))
+ /* Push error, but keep going*/
+ HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(raw_dxpl_id)))
/* Push error, but keep going*/
HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
@@ -1526,8 +1598,13 @@ H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing)
/* Push error, but keep going*/
HDONE_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "unable to flush metadata accumulator")
+ /* Flush the page buffer */
+ if(H5PB_flush(&fio_info) < 0)
+ /* Push error, but keep going*/
+ HDONE_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "page buffer flush failed")
+
/* Flush file buffers to disk. */
- if(H5FD_flush(f->shared->lf, dxpl_id, closing) < 0)
+ if(H5FD_flush(f->shared->lf, meta_dxpl_id, closing) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "low level flush failed")
@@ -1536,7 +1613,7 @@ H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing)
/*-------------------------------------------------------------------------
- * Function: H5F_flush
+ * Function: H5F__flush
*
* Purpose: Flushes cached data.
*
@@ -1549,28 +1626,27 @@ H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing)
*-------------------------------------------------------------------------
*/
herr_t
-H5F_flush(H5F_t *f, hid_t dxpl_id, hbool_t closing)
+H5F__flush(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t closing)
{
herr_t ret_value = SUCCEED; /* Return value */
- FUNC_ENTER_NOAPI(FAIL)
+ FUNC_ENTER_PACKAGE
/* Sanity check arguments */
HDassert(f);
/* First phase of flushing data */
- if(H5F__flush_phase1(f, dxpl_id) < 0)
+ if(H5F__flush_phase1(f, meta_dxpl_id) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush file data")
/* Second phase of flushing data */
- if(H5F__flush_phase2(f, dxpl_id, closing) < 0)
+ if(H5F__flush_phase2(f, meta_dxpl_id, raw_dxpl_id, closing) < 0)
/* Push error, but keep going*/
HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush file data")
-done:
FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5F_flush() */
+} /* end H5F__flush() */
/*-------------------------------------------------------------------------
@@ -1795,7 +1871,7 @@ H5F_try_close(H5F_t *f, hbool_t *was_closed /*out*/)
if(H5F_efc_try_close(f) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't attempt to close EFC")
- /* Delay flush until the shared file struct is closed, in H5F_dest. If the
+ /* Delay flush until the shared file struct is closed, in H5F__dest. If the
* application called H5Fclose, it would have been flushed in that function
* (unless it will have been flushed in H5F_dest anyways). */
@@ -1804,7 +1880,7 @@ H5F_try_close(H5F_t *f, hbool_t *was_closed /*out*/)
* shared H5F_file_t struct. If the reference count for the H5F_file_t
* struct reaches zero then destroy it also.
*/
- if(H5F_dest(f, H5AC_ind_read_dxpl_id, TRUE) < 0)
+ if(H5F__dest(f, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, TRUE) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "problems closing file")
/* Since we closed the file, this should be set to TRUE */
@@ -2362,7 +2438,8 @@ H5F_set_store_msg_crt_idx(H5F_t *f, hbool_t flag)
*-------------------------------------------------------------------------
*/
ssize_t
-H5F_get_file_image(H5F_t *file, void *buf_ptr, size_t buf_len, hid_t dxpl_id)
+H5F_get_file_image(H5F_t *file, void *buf_ptr, size_t buf_len, hid_t meta_dxpl_id,
+ hid_t raw_dxpl_id)
{
H5FD_t *fd_ptr; /* file driver */
haddr_t eoa; /* End of file address */
@@ -2429,10 +2506,10 @@ H5F_get_file_image(H5F_t *file, void *buf_ptr, size_t buf_len, hid_t dxpl_id)
/* test to see if a buffer was provided -- if not, we are done */
if(buf_ptr != NULL) {
+ H5FD_io_info_t fdio_info; /* File driver I/O info */
size_t space_needed; /* size of file image */
hsize_t tmp;
size_t tmp_size;
- H5P_genplist_t *xfer_plist= NULL; /* Dataset transfer property list object */
/* Check for buffer too small */
if((haddr_t)buf_len < eoa)
@@ -2440,13 +2517,16 @@ H5F_get_file_image(H5F_t *file, void *buf_ptr, size_t buf_len, hid_t dxpl_id)
space_needed = (size_t)eoa;
- /* Get the property list object */
- if(NULL == (xfer_plist = (H5P_genplist_t *)H5I_object(dxpl_id)))
- HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get new property list object")
+ /* Set up file driver I/O info object */
+ fdio_info.file = fd_ptr;
+ if(NULL == (fdio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id)))
+ HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get property list object")
+ if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(raw_dxpl_id)))
+ HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get property list object")
/* read in the file image */
/* (Note compensation for base address addition in internal routine) */
- if(H5FD_read(fd_ptr, xfer_plist, H5FD_MEM_DEFAULT, 0, space_needed, buf_ptr) < 0)
+ if(H5FD_read(&fdio_info, H5FD_MEM_DEFAULT, 0, space_needed, buf_ptr) < 0)
HGOTO_ERROR(H5E_FILE, H5E_READERROR, FAIL, "file image read request failed")
/* Offset to "status_flags" in the superblock */
@@ -2648,6 +2728,38 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5F__set_eoa() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5F__set_paged_aggr
+ *
+ * Purpose: Quick and dirty routine to set the file's paged_aggr mode
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Quincey Koziol <koziol@hdfgroup.org>
+ * June 19, 2015
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5F__set_paged_aggr(const H5F_t *f, hbool_t paged)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE
+
+ /* Sanity check */
+ HDassert(f);
+ HDassert(f->shared);
+
+ /* Dispatch to driver */
+ if(H5FD_set_paged_aggr(f->shared->lf, paged) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "driver set paged aggr mode failed")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5F__set_paged_aggr() */
+
#ifdef H5_HAVE_PARALLEL
/*-------------------------------------------------------------------------
diff --git a/src/H5Fio.c b/src/H5Fio.c
index afe1278..6d23995 100644
--- a/src/H5Fio.c
+++ b/src/H5Fio.c
@@ -39,6 +39,7 @@
#include "H5Fpkg.h" /* File access */
#include "H5FDprivate.h" /* File drivers */
#include "H5Iprivate.h" /* IDs */
+#include "H5PBprivate.h" /* Page Buffer */
/****************/
@@ -96,15 +97,11 @@ herr_t
H5F_block_read(const H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size,
hid_t dxpl_id, void *buf/*out*/)
{
- H5F_io_info_t fio_info; /* I/O info for operation */
+ H5F_io_info2_t fio_info; /* I/O info for operation */
H5FD_mem_t map_type; /* Mapped memory type */
- hid_t my_dxpl_id = dxpl_id; /* transfer property to use for I/O */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
-#ifdef QAK
-HDfprintf(stderr, "%s: read from addr = %a, size = %Zu\n", FUNC, addr, size);
-#endif /* QAK */
HDassert(f);
HDassert(f->shared);
@@ -118,20 +115,24 @@ HDfprintf(stderr, "%s: read from addr = %a, size = %Zu\n", FUNC, addr, size);
/* Treat global heap as raw data */
map_type = (type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type;
-#ifdef H5_DEBUG_BUILD
- /* GHEAP type is treated as RAW, so update the dxpl type property too */
- if(H5FD_MEM_GHEAP == type)
- my_dxpl_id = H5AC_rawdata_dxpl_id;
-#endif /* H5_DEBUG_BUILD */
-
- /* Set up I/O info for operation */
+ /* Set up the I/O info object */
fio_info.f = f;
- if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(my_dxpl_id)))
- HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ if(H5FD_MEM_DRAW == type) {
+ if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ } /* end if */
+ else {
+ if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ } /* end else */
- /* Pass through metadata accumulator layer */
- if(H5F__accum_read(&fio_info, map_type, addr, size, buf) < 0)
- HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "read through metadata accumulator failed")
+ /* Pass through page buffer layer */
+ if(H5PB_read(&fio_info, map_type, addr, size, buf) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "read through page buffer failed")
done:
FUNC_LEAVE_NOAPI(ret_value)
@@ -157,15 +158,11 @@ herr_t
H5F_block_write(const H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size,
hid_t dxpl_id, const void *buf)
{
- H5F_io_info_t fio_info; /* I/O info for operation */
+ H5F_io_info2_t fio_info; /* I/O info for operation */
H5FD_mem_t map_type; /* Mapped memory type */
- hid_t my_dxpl_id = dxpl_id; /* transfer property to use for I/O */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
-#ifdef QAK
-HDfprintf(stderr, "%s: write to addr = %a, size = %Zu\n", FUNC, addr, size);
-#endif /* QAK */
HDassert(f);
HDassert(f->shared);
@@ -180,20 +177,24 @@ HDfprintf(stderr, "%s: write to addr = %a, size = %Zu\n", FUNC, addr, size);
/* Treat global heap as raw data */
map_type = (type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type;
-#ifdef H5_DEBUG_BUILD
- /* GHEAP type is treated as RAW, so update the dxpl type property too */
- if(H5FD_MEM_GHEAP == type)
- my_dxpl_id = H5AC_rawdata_dxpl_id;
-#endif /* H5_DEBUG_BUILD */
-
- /* Set up I/O info for operation */
+ /* Set up the I/O info object */
fio_info.f = f;
- if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(my_dxpl_id)))
- HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ if(H5FD_MEM_DRAW == type) {
+ if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ } /* end if */
+ else {
+ if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ } /* end else */
- /* Pass through metadata accumulator layer */
- if(H5F__accum_write(&fio_info, map_type, addr, size, buf) < 0)
- HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write through metadata accumulator failed")
+ /* Pass through page buffer layer */
+ if(H5PB_write(&fio_info, map_type, addr, size, buf) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write through page buffer failed")
done:
FUNC_LEAVE_NOAPI(ret_value)
@@ -216,7 +217,7 @@ done:
herr_t
H5F_flush_tagged_metadata(H5F_t * f, haddr_t tag, hid_t dxpl_id)
{
- H5F_io_info_t fio_info; /* I/O info for operation */
+ H5F_io_info2_t fio_info; /* I/O info for operation */
herr_t ret_value = SUCCEED;
FUNC_ENTER_NOAPI(FAIL)
@@ -227,10 +228,10 @@ H5F_flush_tagged_metadata(H5F_t * f, haddr_t tag, hid_t dxpl_id)
/* Set up I/O info for operation */
fio_info.f = f;
-
- if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
+ if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
-
/* Flush and reset the accumulator */
if(H5F__accum_reset(&fio_info, TRUE) < 0)
diff --git a/src/H5Fmount.c b/src/H5Fmount.c
index e3d4952..859b9d6 100644
--- a/src/H5Fmount.c
+++ b/src/H5Fmount.c
@@ -614,7 +614,7 @@ H5F_mount_count_ids(H5F_t *f, unsigned *nopen_files, unsigned *nopen_objs)
*-------------------------------------------------------------------------
*/
static herr_t
-H5F_flush_mounts_recurse(H5F_t *f, hid_t dxpl_id)
+H5F_flush_mounts_recurse(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id)
{
unsigned nerrors = 0; /* Errors from recursive flushes */
unsigned u; /* Index variable */
@@ -627,11 +627,11 @@ H5F_flush_mounts_recurse(H5F_t *f, hid_t dxpl_id)
/* Flush all child files, not stopping for errors */
for(u = 0; u < f->shared->mtab.nmounts; u++)
- if(H5F_flush_mounts_recurse(f->shared->mtab.child[u].file, dxpl_id) < 0)
+ if(H5F_flush_mounts_recurse(f->shared->mtab.child[u].file, meta_dxpl_id, raw_dxpl_id) < 0)
nerrors++;
/* Call the "real" flush routine, for this file */
- if(H5F_flush(f, dxpl_id, FALSE) < 0)
+ if(H5F__flush(f, meta_dxpl_id, raw_dxpl_id, FALSE) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file's cached information")
/* Check flush errors for children - errors are already on the stack */
@@ -656,7 +656,7 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
-H5F_flush_mounts(H5F_t *f, hid_t dxpl_id)
+H5F_flush_mounts(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id)
{
herr_t ret_value = SUCCEED; /* Return value */
@@ -670,7 +670,7 @@ H5F_flush_mounts(H5F_t *f, hid_t dxpl_id)
f = f->parent;
/* Flush the mounted file hierarchy */
- if(H5F_flush_mounts_recurse(f, dxpl_id) < 0)
+ if(H5F_flush_mounts_recurse(f, meta_dxpl_id, raw_dxpl_id) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush mounted file hierarchy")
done:
diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h
index 93a3978..4dd9e20 100644
--- a/src/H5Fpkg.h
+++ b/src/H5Fpkg.h
@@ -42,6 +42,7 @@
#include "H5FSprivate.h" /* File free space */
#include "H5Gprivate.h" /* Groups */
#include "H5Oprivate.h" /* Object header messages */
+#include "H5PBprivate.h" /* Page buffer */
#include "H5UCprivate.h" /* Reference counted object functions */
@@ -68,8 +69,8 @@
/* Macro to abstract checking whether file is using a free space manager */
#define H5F_HAVE_FREE_SPACE_MANAGER(F) \
- ((F)->shared->fs_strategy == H5F_FILE_SPACE_ALL || \
- (F)->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST)
+ ((F)->shared->fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR || \
+ (F)->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE)
/* Macros for encoding/decoding superblock */
#define H5F_MAX_DRVINFOBLOCK_SIZE 1024 /* Maximum size of superblock driver info buffer */
@@ -200,13 +201,6 @@ typedef struct H5F_meta_accum_t {
hbool_t dirty; /* Flag to indicate that the accumulated metadata is dirty */
} H5F_meta_accum_t;
-/* Enum for free space manager state */
-typedef enum H5F_fs_state_t {
- H5F_FS_STATE_CLOSED, /* Free space manager is closed */
- H5F_FS_STATE_OPEN, /* Free space manager has been opened */
- H5F_FS_STATE_DELETING /* Free space manager is being deleted */
-} H5F_fs_state_t;
-
/* A record of the mount table */
typedef struct H5F_mount_t {
struct H5G_t *group; /* Mount point group held open */
@@ -281,6 +275,7 @@ struct H5F_file_t {
unsigned long feature_flags; /* VFL Driver feature Flags */
haddr_t maxaddr; /* Maximum address for file */
+ H5PB_t *page_buf; /* The page buffer cache */
H5AC_t *cache; /* The object cache */
H5AC_cache_config_t
mdc_initCacheCfg; /* initial configuration for the */
@@ -316,19 +311,38 @@ struct H5F_file_t {
H5UC_t *grp_btree_shared; /* Ref-counted group B-tree node info */
/* File space allocation information */
- H5F_file_space_type_t fs_strategy; /* File space handling strategy */
+ H5F_fspace_strategy_t fs_strategy; /* File space handling strategy */
hsize_t fs_threshold; /* Free space section threshold */
+ hbool_t fs_persist; /* Free-space persist or not */
hbool_t use_tmp_space; /* Whether temp. file space allocation is allowed */
haddr_t tmp_addr; /* Next address to use for temp. space in the file */
+ hbool_t point_of_no_return; /* flag to indicate that we can't go back and delete a freespace header when it's used up */
+
+ H5F_fs_state_t fs_state[H5F_MEM_PAGE_NTYPES]; /* State of free space manager for each type */
+ haddr_t fs_addr[H5F_MEM_PAGE_NTYPES]; /* Address of free space manager info for each type */
+ H5FS_t *fs_man[H5F_MEM_PAGE_NTYPES]; /* Free space manager for each file space type */
+ hbool_t first_alloc_dealloc; /* TRUE iff free space managers */
+ /* are persistant and have not */
+ /* been used accessed for either */
+ /* allocation or deallocation */
+ /* since file open. */
+ haddr_t eoa_pre_fsm_fsalloc; /* eoa pre file space allocation */
+ /* for self referential FSMs */
+ haddr_t eoa_post_fsm_fsalloc; /* eoa post file space allocation */
+ /* for self referential FSMs */
+ haddr_t eoa_post_mdci_fsalloc; /* eoa past file space allocation */
+ /* for metadata cache image, or */
+ /* HADDR_UNDEF if no cache image. */
+
+ /* Free-space aggregation info */
unsigned fs_aggr_merge[H5FD_MEM_NTYPES]; /* Flags for whether free space can merge with aggregator(s) */
- H5F_fs_state_t fs_state[H5FD_MEM_NTYPES]; /* State of free space manager for each type */
- haddr_t fs_addr[H5FD_MEM_NTYPES]; /* Address of free space manager info for each type */
- H5FS_t *fs_man[H5FD_MEM_NTYPES]; /* Free space manager for each file space type */
- H5FD_mem_t fs_type_map[H5FD_MEM_NTYPES]; /* Mapping of "real" file space type into tracked type */
- H5F_blk_aggr_t meta_aggr; /* Metadata aggregation info */
- /* (if aggregating metadata allocations) */
- H5F_blk_aggr_t sdata_aggr; /* "Small data" aggregation info */
- /* (if aggregating "small data" allocations) */
+ H5FD_mem_t fs_type_map[H5FD_MEM_NTYPES]; /* Mapping of "real" file space type into tracked type */
+ H5F_blk_aggr_t meta_aggr; /* Metadata aggregation info (if aggregating metadata allocations) */
+ H5F_blk_aggr_t sdata_aggr; /* "Small data" aggregation info (if aggregating "small data" allocations) */
+
+ /* Paged aggregation info */
+ hsize_t fs_page_size; /* File space page size */
+ size_t pgend_meta_thres; /* Do not track page end meta section <= this threshold */
/* Metadata accumulator information */
H5F_meta_accum_t accum; /* Metadata accumulator info */
@@ -383,11 +397,12 @@ H5FL_EXTERN(H5F_file_t);
/* General routines */
H5F_t *H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id,
hid_t fapl_id, H5FD_t *lf);
-herr_t H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush);
-H5_DLL herr_t H5F_flush(H5F_t *f, hid_t dxpl_id, hbool_t closing);
-H5_DLL htri_t H5F_is_hdf5(const char *name, hid_t dxpl_id);
+H5_DLL herr_t H5F__dest(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t flush);
+H5_DLL herr_t H5F__flush(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t closing);
+H5_DLL htri_t H5F__is_hdf5(const char *name, hid_t meta_dxpl_id, hid_t raw_dxpl_id);
H5_DLL herr_t H5F_get_objects(const H5F_t *f, unsigned types, size_t max_index, hid_t *obj_id_list, hbool_t app_ref, size_t *obj_id_count_ptr);
-H5_DLL ssize_t H5F_get_file_image(H5F_t *f, void *buf_ptr, size_t buf_len, hid_t dxpl_id);
+H5_DLL ssize_t H5F_get_file_image(H5F_t *f, void *buf_ptr, size_t buf_len,
+ hid_t meta_dxpl_id, hid_t raw_dxpl_id);
H5_DLL herr_t H5F_close(H5F_t *f);
/* File mount related routines */
@@ -397,7 +412,8 @@ H5_DLL herr_t H5F_mount_count_ids(H5F_t *f, unsigned *nopen_files, unsigned *nop
/* Superblock related routines */
H5_DLL herr_t H5F__super_init(H5F_t *f, hid_t dxpl_id);
-H5_DLL herr_t H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read);
+H5_DLL herr_t H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id,
+ hbool_t initial_read);
H5_DLL herr_t H5F__super_size(H5F_t *f, hid_t dxpl_id, hsize_t *super_size, hsize_t *super_ext_size);
H5_DLL herr_t H5F__super_free(H5F_super_t *sblock);
@@ -410,14 +426,14 @@ H5_DLL herr_t H5F_super_ext_close(H5F_t *f, H5O_loc_t *ext_ptr, hid_t dxpl_id,
hbool_t was_created);
/* Metadata accumulator routines */
-H5_DLL herr_t H5F__accum_read(const H5F_io_info_t *fio_info, H5FD_mem_t type,
+H5_DLL herr_t H5F__accum_read(const H5F_io_info2_t *fio_info, H5FD_mem_t type,
haddr_t addr, size_t size, void *buf);
-H5_DLL herr_t H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t type,
+H5_DLL herr_t H5F__accum_write(const H5F_io_info2_t *fio_info, H5FD_mem_t type,
haddr_t addr, size_t size, const void *buf);
-H5_DLL herr_t H5F__accum_free(const H5F_io_info_t *fio_info, H5FD_mem_t type,
+H5_DLL herr_t H5F__accum_free(const H5F_io_info2_t *fio_info, H5FD_mem_t type,
haddr_t addr, hsize_t size);
-H5_DLL herr_t H5F__accum_flush(const H5F_io_info_t *fio_info);
-H5_DLL herr_t H5F__accum_reset(const H5F_io_info_t *fio_info, hbool_t flush);
+H5_DLL herr_t H5F__accum_flush(const H5F_io_info2_t *fio_info);
+H5_DLL herr_t H5F__accum_reset(const H5F_io_info2_t *fio_info, hbool_t flush);
/* Shared file list related routines */
H5_DLL herr_t H5F_sfile_add(H5F_file_t *shared);
@@ -431,9 +447,16 @@ H5_DLL herr_t H5F_efc_release(H5F_efc_t *efc);
H5_DLL herr_t H5F_efc_destroy(H5F_efc_t *efc);
H5_DLL herr_t H5F_efc_try_close(H5F_t *f);
+/* Space allocation routines */
+H5_DLL haddr_t H5F_alloc(H5F_t *f, hid_t dxpl_id, H5F_mem_t type, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size);
+H5_DLL herr_t H5F_free(H5F_t *f, hid_t dxpl_id, H5F_mem_t type, haddr_t addr, hsize_t size);
+H5_DLL htri_t H5F_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type,
+ haddr_t blk_end, hsize_t extra_requested);
+
/* Functions that get/retrieve values from VFD layer */
H5_DLL herr_t H5F__set_eoa(const H5F_t *f, H5F_mem_t type, haddr_t addr);
H5_DLL herr_t H5F__set_base_addr(const H5F_t *f, haddr_t addr);
+H5_DLL herr_t H5F__set_paged_aggr(const H5F_t *f, hbool_t paged);
/* Functions that flush or evict */
H5_DLL herr_t H5F__evict_cache_entries(H5F_t *f, hid_t dxpl_id);
diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
index 7d288fa..886063a 100644
--- a/src/H5Fprivate.h
+++ b/src/H5Fprivate.h
@@ -315,8 +315,8 @@
#define H5F_SET_STORE_MSG_CRT_IDX(F, FL) ((F)->shared->store_msg_crt_idx = (FL))
#define H5F_GRP_BTREE_SHARED(F) ((F)->shared->grp_btree_shared)
#define H5F_SET_GRP_BTREE_SHARED(F, RC) (((F)->shared->grp_btree_shared = (RC)) ? SUCCEED : FAIL)
-#define H5F_USE_TMP_SPACE(F) ((F)->shared->use_tmp_space)
-#define H5F_IS_TMP_ADDR(F, ADDR) (H5F_addr_le((F)->shared->tmp_addr, (ADDR)))
+#define H5F_USE_TMP_SPACE(F) ((F)->shared->fs.use_tmp_space)
+#define H5F_IS_TMP_ADDR(F, ADDR) (H5F_addr_le((F)->shared->fs.tmp_addr, (ADDR)))
#define H5F_SET_LATEST_FLAGS(F, FL) ((F)->shared->latest_flags = (FL))
#ifdef H5_HAVE_PARALLEL
#define H5F_COLL_MD_READ(F) ((F)->coll_md_read)
@@ -324,6 +324,12 @@
#define H5F_USE_MDC_LOGGING(F) ((F)->shared->use_mdc_logging)
#define H5F_START_MDC_LOG_ON_ACCESS(F) ((F)->shared->start_mdc_log_on_access)
#define H5F_MDC_LOG_LOCATION(F) ((F)->shared->mdc_log_location)
+#define H5F_ALIGNMENT(F) ((F)->shared->alignment)
+#define H5F_THRESHOLD(F) ((F)->shared->threshold)
+#define H5F_PGEND_META_THRES(F) ((F)->shared->fs.pgend_meta_thres)
+#define H5F_POINT_OF_NO_RETURN(F) ((F)->shared->fs.point_of_no_return)
+#define H5F_FIRST_ALLOC_DEALLOC(F) ((F)->shared->first_alloc_dealloc)
+#define H5F_EOA_PRE_FSM_FSALLOC(F) ((F)->shared->eoa_pre_fsm_fsalloc)
#else /* H5F_MODULE */
#define H5F_INTENT(F) (H5F_get_intent(F))
#define H5F_OPEN_NAME(F) (H5F_get_open_name(F))
@@ -375,6 +381,12 @@
#define H5F_USE_MDC_LOGGING(F) (H5F_use_mdc_logging(F))
#define H5F_START_MDC_LOG_ON_ACCESS(F) (H5F_start_mdc_log_on_access(F))
#define H5F_MDC_LOG_LOCATION(F) (H5F_mdc_log_location(F))
+#define H5F_ALIGNMENT(F) (H5F_get_alignment(F))
+#define H5F_THRESHOLD(F) (H5F_get_threshold(F))
+#define H5F_PGEND_META_THRES(F) (H5F_get_pgend_meta_thres(F))
+#define H5F_POINT_OF_NO_RETURN(F) (H5F_get_point_of_no_return(F))
+#define H5F_FIRST_ALLOC_DEALLOC(F) (H5F_get_first_alloc_dealloc(F))
+#define H5F_EOA_PRE_FSM_FSALLOC(F) (H5F_get_eoa_pre_fsm_fsalloc(F))
#endif /* H5F_MODULE */
@@ -448,7 +460,9 @@
#define H5F_CRT_SHMSG_LIST_MAX_NAME "shmsg_list_max" /* Shared message list maximum size */
#define H5F_CRT_SHMSG_BTREE_MIN_NAME "shmsg_btree_min" /* Shared message B-tree minimum size */
#define H5F_CRT_FILE_SPACE_STRATEGY_NAME "file_space_strategy" /* File space handling strategy */
+#define H5F_CRT_FREE_SPACE_PERSIST_NAME "free_space_persist" /* Free-space persisting status */
#define H5F_CRT_FREE_SPACE_THRESHOLD_NAME "free_space_threshold" /* Free space section threshold */
+#define H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME "file_space_page_size" /* File space page size */
@@ -484,6 +498,9 @@
#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME "core_write_tracking_page_size" /* The page size in kiB when core VFD write tracking is enabled */
#define H5F_ACS_COLL_MD_WRITE_FLAG_NAME "collective_metadata_write" /* property indicating whether metadata writes are done collectively or not */
#define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_NAME "mdc_initCacheImageCfg" /* Initial metadata cache image creation configuration */
+#define H5F_ACS_PAGE_BUFFER_SIZE_NAME "page_buffer_size" /* the maximum size for the page buffer cache */
+#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME "page_buffer_min_meta_perc" /* the min metadata percentage for the page buffer cache */
+#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME "page_buffer_min_raw_perc" /* the min raw data percentage for the page buffer cache */
/* ======================== File Mount properties ====================*/
#define H5F_MNT_SYM_LOCAL_NAME "local" /* Whether absolute symlinks local to file. */
@@ -522,10 +539,34 @@
/* See format specification on version 1 B-trees */
/* Default file space handling strategy */
-#define H5F_FILE_SPACE_STRATEGY_DEF H5F_FILE_SPACE_ALL
+#define H5F_FILE_SPACE_STRATEGY_DEF H5F_FSPACE_STRATEGY_FSM_AGGR
+
+/* Default free space section threshold used by free-space managers */
+#define H5F_FREE_SPACE_PERSIST_DEF FALSE
+
/* Default free space section threshold used by free-space managers */
#define H5F_FREE_SPACE_THRESHOLD_DEF 1
+/* For paged aggregation: default file space page size when not set */
+#define H5F_FILE_SPACE_PAGE_SIZE_DEF 4096
+/* For paged aggregation: minimum value for file space page size */
+#define H5F_FILE_SPACE_PAGE_SIZE_MIN 512
+
+/* For paged aggregation: drop free-space with size <= this threshold for small meta section */
+#define H5F_FILE_SPACE_PGEND_META_THRES 10
+
+/* Default for threshold for alignment (can be set via H5Pset_alignment()) */
+#define H5F_ALIGN_DEF 1
+/* Default for alignment (can be set via H5Pset_alignment()) */
+#define H5F_ALIGN_THRHD_DEF 1
+/* Default size for meta data aggregation block (can be set via H5Pset_meta_block_size()) */
+#define H5F_META_BLOCK_SIZE_DEF 2048
+/* Default size for small data aggregation block (can be set via H5Pset_small_data_block_size()) */
+#define H5F_SDATA_BLOCK_SIZE_DEF 2048
+
+/* Check for file using paged aggregation */
+#define H5F_PAGED_AGGR(F) (F->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE && F->shared->fs_page_size)
+
/* Metadata read attempt values */
#define H5F_METADATA_READ_ATTEMPTS 1 /* Default # of read attempts for non-SWMR access */
#define H5F_SWMR_METADATA_READ_ATTEMPTS 100 /* Default # of read attempts for SWMR access */
@@ -628,18 +669,54 @@ typedef struct H5F_object_flush_t {
void *udata; /* User data */
} H5F_object_flush_t;
-/* I/O Info for an operation */
+/* I/O Info for an operation (old) */
typedef struct H5F_io_info_t {
const H5F_t *f; /* File object */
const struct H5P_genplist_t *dxpl; /* DXPL object */
} H5F_io_info_t;
+/* I/O Info for an operation */
+/* (Migrate toward this one, so that both raw data & metadata DXPLs are available) */
+typedef struct H5F_io_info2_t {
+ const H5F_t *f; /* File object */
+ const struct H5P_genplist_t *meta_dxpl; /* Metadata DXPL object */
+ const struct H5P_genplist_t *raw_dxpl; /* Raw data DXPL object */
+} H5F_io_info2_t;
+
/* Concise info about a block of bytes in a file */
typedef struct H5F_block_t {
haddr_t offset; /* Offset of the block in the file */
hsize_t length; /* Length of the block in the file */
} H5F_block_t;
+/* Enum for free space manager state */
+typedef enum H5F_fs_state_t {
+ H5F_FS_STATE_CLOSED = 0, /* Free space manager is closed */
+ H5F_FS_STATE_OPEN = 1, /* Free space manager has been opened */
+ H5F_FS_STATE_DELETING = 2 /* Free space manager is being deleted */
+} H5F_fs_state_t;
+
+/* For paged aggregation */
+/* The values 0 to 6 is the same as H5F_mem_t */
+typedef enum H5F_mem_page_t {
+ H5F_MEM_PAGE_DEFAULT = 0, /* Not used */
+ H5F_MEM_PAGE_SUPER = 1,
+ H5F_MEM_PAGE_BTREE = 2,
+ H5F_MEM_PAGE_DRAW = 3,
+ H5F_MEM_PAGE_GHEAP = 4,
+ H5F_MEM_PAGE_LHEAP = 5,
+ H5F_MEM_PAGE_OHDR = 6,
+ H5F_MEM_PAGE_LARGE_SUPER = 7,
+ H5F_MEM_PAGE_LARGE_BTREE = 8,
+ H5F_MEM_PAGE_LARGE_DRAW = 9,
+ H5F_MEM_PAGE_LARGE_GHEAP = 10,
+ H5F_MEM_PAGE_LARGE_LHEAP = 11,
+ H5F_MEM_PAGE_LARGE_OHDR = 12,
+ H5F_MEM_PAGE_NTYPES = 13 /* Sentinel value - must be last */
+} H5F_mem_page_t;
+
+#define H5F_MEM_PAGE_META H5F_MEM_PAGE_SUPER /* Small-sized meta data */
+#define H5F_MEM_PAGE_GENERIC H5F_MEM_PAGE_LARGE_SUPER /* Large-sized generic: meta and raw */
/*****************************/
/* Library-private Variables */
@@ -674,6 +751,10 @@ H5_DLL hid_t H5F_get_access_plist(H5F_t *f, hbool_t app_ref);
H5_DLL hid_t H5F_get_id(H5F_t *file, hbool_t app_ref);
H5_DLL herr_t H5F_get_obj_count(const H5F_t *f, unsigned types, hbool_t app_ref, size_t *obj_id_count_ptr);
H5_DLL herr_t H5F_get_obj_ids(const H5F_t *f, unsigned types, size_t max_objs, hid_t *oid_list, hbool_t app_ref, size_t *obj_id_count_ptr);
+H5_DLL hsize_t H5F_get_pgend_meta_thres(const H5F_t *f);
+H5_DLL hbool_t H5F_get_point_of_no_return(const H5F_t *f);
+H5_DLL hbool_t H5F_get_first_alloc_dealloc(const H5F_t *f);
+H5_DLL hbool_t H5F_get_eoa_pre_fsm_fsalloc(const H5F_t *f);
/* Functions than retrieve values set/cached from the superblock/FCPL */
H5_DLL haddr_t H5F_get_base_addr(const H5F_t *f);
@@ -704,6 +785,8 @@ H5_DLL herr_t H5F_set_grp_btree_shared(H5F_t *f, struct H5UC_t *rc);
H5_DLL hbool_t H5F_use_tmp_space(const H5F_t *f);
H5_DLL hbool_t H5F_is_tmp_addr(const H5F_t *f, haddr_t addr);
H5_DLL herr_t H5F_set_latest_flags(H5F_t *f, unsigned flags);
+H5_DLL hsize_t H5F_get_alignment(const H5F_t *f);
+H5_DLL hsize_t H5F_get_threshold(const H5F_t *f);
#ifdef H5_HAVE_PARALLEL
H5_DLL H5P_coll_md_read_flag_t H5F_coll_md_read(const H5F_t *f);
H5_DLL void H5F_set_coll_md_read(H5F_t *f, H5P_coll_md_read_flag_t flag);
@@ -723,7 +806,7 @@ H5_DLL herr_t H5F_get_vfd_handle(const H5F_t *file, hid_t fapl, void **file_hand
H5_DLL hbool_t H5F_is_mount(const H5F_t *file);
H5_DLL hbool_t H5F_has_mount(const H5F_t *file);
H5_DLL herr_t H5F_traverse_mount(struct H5O_loc_t *oloc/*in,out*/);
-H5_DLL herr_t H5F_flush_mounts(H5F_t *f, hid_t dxpl_id);
+H5_DLL herr_t H5F_flush_mounts(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id);
/* Functions that operate on blocks of bytes wrt super block */
H5_DLL herr_t H5F_block_read(const H5F_t *f, H5FD_mem_t type, haddr_t addr,
diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h
index c57a821..f87aaad 100644
--- a/src/H5Fpublic.h
+++ b/src/H5Fpublic.h
@@ -182,6 +182,17 @@ typedef enum H5F_libver_t {
} H5F_libver_t;
/* File space handling strategy */
+typedef enum H5F_fspace_strategy_t {
+ H5F_FSPACE_STRATEGY_FSM_AGGR = 0, /* Mechanisms: free-space managers, aggregators, and virtual file drivers */
+ /* This is the library default when not set */
+ H5F_FSPACE_STRATEGY_PAGE = 1, /* Mechanisms: free-space managers with embedded paged aggregation and virtual file drivers */
+ H5F_FSPACE_STRATEGY_AGGR = 2, /* Mechanisms: aggregators and virtual file drivers */
+ H5F_FSPACE_STRATEGY_NONE = 3, /* Mechanisms: virtual file drivers */
+ H5F_FSPACE_STRATEGY_NTYPES /* must be last */
+} H5F_fspace_strategy_t;
+
+/* Deprecated: File space handling strategy for release 1.10.0 */
+/* They are mapped to H5F_fspace_strategy_t as defined above from release 1.10.1 onwards */
typedef enum H5F_file_space_type_t {
H5F_FILE_SPACE_DEFAULT = 0, /* Default (or current) free space strategy setting */
H5F_FILE_SPACE_ALL_PERSIST = 1, /* Persistent free space managers, aggregators, virtual file driver */
@@ -253,6 +264,10 @@ H5_DLL herr_t H5Fget_mdc_logging_status(hid_t file_id,
/*OUT*/ hbool_t *is_enabled,
/*OUT*/ hbool_t *is_currently_logging);
H5_DLL herr_t H5Fformat_convert(hid_t fid);
+H5_DLL herr_t H5Freset_page_buffering_stats(hid_t file_id);
+H5_DLL herr_t H5Fget_page_buffering_stats(hid_t file_id, unsigned accesses[2],
+ unsigned hits[2], unsigned misses[2], unsigned evictions[2], unsigned bypasses[2]);
+
#ifdef H5_HAVE_PARALLEL
H5_DLL herr_t H5Fset_mpi_atomicity(hid_t file_id, hbool_t flag);
H5_DLL herr_t H5Fget_mpi_atomicity(hid_t file_id, hbool_t *flag);
diff --git a/src/H5Fquery.c b/src/H5Fquery.c
index 14dd655..34fe8f9 100644
--- a/src/H5Fquery.c
+++ b/src/H5Fquery.c
@@ -1235,3 +1235,150 @@ H5F_mdc_log_location(const H5F_t *f)
FUNC_LEAVE_NOAPI(f->shared->mdc_log_location)
} /* end H5F_mdc_log_location() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_get_alignment
+ *
+ * Purpose: Retrieve the 'alignment' for the file.
+ *
+ * Return: Success: Non-negative, the 'alignment'
+ *
+ * Failure: (can't happen)
+ *
+ * Programmer: Vailin Choi; Dec 2012
+ *
+ *-------------------------------------------------------------------------
+ */
+hsize_t
+H5F_get_alignment(const H5F_t *f)
+{
+ /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ HDassert(f);
+ HDassert(f->shared);
+
+ FUNC_LEAVE_NOAPI(f->shared->alignment)
+} /* end H5F_get_alignment() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_get_threshold
+ *
+ * Purpose: Retrieve the 'threshold' for alignment in the file.
+ *
+ * Return: Success: Non-negative, the 'threshold'
+ *
+ * Failure: (can't happen)
+ *
+ * Programmer: Vailin Choi; Dec 2012
+ *
+ *-------------------------------------------------------------------------
+ */
+hsize_t
+H5F_get_threshold(const H5F_t *f)
+{
+ /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ HDassert(f);
+ HDassert(f->shared);
+
+ FUNC_LEAVE_NOAPI(f->shared->threshold)
+} /* end H5F_get_threshold() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_get_pgend_meta_thres
+ *
+ * Purpose: Retrieve the 'page end meta threshold size' for the file.
+ *
+ * Return: Success: Non-negative, the 'pgend_meta_thres'
+ *
+ * Failure: (can't happen)
+ *
+ * Programmer: Vailin Choi; Dec 2012
+ *
+ *-------------------------------------------------------------------------
+ */
+hsize_t
+H5F_get_pgend_meta_thres(const H5F_t *f)
+{
+ /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ HDassert(f);
+ HDassert(f->shared);
+
+ FUNC_LEAVE_NOAPI(f->shared->pgend_meta_thres)
+} /* end H5F_get_pgend_meta_thres() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_get_point_of_no_return
+ *
+ * Purpose: Retrieve the 'point of no return' value for the file.
+ *
+ * Return: Success: Non-negative, the 'point_of_no_return'
+ * Failure: (can't happen)
+ *
+ *-------------------------------------------------------------------------
+ */
+hbool_t
+H5F_get_point_of_no_return(const H5F_t *f)
+{
+ /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ HDassert(f);
+ HDassert(f->shared);
+
+ FUNC_LEAVE_NOAPI(f->shared->point_of_no_return)
+} /* end H5F_get_point_of_no_return() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_get_first_alloc_dealloc
+ *
+ * Purpose: Retrieve the 'first alloc / dealloc' value for the file.
+ *
+ * Return: Success: Non-negative, the 'first_alloc_dealloc'
+ * Failure: (can't happen)
+ *
+ *-------------------------------------------------------------------------
+ */
+hbool_t
+H5F_get_first_alloc_dealloc(const H5F_t *f)
+{
+ /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ HDassert(f);
+ HDassert(f->shared);
+
+ FUNC_LEAVE_NOAPI(f->shared->first_alloc_dealloc)
+} /* end H5F_get_first_alloc_dealloc() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_get_eoa_pre_fsm_fsalloc
+ *
+ * Purpose: Retrieve the 'EOA pre-FSM fsalloc' value for the file.
+ *
+ * Return: Success: Non-negative, the 'EOA pre-FSM fsalloc'
+ * Failure: (can't happen)
+ *
+ *-------------------------------------------------------------------------
+ */
+hbool_t
+H5F_get_eoa_pre_fsm_fsalloc(const H5F_t *f)
+{
+ /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ HDassert(f);
+ HDassert(f->shared);
+
+ FUNC_LEAVE_NOAPI(f->shared->eoa_pre_fsm_fsalloc)
+} /* end H5F_get_eoa_pre_fsm_fsalloc() */
+
diff --git a/src/H5Fspace.c b/src/H5Fspace.c
new file mode 100644
index 0000000..53570f6
--- /dev/null
+++ b/src/H5Fspace.c
@@ -0,0 +1,226 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the files COPYING and Copyright.html. COPYING can be found at the root *
+ * of the source code distribution tree; Copyright.html can be found at the *
+ * root level of an installed copy of the electronic HDF5 document set and *
+ * is linked from the top-level documents page. It can also be found at *
+ * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have *
+ * access to either file, you may request a copy from help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*-------------------------------------------------------------------------
+ *
+ * Created: H5Fspace.c
+ * Dec 30 2013
+ * Quincey Koziol <koziol@hdfgroup.org>
+ *
+ * Purpose: Space allocation routines for the file.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+/****************/
+/* Module Setup */
+/****************/
+
+#include "H5Fmodule.h" /* This source code file is part of the H5F module */
+
+
+/***********/
+/* Headers */
+/***********/
+#include "H5private.h" /* Generic Functions */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5Fpkg.h" /* File access */
+#include "H5FDprivate.h" /* File drivers */
+
+
+/****************/
+/* Local Macros */
+/****************/
+
+
+/******************/
+/* Local Typedefs */
+/******************/
+
+
+/********************/
+/* Package Typedefs */
+/********************/
+
+
+/********************/
+/* Local Prototypes */
+/********************/
+
+
+/*********************/
+/* Package Variables */
+/*********************/
+
+
+/*****************************/
+/* Library Private Variables */
+/*****************************/
+
+
+/*******************/
+/* Local Variables */
+/*******************/
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_alloc
+ *
+ * Purpose: Wrapper for H5FD_alloc, to make certain EOA changes are
+ * reflected in superblock.
+ *
+ * Note: When the metadata cache routines are updated to allow
+ * marking an entry dirty without a H5F_t*, this routine should
+ * be changed to take a H5F_super_t* directly.
+ *
+ * Return: Success: The format address of the new file memory.
+ * Failure: The undefined address HADDR_UNDEF
+ *
+ * Programmer: Quincey Koziol
+ * Monday, December 30, 2013
+ *
+ *-------------------------------------------------------------------------
+ */
+haddr_t
+H5F_alloc(H5F_t *f, hid_t dxpl_id, H5F_mem_t type, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size)
+{
+ haddr_t ret_value = 0; /* Return value */
+
+ FUNC_ENTER_NOAPI(HADDR_UNDEF)
+
+ /* check args */
+ HDassert(f);
+ HDassert(f->shared);
+ HDassert(f->shared->lf);
+ HDassert(type >= H5FD_MEM_DEFAULT && type < H5FD_MEM_NTYPES);
+ HDassert(size > 0);
+
+ /* Check whether the file can use temporary addresses */
+ if(f->shared->use_tmp_space) {
+ haddr_t eoa; /* Current EOA for the file */
+
+ /* Get the EOA for the file */
+ if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, type)))
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, HADDR_UNDEF, "Unable to get eoa")
+
+ /* Check for overlapping into file's temporary allocation space */
+ if(H5F_addr_gt((eoa + size), f->shared->tmp_addr))
+ HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space")
+ } /* end if */
+
+ /* Call the file driver 'alloc' routine */
+ ret_value = H5FD_alloc(f->shared->lf, dxpl_id, type, f, size, frag_addr, frag_size);
+ if(!H5F_addr_defined(ret_value))
+ HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, HADDR_UNDEF, "file driver 'alloc' request failed")
+
+ /* Mark EOA dirty */
+ if(H5F_eoa_dirty(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, HADDR_UNDEF, "unable to mark EOA as dirty")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5F_alloc() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_free
+ *
+ * Purpose: Wrapper for H5FD_free, to make certain EOA changes are
+ * reflected in superblock.
+ *
+ * Note: When the metadata cache routines are updated to allow
+ * marking an entry dirty without a H5F_t*, this routine should
+ * be changed to take a H5F_super_t* directly.
+ *
+ * Return: Success: Non-negative
+ * Failure: Negative
+ *
+ * Programmer: Quincey Koziol
+ * Monday, December 30, 2013
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5F_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, hsize_t size)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* Check args */
+ HDassert(f);
+ HDassert(f->shared);
+ HDassert(f->shared->lf);
+ HDassert(type >= H5FD_MEM_DEFAULT && type < H5FD_MEM_NTYPES);
+ HDassert(size > 0);
+
+ /* Call the file driver 'free' routine */
+ if(H5FD_free(f->shared->lf, dxpl_id, type, f, addr, size) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "file driver 'free' request failed")
+
+ /* Mark EOA dirty */
+ if(H5F_eoa_dirty(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark EOA as dirty")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5F_free() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_try_extend
+ *
+ * Purpose: Extend a block at the end of the file, if possible.
+ *
+ * Note: When the metadata cache routines are updated to allow
+ * marking an entry dirty without a H5F_t*, this routine should
+ * be changed to take a H5F_super_t* directly.
+ *
+ * Return: Success: TRUE(1) - Block was extended
+ * FALSE(0) - Block could not be extended
+ * Failure: FAIL
+ *
+ * Programmer: Quincey Koziol
+ * Monday, 30 December, 2013
+ *
+ *-------------------------------------------------------------------------
+ */
+htri_t
+H5F_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t blk_end, hsize_t extra_requested)
+{
+ htri_t ret_value = FALSE; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* check args */
+ HDassert(f);
+ HDassert(f->shared);
+ HDassert(f->shared->lf);
+ HDassert(type >= H5FD_MEM_DEFAULT && type < H5FD_MEM_NTYPES);
+ HDassert(extra_requested > 0);
+
+ /* Extend the object by extending the underlying file */
+ if((ret_value = H5FD_try_extend(f->shared->lf, type, f, dxpl_id, blk_end, extra_requested)) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTEXTEND, FAIL, "driver try extend request failed")
+
+ /* H5FD_try_extend() updates driver message and marks the superblock
+ * dirty, so no need to do it again here.
+ */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5F_try_extend() */
+
diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c
index 3b86dae..58ef9bb 100644
--- a/src/H5Fsuper.c
+++ b/src/H5Fsuper.c
@@ -29,6 +29,7 @@
#include "H5Fpkg.h" /* File access */
#include "H5FDprivate.h" /* File drivers */
#include "H5Iprivate.h" /* IDs */
+#include "H5MFprivate.h" /* File memory management */
#include "H5MMprivate.h" /* Memory management */
#include "H5Pprivate.h" /* Property lists */
#include "H5SMprivate.h" /* Shared Object Header Messages */
@@ -320,13 +321,14 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
-H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read)
+H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial_read)
{
H5P_genplist_t *dxpl = NULL; /* DXPL object */
H5AC_ring_t ring, orig_ring = H5AC_RING_INV;
H5F_super_t * sblock = NULL; /* Superblock structure */
H5F_superblock_cache_ud_t udata; /* User data for cache callbacks */
H5P_genplist_t *c_plist; /* File creation property list */
+ H5FD_io_info_t fdio_info; /* File driver I/O info */
unsigned sblock_flags = H5AC__NO_FLAGS_SET; /* flags used in superblock unprotect call */
haddr_t super_addr; /* Absolute address of superblock */
haddr_t eof; /* End of file address */
@@ -334,7 +336,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read)
hbool_t skip_eof_check = FALSE; /* Whether to skip checking the EOF value */
herr_t ret_value = SUCCEED; /* Return value */
- FUNC_ENTER_PACKAGE_TAG(dxpl_id, H5AC__SUPERBLOCK_TAG, FAIL)
+ FUNC_ENTER_PACKAGE_TAG(meta_dxpl_id, H5AC__SUPERBLOCK_TAG, FAIL)
/* initialize the drvinfo to NULL -- we will overwrite this if there
* is a driver information block
@@ -342,13 +344,19 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read)
f->shared->drvinfo = NULL;
/* Get the DXPL plist object for DXPL ID */
- if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
+ if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id)))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
if((H5P_get(dxpl, H5AC_RING_NAME, &orig_ring)) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get property value");
+ /* Set up file driver I/O info */
+ fdio_info.file = f->shared->lf;
+ fdio_info.meta_dxpl = dxpl;
+ if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(raw_dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+
/* Find the superblock */
- if(H5FD_locate_signature(f->shared->lf, dxpl, &super_addr) < 0)
+ if(H5FD_locate_signature(&fdio_info, &super_addr) < 0)
HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable to locate file signature")
if(HADDR_UNDEF == super_addr)
HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "file signature not found")
@@ -396,7 +404,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read)
HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set property value");
/* Look up the superblock */
- if(NULL == (sblock = (H5F_super_t *)H5AC_protect(f, dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, &udata, rw_flags)))
+ if(NULL == (sblock = (H5F_super_t *)H5AC_protect(f, meta_dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, &udata, rw_flags)))
HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "unable to load superblock")
if(H5F_INTENT(f) & H5F_ACC_SWMR_WRITE)
@@ -556,7 +564,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read)
HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed")
/* Look up the driver info block */
- if(NULL == (drvinfo = (H5O_drvinfo_t *)H5AC_protect(f, dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, &drvrinfo_udata, rw_flags)))
+ if(NULL == (drvinfo = (H5O_drvinfo_t *)H5AC_protect(f, meta_dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, &drvrinfo_udata, rw_flags)))
HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "unable to load driver info block")
/* Loading the driver info block is enough to set up the right info */
@@ -571,7 +579,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read)
drvinfo_flags |= H5AC__PIN_ENTRY_FLAG;
/* Release the driver info block */
- if(H5AC_unprotect(f, dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, drvinfo, drvinfo_flags) < 0)
+ if(H5AC_unprotect(f, meta_dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, drvinfo, drvinfo_flags) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTUNPROTECT, FAIL, "unable to release driver info block")
/* save a pointer to the driver information cache entry */
@@ -612,14 +620,14 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read)
HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, FAIL, "unable to open file's superblock extension")
/* Check for the extension having a 'driver info' message */
- if((status = H5O_msg_exists(&ext_loc, H5O_DRVINFO_ID, dxpl_id)) < 0)
+ if((status = H5O_msg_exists(&ext_loc, H5O_DRVINFO_ID, meta_dxpl_id)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_EXISTS, FAIL, "unable to read object header")
if(status) {
/* Check for ignoring the driver info for this file */
if(!udata.ignore_drvrinfo) {
/* Retrieve the 'driver info' structure */
- if(NULL == H5O_msg_read(&ext_loc, H5O_DRVINFO_ID, &drvinfo, dxpl_id))
+ if(NULL == H5O_msg_read(&ext_loc, H5O_DRVINFO_ID, &drvinfo, meta_dxpl_id))
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "driver info message not present")
/* Validate and decode driver information */
@@ -637,15 +645,15 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read)
} /* end if */
/* Read in the shared OH message information if there is any */
- if(H5SM_get_info(&ext_loc, c_plist, dxpl_id) < 0)
+ if(H5SM_get_info(&ext_loc, c_plist, meta_dxpl_id) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to read SOHM table information")
/* Check for the extension having a 'v1 B-tree "K"' message */
- if((status = H5O_msg_exists(&ext_loc, H5O_BTREEK_ID, dxpl_id)) < 0)
+ if((status = H5O_msg_exists(&ext_loc, H5O_BTREEK_ID, meta_dxpl_id)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_EXISTS, FAIL, "unable to read object header")
if(status) {
/* Retrieve the 'v1 B-tree "K"' structure */
- if(NULL == H5O_msg_read(&ext_loc, H5O_BTREEK_ID, &btreek, dxpl_id))
+ if(NULL == H5O_msg_read(&ext_loc, H5O_BTREEK_ID, &btreek, meta_dxpl_id))
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "v1 B-tree 'K' info message not present")
/* Set non-default v1 B-tree 'K' value info from file */
@@ -661,39 +669,105 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read)
} /* end if */
/* Check for the extension having a 'free-space manager info' message */
- if((status = H5O_msg_exists(&ext_loc, H5O_FSINFO_ID, dxpl_id)) < 0)
+ if((status = H5O_msg_exists(&ext_loc, H5O_FSINFO_ID, meta_dxpl_id)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_EXISTS, FAIL, "unable to read object header")
if(status) {
- H5O_fsinfo_t fsinfo; /* Free-space manager info message from superblock extension */
+ H5O_fsinfo_t fsinfo; /* File space info message from superblock extension */
+ uint8_t flags; /* Message flags */
- /* Retrieve the 'free-space manager info' structure */
- if(NULL == H5O_msg_read(&ext_loc, H5O_FSINFO_ID, &fsinfo, dxpl_id))
- HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get free-space manager info message")
+ /* Get message flags */
+ if(H5O_msg_get_flags(&ext_loc, H5O_FSINFO_ID, meta_dxpl_id, &flags) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to message flags for free-space manager info message")
- /* Check for non-default info */
- if(f->shared->fs_strategy != fsinfo.strategy) {
- f->shared->fs_strategy = fsinfo.strategy;
+ /* If message is NOT marked "unknown"--set up file space info */
+ if(!(flags & H5O_MSG_FLAG_WAS_UNKNOWN)) {
- /* Set non-default strategy in the property list */
- if(H5P_set(c_plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &fsinfo.strategy) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy")
- } /* end if */
- if(f->shared->fs_threshold != fsinfo.threshold) {
- f->shared->fs_threshold = fsinfo.threshold;
+ /* Retrieve the 'file space info' structure */
+ if(NULL == H5O_msg_read(&ext_loc, H5O_FSINFO_ID, &fsinfo, meta_dxpl_id))
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get free-space manager info message")
- /* Set non-default threshold in the property list */
- if(H5P_set(c_plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &fsinfo.threshold) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy")
- } /* end if */
+ /* Update changed values */
+ if(f->shared->fs_strategy != fsinfo.strategy) {
+ f->shared->fs_strategy = fsinfo.strategy;
+
+ /* Set non-default strategy in the property list */
+ if(H5P_set(c_plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &fsinfo.strategy) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy")
+ } /* end if */
+ if(f->shared->fs_persist != fsinfo.persist) {
+ f->shared->fs_persist = fsinfo.persist;
+
+ /* Set non-default strategy in the property list */
+ if(H5P_set(c_plist, H5F_CRT_FREE_SPACE_PERSIST_NAME, &fsinfo.persist) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy")
+ } /* end if */
+ if(f->shared->fs_threshold != fsinfo.threshold) {
+ f->shared->fs_threshold = fsinfo.threshold;
+
+ /* Set non-default threshold in the property list */
+ if(H5P_set(c_plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &fsinfo.threshold) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy")
+ } /* end if */
+
+ HDassert(f->shared->fs_page_size >= H5F_FILE_SPACE_PAGE_SIZE_MIN);
+ HDassert(fsinfo.page_size >= H5F_FILE_SPACE_PAGE_SIZE_MIN);
+ if(f->shared->fs_page_size != fsinfo.page_size) {
+ f->shared->fs_page_size = fsinfo.page_size;
+
+ /* Set file space page size in the property list */
+ if(H5P_set(c_plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, &fsinfo.page_size) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space page size")
+ } /* end if */
+ if(f->shared->pgend_meta_thres != fsinfo.pgend_meta_thres)
+ /* Initialize page end meta threshold */
+ f->shared->pgend_meta_thres = fsinfo.pgend_meta_thres;
+
+ if(f->shared->eoa_pre_fsm_fsalloc != fsinfo.eoa_pre_fsm_fsalloc)
+ f->shared->eoa_pre_fsm_fsalloc = fsinfo.eoa_pre_fsm_fsalloc;
+
+ /* f->shared->eoa_pre_fsm_fsalloc must always be HADDR_UNDEF
+ * in the absence of persistant free space managers.
+ */
+ HDassert((!f->shared->fs_persist) || (f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF));
+ HDassert(!f->shared->first_alloc_dealloc);
+
+ if((f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF) &&
+ (H5F_INTENT(f) & H5F_ACC_RDWR))
+ f->shared->first_alloc_dealloc = TRUE;
+
+ f->shared->fs_addr[0] = HADDR_UNDEF;
+ for(u = 1; u < NELMTS(f->shared->fs_addr); u++)
+ f->shared->fs_addr[u] = fsinfo.fs_addr[u - 1];
- /* Set free-space manager addresses */
- f->shared->fs_addr[0] = HADDR_UNDEF;
- for(u = 1; u < NELMTS(f->shared->fs_addr); u++)
- f->shared->fs_addr[u] = fsinfo.fs_addr[u-1];
+ if(fsinfo.mapped && (rw_flags & H5AC__READ_ONLY_FLAG) == 0) {
+
+ /* Do the same kluge until we know for sure. VC */
+#if 1 /* bug fix test code -- tidy this up if all goes well */ /* JRM */
+ /* KLUGE ALERT!!
+ *
+ * H5F_super_ext_write_msg() expects f->shared->sblock to
+ * be set -- verify that it is NULL, and then set it.
+ * Set it back to NULL when we are done.
+ */
+ HDassert(f->shared->sblock == NULL);
+ f->shared->sblock = sblock;
+#endif /* JRM */
+
+ if(H5F_super_ext_remove_msg(f, meta_dxpl_id, H5O_FSINFO_ID) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTDELETE, FAIL, "error in removing message from superblock extension")
+
+ if(H5F_super_ext_write_msg(f, meta_dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing fsinfo message to superblock extension")
+#if 1 /* bug fix test code -- tidy this up if all goes well */ /* JRM */
+ f->shared->sblock = NULL;
+#endif /* JRM */
+
+ }
+ } /* end if not marked "unknown" */
} /* end if */
/* Check for the extension having a 'metadata cache image' message */
- if((status = H5O_msg_exists(&ext_loc, H5O_MDCI_MSG_ID, dxpl_id)) < 0)
+ if((status = H5O_msg_exists(&ext_loc, H5O_MDCI_MSG_ID, meta_dxpl_id)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_EXISTS, FAIL, "unable to read object header")
if(status) {
hbool_t rw = ((rw_flags & H5AC__READ_ONLY_FLAG) == 0);
@@ -712,7 +786,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read)
*/
/* Retrieve the 'metadata cache image message' structure */
- if(NULL == H5O_msg_read(&ext_loc, H5O_MDCI_MSG_ID, &mdci_msg, dxpl_id))
+ if(NULL == H5O_msg_read(&ext_loc, H5O_MDCI_MSG_ID, &mdci_msg, meta_dxpl_id))
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get metadata cache image message")
/* Indicate to the cache that there's an image to load on first protect call */
@@ -721,7 +795,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read)
} /* end if */
/* Close superblock extension */
- if(H5F_super_ext_close(f, &ext_loc, dxpl_id, FALSE) < 0)
+ if(H5F_super_ext_close(f, &ext_loc, meta_dxpl_id, FALSE) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension")
} /* end if */
@@ -765,7 +839,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read)
HDassert(f->shared->sblock == NULL);
f->shared->sblock = sblock;
#endif /* JRM */
- if(H5F_super_ext_write_msg(f, dxpl_id, H5O_DRVINFO_ID, &drvinfo, FALSE, H5O_MSG_NO_FLAGS_SET) < 0)
+ if(H5F_super_ext_write_msg(f, meta_dxpl_id, H5O_DRVINFO_ID, &drvinfo, FALSE, H5O_MSG_NO_FLAGS_SET) < 0)
HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension")
#if 1 /* bug fix test code -- tidy this up if all goes well */ /* JRM */
@@ -777,7 +851,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read)
/* Check for eliminating the driver info block */
else if(H5F_HAS_FEATURE(f, H5FD_FEAT_IGNORE_DRVRINFO)) {
/* Remove the driver info message from the superblock extension */
- if(H5F_super_ext_remove_msg(f, dxpl_id, H5O_DRVINFO_ID) < 0)
+ if(H5F_super_ext_remove_msg(f, meta_dxpl_id, H5O_DRVINFO_ID) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "error in removing message from superblock extension")
/* Check if the superblock extension was removed */
@@ -789,13 +863,17 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read)
/* Set the pointer to the pinned superblock */
f->shared->sblock = sblock;
+ /* Set the page aggregation mode */
+ if(H5F__set_paged_aggr(f, (hbool_t)H5F_PAGED_AGGR(f)) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "failed to set paged_aggr status for file driver")
+
done:
/* Reset the ring in the DXPL */
if(H5AC_reset_ring(dxpl, orig_ring) < 0)
HDONE_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set property value")
/* Release the superblock */
- if(sblock && H5AC_unprotect(f, dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, sblock, sblock_flags) < 0)
+ if(sblock && H5AC_unprotect(f, meta_dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, sblock, sblock_flags) < 0)
HDONE_ERROR(H5E_FILE, H5E_CANTUNPROTECT, FAIL, "unable to close superblock")
/* If we have failed, make sure no entries are left in the
@@ -808,7 +886,7 @@ done:
HDONE_ERROR(H5E_FILE, H5E_CANTUNPIN, FAIL, "unable to unpin driver info")
/* Evict the driver info block from the cache */
- if(H5AC_expunge_entry(f, dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, H5AC__NO_FLAGS_SET) < 0)
+ if(H5AC_expunge_entry(f, meta_dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, H5AC__NO_FLAGS_SET) < 0)
HDONE_ERROR(H5E_FILE, H5E_CANTEXPUNGE, FAIL, "unable to expunge driver info block")
} /* end if */
@@ -819,7 +897,7 @@ done:
HDONE_ERROR(H5E_FILE, H5E_CANTUNPIN, FAIL, "unable to unpin superblock")
/* Evict the superblock from the cache */
- if(H5AC_expunge_entry(f, dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, H5AC__NO_FLAGS_SET) < 0)
+ if(H5AC_expunge_entry(f, meta_dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, H5AC__NO_FLAGS_SET) < 0)
HDONE_ERROR(H5E_FILE, H5E_CANTEXPUNGE, FAIL, "unable to expunge superblock")
} /* end if */
} /* end if */
@@ -890,7 +968,9 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id)
/* Check for non-default free-space settings */
if(!(f->shared->fs_strategy == H5F_FILE_SPACE_STRATEGY_DEF &&
- f->shared->fs_threshold == H5F_FREE_SPACE_THRESHOLD_DEF))
+ f->shared->fs_persist == H5F_FREE_SPACE_PERSIST_DEF &&
+ f->shared->fs_threshold == H5F_FREE_SPACE_THRESHOLD_DEF &&
+ f->shared->fs_page_size == H5F_FILE_SPACE_PAGE_SIZE_DEF))
non_default_fs_settings = TRUE;
/* Bump superblock version if latest superblock version support is enabled */
@@ -899,8 +979,12 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id)
/* Bump superblock version to create superblock extension for SOHM info */
else if(f->shared->sohm_nindexes > 0)
super_vers = HDF5_SUPERBLOCK_VERSION_2;
- /* Bump superblock version to create superblock extension for
- * non-default file space strategy or non-default free-space threshold
+ /*
+ * Bump superblock version to create superblock extension for:
+ * -- non-default file space strategy or
+ * -- non-default persisting free-space or
+ * -- non-default free-space threshold or
+ * -- non-default page_size
*/
else if(non_default_fs_settings)
super_vers = HDF5_SUPERBLOCK_VERSION_2;
@@ -921,6 +1005,9 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id)
HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set superblock version")
} /* end if */
+ if(H5FD_set_paged_aggr(f->shared->lf, (hbool_t)H5F_PAGED_AGGR(f)) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "failed to set paged_aggr status for file driver")
+
/*
* The superblock starts immediately after the user-defined
* header, which we have already insured is a proper size. The
@@ -932,9 +1019,12 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id)
/* Sanity check the userblock size vs. the file's allocation alignment */
if(userblock_size > 0) {
- if(userblock_size < f->shared->alignment)
+ /* Set up the alignment to use for page or aggr fs */
+ hsize_t alignment = H5F_PAGED_AGGR(f) ? f->shared->fs_page_size : f->shared->alignment;
+
+ if(userblock_size < alignment)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "userblock size must be > file object alignment")
- if(0 != (userblock_size % f->shared->alignment))
+ if(0 != (userblock_size % alignment))
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "userblock size must be an integral multiple of file object alignment")
} /* end if */
@@ -990,10 +1080,6 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id)
if(super_vers < HDF5_SUPERBLOCK_VERSION_2)
superblock_size += driver_size;
- /* Reserve space in the file for the superblock, instead of allocating it */
- if(H5F__set_eoa(f, H5FD_MEM_SUPER, superblock_size) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to set EOA value for superblock")
-
/* Set the ring type in the DXPL */
if(H5AC_set_ring(dxpl_id, H5AC_RING_SB, &dxpl, &orig_ring) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set ring value")
@@ -1006,6 +1092,10 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id)
/* Keep a copy of the superblock info */
f->shared->sblock = sblock;
+ /* Allocate space for the superblock */
+ if(HADDR_UNDEF == H5MF_alloc(f, H5FD_MEM_SUPER, dxpl_id, superblock_size))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "file allocation failed for superblock")
+
/* set the drvinfo filed to NULL -- will overwrite this later if needed */
f->shared->drvinfo = NULL;
@@ -1108,17 +1198,23 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id)
} /* end if */
/* Check for non-default free-space info settings */
- if(non_default_fs_settings) {
- H5FD_mem_t type; /* Memory type for iteration */
- H5O_fsinfo_t fsinfo; /* Free space manager info message */
-
- /* Write free-space manager info message to superblock extension object header if needed */
- fsinfo.strategy = f->shared->fs_strategy;
- fsinfo.threshold = f->shared->fs_threshold;
- for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
- fsinfo.fs_addr[type-1] = HADDR_UNDEF;
-
- if(H5O_msg_create(&ext_loc, H5O_FSINFO_ID, H5O_MSG_FLAG_DONTSHARE, H5O_UPDATE_TIME, &fsinfo, dxpl_id) < 0)
+ if(non_default_fs_settings) {
+ H5F_mem_page_t ptype;
+ H5O_fsinfo_t fsinfo; /* Free space manager info message */
+
+ /* Write free-space manager info message to superblock extension object header if needed */
+ fsinfo.strategy = f->shared->fs_strategy;
+ fsinfo.persist = f->shared->fs_persist;
+ fsinfo.threshold = f->shared->fs_threshold;
+ fsinfo.page_size = f->shared->fs_page_size;
+ fsinfo.pgend_meta_thres = f->shared->pgend_meta_thres;
+ fsinfo.eoa_pre_fsm_fsalloc = HADDR_UNDEF;
+ fsinfo.mapped = FALSE;
+
+ for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype))
+ fsinfo.fs_addr[ptype - 1] = HADDR_UNDEF;
+
+ if(H5O_msg_create(&ext_loc, H5O_FSINFO_ID, H5O_MSG_FLAG_DONTSHARE | H5O_MSG_FLAG_MARK_IF_UNKNOWN, H5O_UPDATE_TIME, &fsinfo, dxpl_id) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to update free-space info header message")
} /* end if */
} /* end if */
diff --git a/src/H5HFsection.c b/src/H5HFsection.c
index 37ff8f4..b113ca1 100644
--- a/src/H5HFsection.c
+++ b/src/H5HFsection.c
@@ -85,14 +85,14 @@ static herr_t H5HF_sect_single_full_dblock(H5HF_hdr_t *hdr, hid_t dxpl_id,
H5HF_free_section_t *sect);
/* 'single' section callbacks */
-static herr_t H5HF_sect_single_add(H5FS_section_info_t *sect, unsigned *flags,
+static herr_t H5HF_sect_single_add(H5FS_section_info_t **sect, unsigned *flags,
void *udata);
static H5FS_section_info_t *H5HF_sect_single_deserialize(const H5FS_section_class_t *cls,
hid_t dxpl_id, const uint8_t *buf, haddr_t sect_addr, hsize_t sect_size,
unsigned *des_flags);
static htri_t H5HF_sect_single_can_merge(const H5FS_section_info_t *sect1,
const H5FS_section_info_t *sect2, void *udata);
-static herr_t H5HF_sect_single_merge(H5FS_section_info_t *sect1,
+static herr_t H5HF_sect_single_merge(H5FS_section_info_t **sect1,
H5FS_section_info_t *sect2, void *udata);
static htri_t H5HF_sect_single_can_shrink(const H5FS_section_info_t *sect,
void *udata);
@@ -121,7 +121,7 @@ static H5FS_section_info_t *H5HF_sect_row_deserialize(const H5FS_section_class_t
unsigned *des_flags);
static htri_t H5HF_sect_row_can_merge(const H5FS_section_info_t *sect1,
const H5FS_section_info_t *sect2, void *udata);
-static herr_t H5HF_sect_row_merge(H5FS_section_info_t *sect1,
+static herr_t H5HF_sect_row_merge(H5FS_section_info_t **sect1,
H5FS_section_info_t *sect2, void *udata);
static htri_t H5HF_sect_row_can_shrink(const H5FS_section_info_t *sect,
void *udata);
@@ -811,7 +811,7 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
-H5HF_sect_single_add(H5FS_section_info_t *_sect, unsigned *flags, void *_udata)
+H5HF_sect_single_add(H5FS_section_info_t **_sect, unsigned *flags, void *_udata)
{
herr_t ret_value = SUCCEED; /* Return value */
@@ -821,7 +821,7 @@ H5HF_sect_single_add(H5FS_section_info_t *_sect, unsigned *flags, void *_udata)
* have already been checked when it was first added
*/
if(!(*flags & H5FS_ADD_DESERIALIZING)) {
- H5HF_free_section_t *sect = (H5HF_free_section_t *)_sect; /* Fractal heap free section */
+ H5HF_free_section_t **sect = (H5HF_free_section_t **)_sect; /* Fractal heap free section */
H5HF_sect_add_ud_t *udata = (H5HF_sect_add_ud_t *)_udata; /* User callback data */
H5HF_hdr_t *hdr = udata->hdr; /* Fractal heap header */
hid_t dxpl_id = udata->dxpl_id; /* DXPL ID for operation */
@@ -832,14 +832,14 @@ H5HF_sect_single_add(H5FS_section_info_t *_sect, unsigned *flags, void *_udata)
/* Check if single section covers entire direct block it's in */
/* (converts to row section possibly) */
- if(H5HF_sect_single_full_dblock(hdr, dxpl_id, sect) < 0)
+ if(H5HF_sect_single_full_dblock(hdr, dxpl_id, (*sect)) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_CANTCONVERT, FAIL, "can't check/convert single section")
/* Set the "returned space" flag if the single section was changed
* into a row section, so the "merging & shrinking" algorithm
* gets executed in the free space manager
*/
- if(sect->sect_info.type != H5HF_FSPACE_SECT_SINGLE)
+ if((*sect)->sect_info.type != H5HF_FSPACE_SECT_SINGLE)
*flags |= H5FS_ADD_RETURNED_SPACE;
} /* end if */
@@ -949,10 +949,10 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
-H5HF_sect_single_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2,
+H5HF_sect_single_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2,
void *_udata)
{
- H5HF_free_section_t *sect1 = (H5HF_free_section_t *)_sect1; /* Fractal heap free section */
+ H5HF_free_section_t **sect1 = (H5HF_free_section_t **)_sect1; /* Fractal heap free section */
H5HF_free_section_t *sect2 = (H5HF_free_section_t *)_sect2; /* Fractal heap free section */
H5HF_sect_add_ud_t *udata = (H5HF_sect_add_ud_t *)_udata; /* User callback data */
H5HF_hdr_t *hdr = udata->hdr; /* Fractal heap header */
@@ -963,26 +963,26 @@ H5HF_sect_single_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2,
/* Check arguments. */
HDassert(sect1);
- HDassert(sect1->sect_info.type == H5HF_FSPACE_SECT_SINGLE);
+ HDassert((*sect1)->sect_info.type == H5HF_FSPACE_SECT_SINGLE);
HDassert(sect2);
HDassert(sect2->sect_info.type == H5HF_FSPACE_SECT_SINGLE);
- HDassert(H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr));
+ HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr));
/* Add second section's size to first section */
- sect1->sect_info.size += sect2->sect_info.size;
+ (*sect1)->sect_info.size += sect2->sect_info.size;
/* Get rid of second section */
if(H5HF_sect_single_free((H5FS_section_info_t *)sect2) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_CANTRELEASE, FAIL, "can't free section node")
/* Check to see if we should revive first section */
- if(sect1->sect_info.state != H5FS_SECT_LIVE)
- if(H5HF_sect_single_revive(hdr, dxpl_id, sect1) < 0)
+ if((*sect1)->sect_info.state != H5FS_SECT_LIVE)
+ if(H5HF_sect_single_revive(hdr, dxpl_id, (*sect1)) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, FAIL, "can't revive single free section")
/* Check if single section covers entire direct block it's in */
/* (converts to row section possibly) */
- if(H5HF_sect_single_full_dblock(hdr, dxpl_id, sect1) < 0)
+ if(H5HF_sect_single_full_dblock(hdr, dxpl_id, (*sect1)) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_CANTCONVERT, FAIL, "can't check/convert single section")
done:
@@ -1771,10 +1771,10 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
-H5HF_sect_row_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2,
+H5HF_sect_row_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2,
void *_udata)
{
- H5HF_free_section_t *sect1 = (H5HF_free_section_t *)_sect1; /* Fractal heap free section */
+ H5HF_free_section_t **sect1 = (H5HF_free_section_t **)_sect1; /* Fractal heap free section */
H5HF_free_section_t *sect2 = (H5HF_free_section_t *)_sect2; /* Fractal heap free section */
H5HF_sect_add_ud_t *udata = (H5HF_sect_add_ud_t *)_udata; /* User callback data */
H5HF_hdr_t *hdr = udata->hdr; /* Fractal heap header */
@@ -1785,7 +1785,7 @@ H5HF_sect_row_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2,
/* Check arguments. */
HDassert(sect1);
- HDassert(sect1->sect_info.type == H5HF_FSPACE_SECT_FIRST_ROW);
+ HDassert((*sect1)->sect_info.type == H5HF_FSPACE_SECT_FIRST_ROW);
HDassert(sect2);
HDassert(sect2->sect_info.type == H5HF_FSPACE_SECT_FIRST_ROW);
@@ -1802,8 +1802,8 @@ H5HF_sect_row_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2,
} /* end if */
else {
/* Check to see if we should revive first section */
- if(sect1->sect_info.state != H5FS_SECT_LIVE)
- if(H5HF_sect_row_revive(hdr, dxpl_id, sect1) < 0)
+ if((*sect1)->sect_info.state != H5FS_SECT_LIVE)
+ if(H5HF_sect_row_revive(hdr, dxpl_id, (*sect1)) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, FAIL, "can't revive single free section")
/* Check to see if we should revive second section */
@@ -1812,7 +1812,7 @@ H5HF_sect_row_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2,
HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, FAIL, "can't revive single free section")
/* Merge rows' underlying indirect sections together */
- if(H5HF_sect_indirect_merge_row(hdr, dxpl_id, sect1, sect2) < 0)
+ if(H5HF_sect_indirect_merge_row(hdr, dxpl_id, (*sect1), sect2) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_CANTMERGE, FAIL, "can't merge underlying indirect sections")
} /* end else */
diff --git a/src/H5MF.c b/src/H5MF.c
index 23f128f..358e326 100644
--- a/src/H5MF.c
+++ b/src/H5MF.c
@@ -29,6 +29,7 @@
/****************/
#define H5F_FRIEND /*suppress error about including H5Fpkg */
+#define H5FS_FRIEND /*suppress error about including H5Fpkg */
#include "H5MFmodule.h" /* This source code file is part of the H5MF module */
@@ -38,6 +39,7 @@
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
#include "H5Fpkg.h" /* File access */
+#include "H5FSpkg.h" /* File access */
#include "H5Iprivate.h" /* IDs */
#include "H5MFpkg.h" /* File memory management */
#include "H5VMprivate.h" /* Vectors and arrays */
@@ -50,11 +52,6 @@
#define H5MF_FSPACE_SHRINK 80 /* Percent of "normal" size to shrink serialized free space size */
#define H5MF_FSPACE_EXPAND 120 /* Percent of "normal" size to expand serialized free space size */
-/* Map an allocation request type to a free list */
-#define H5MF_ALLOC_TO_FS_TYPE(F, T) ((H5FD_MEM_DEFAULT == (F)->shared->fs_type_map[T]) \
- ? (T) : (F)->shared->fs_type_map[T])
-
-
/******************/
/* Local Typedefs */
/******************/
@@ -84,11 +81,24 @@ typedef struct {
/********************/
/* Allocator routines */
-static herr_t H5MF_alloc_create(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type);
-static herr_t H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type);
-static herr_t H5MF__close_delete(H5F_t *f, hid_t dxpl_id, H5P_genplist_t **dxpl);
+static haddr_t H5MF__alloc_pagefs(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size);
+
+/* "File closing" routines */
+static herr_t H5MF__close_aggrfs(H5F_t *f, hid_t dxpl_id);
+static herr_t H5MF__close_pagefs(H5F_t *f, hid_t dxpl_id);
static herr_t H5MF__close_shrink_eoa(H5F_t *f, hid_t dxpl_id);
+/* General routines */
+static herr_t H5MF__get_free_sects(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5MF_sect_iter_ud_t *sect_udata, size_t *nums);
+static hbool_t H5MF__fsm_type_is_self_referential(H5F_t *f, H5F_mem_page_t fsm_type);
+static hbool_t H5MF__fsm_is_self_referential(H5F_t *f, H5FS_t *fspace);
+
+/* Free-space type manager routines */
+static herr_t H5MF__create_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type);
+static herr_t H5MF__close_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type);
+static herr_t H5MF__delete_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type);
+static herr_t H5MF__close_delete_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type);
+
/*********************/
/* Package Variables */
@@ -223,10 +233,54 @@ done:
/*-------------------------------------------------------------------------
- * Function: H5MF__alloc_open
+ * Function: H5MF_alloc_to_fs_type
+ *
+ * Purpose: Map "alloc_type" to the free-space manager type
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ * Programmer: Vailin Choi; Nov 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+void
+H5MF_alloc_to_fs_type(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size, H5F_mem_page_t *fs_type)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ HDassert(f);
+ HDassert(fs_type);
+
+ if(H5F_PAGED_AGGR(f)) { /* paged aggregation */
+ if(size >= f->shared->fs_page_size) {
+ if(H5F_HAS_FEATURE(f, H5FD_FEAT_PAGED_AGGR)) { /* multi or split driver */
+ /* For non-contiguous address space, map to large size free-space manager for each alloc_type */
+ if(H5FD_MEM_DEFAULT == f->shared->fs_type_map[alloc_type])
+ *fs_type = (H5F_mem_page_t) (alloc_type + (H5FD_MEM_NTYPES - 1));
+ else
+ *fs_type = (H5F_mem_page_t) (f->shared->fs_type_map[alloc_type] + (H5FD_MEM_NTYPES - 1));
+ } /* end if */
+ else
+ /* For contiguous address space, map to generic large size free-space manager */
+ *fs_type = H5F_MEM_PAGE_GENERIC; /* H5F_MEM_PAGE_SUPER */
+ } /* end if */
+ else
+ *fs_type = (H5F_mem_page_t)H5MF_ALLOC_TO_FS_AGGR_TYPE(f, alloc_type);
+ } /* end if */
+ else /* non-paged aggregation */
+ *fs_type = (H5F_mem_page_t)H5MF_ALLOC_TO_FS_AGGR_TYPE(f, alloc_type);
+
+ FUNC_LEAVE_NOAPI_VOID
+} /* end H5MF_alloc_to_fs_type() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF_open_fstype
*
* Purpose: Open an existing free space manager of TYPE for file by
- * creating a free-space structure
+ * creating a free-space structure.
+ * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types.
*
* Return: Success: non-negative
* Failure: negative
@@ -238,31 +292,48 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
-H5MF__alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type)
+H5MF_open_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type)
{
const H5FS_section_class_t *classes[] = { /* Free space section classes implemented for file */
- H5MF_FSPACE_SECT_CLS_SIMPLE};
- H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
- H5AC_ring_t fsm_ring; /* Free space manager ring */
- H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
- hbool_t reset_ring = FALSE; /* Whether the ring was set */
- herr_t ret_value = SUCCEED; /* Return value */
+ H5MF_FSPACE_SECT_CLS_SIMPLE,
+ H5MF_FSPACE_SECT_CLS_SMALL,
+ H5MF_FSPACE_SECT_CLS_LARGE };
+ hsize_t alignment; /* Alignment to use */
+ hsize_t threshold; /* Threshold to use */
+ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
+ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
+ H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */
+ hbool_t reset_ring = FALSE; /* Whether the ring was set */
+ herr_t ret_value = SUCCEED; /* Return value */
- FUNC_ENTER_NOAPI_NOINIT_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
+ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
/*
* Check arguments.
*/
HDassert(f);
+ if(H5F_PAGED_AGGR(f))
+ HDassert(type < H5F_MEM_PAGE_NTYPES);
+ else {
+ HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES);
+ HDassert((H5FD_mem_t)type != H5FD_MEM_NOLIST);
+ } /* end else */
HDassert(f->shared);
- HDassert(type != H5FD_MEM_NOLIST);
HDassert(H5F_addr_defined(f->shared->fs_addr[type]));
HDassert(f->shared->fs_state[type] == H5F_FS_STATE_CLOSED);
- HDassert(type == H5MF_ALLOC_TO_FS_TYPE(f, type));
+
+ /* Set up the aligment and threshold to use depending on the manager type */
+ if(H5F_PAGED_AGGR(f)) {
+ alignment = (type == H5F_MEM_PAGE_GENERIC) ? f->shared->fs_page_size : (hsize_t)H5F_ALIGN_DEF;
+ threshold = H5F_ALIGN_THRHD_DEF;
+ } /* end if */
+ else {
+ alignment = f->shared->alignment;
+ threshold = f->shared->threshold;
+ } /* end else */
/* Set the ring type in the DXPL */
- if((type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
- || (type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+ if(H5MF__fsm_type_is_self_referential(f, type))
fsm_ring = H5AC_RING_MDFSM;
else
fsm_ring = H5AC_RING_RDFSM;
@@ -272,7 +343,7 @@ H5MF__alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type)
/* Open an existing free space structure for the file */
if(NULL == (f->shared->fs_man[type] = H5FS_open(f, dxpl_id, f->shared->fs_addr[type],
- NELMTS(classes), classes, f, f->shared->alignment, f->shared->threshold)))
+ NELMTS(classes), classes, f, alignment, threshold)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space info")
/* Set the state for the free space manager to "open", if it is now */
@@ -286,14 +357,15 @@ done:
HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
-} /* end H5MF__alloc_open() */
+} /* end H5MF_open_fstype() */
/*-------------------------------------------------------------------------
- * Function: H5MF_alloc_create
+ * Function: H5MF__create_fstype
*
* Purpose: Create free space manager of TYPE for the file by creating
- * a free-space structure
+ * a free-space structure
+ * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types.
*
* Return: Success: non-negative
* Failure: negative
@@ -305,21 +377,34 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
-H5MF_alloc_create(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type)
+H5MF__create_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type)
{
const H5FS_section_class_t *classes[] = { /* Free space section classes implemented for file */
- H5MF_FSPACE_SECT_CLS_SIMPLE};
- herr_t ret_value = SUCCEED; /* Return value */
- H5FS_create_t fs_create; /* Free space creation parameters */
+ H5MF_FSPACE_SECT_CLS_SIMPLE,
+ H5MF_FSPACE_SECT_CLS_SMALL,
+ H5MF_FSPACE_SECT_CLS_LARGE };
+ H5FS_create_t fs_create; /* Free space creation parameters */
+ hsize_t alignment; /* Alignment to use */
+ hsize_t threshold; /* Threshold to use */
+ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
+ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
+ H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */
+ hbool_t reset_ring = FALSE; /* Whether the ring was set */
+ herr_t ret_value = SUCCEED; /* Return value */
- FUNC_ENTER_NOAPI_NOINIT_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
+ FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
/*
* Check arguments.
*/
HDassert(f);
+ if(H5F_PAGED_AGGR(f))
+ HDassert(type < H5F_MEM_PAGE_NTYPES);
+ else {
+ HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES);
+ HDassert((H5FD_mem_t)type != H5FD_MEM_NOLIST);
+ } /* end else */
HDassert(f->shared);
- HDassert(type != H5FD_MEM_NOLIST);
HDassert(!H5F_addr_defined(f->shared->fs_addr[type]));
HDassert(f->shared->fs_state[type] == H5F_FS_STATE_CLOSED);
@@ -330,24 +415,48 @@ H5MF_alloc_create(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type)
fs_create.max_sect_addr = 1 + H5VM_log2_gen((uint64_t)f->shared->maxaddr);
fs_create.max_sect_size = f->shared->maxaddr;
- if(NULL == (f->shared->fs_man[type] = H5FS_create(f, dxpl_id, NULL,
- &fs_create, NELMTS(classes), classes, f, f->shared->alignment, f->shared->threshold)))
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space info")
+ /* Set up alignment and threshold to use depending on TYPE */
+ if(H5F_PAGED_AGGR(f)) {
+ alignment = (type == H5F_MEM_PAGE_GENERIC) ? f->shared->fs_page_size : (hsize_t)H5F_ALIGN_DEF;
+ threshold = H5F_ALIGN_THRHD_DEF;
+ } /* end if */
+ else {
+ alignment = f->shared->alignment;
+ threshold = f->shared->threshold;
+ } /* end else */
+ /* Set the ring type in the DXPL */
+ if(H5MF__fsm_type_is_self_referential(f, type))
+ fsm_ring = H5AC_RING_MDFSM;
+ else
+ fsm_ring = H5AC_RING_RDFSM;
+ if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+ reset_ring = TRUE;
+
+ if(NULL == (f->shared->fs_man[type] = H5FS_create(f, dxpl_id, NULL,
+ &fs_create, NELMTS(classes), classes, f, alignment, threshold)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space info")
/* Set the state for the free space manager to "open", if it is now */
if(f->shared->fs_man[type])
f->shared->fs_state[type] = H5F_FS_STATE_OPEN;
done:
+ /* Reset the ring in the DXPL */
+ if(reset_ring)
+ if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+ HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+
FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
-} /* end H5MF_alloc_create() */
+} /* end H5MF__create_fstype() */
/*-------------------------------------------------------------------------
- * Function: H5MF__alloc_start
+ * Function: H5MF_start_fstype
*
- * Purpose: Open or create a free space manager of a given type
+ * Purpose: Open or create a free space manager of a given TYPE.
+ * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types.
*
* Return: Success: non-negative
* Failure: negative
@@ -359,40 +468,123 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
-H5MF__alloc_start(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type)
+H5MF_start_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type)
{
herr_t ret_value = SUCCEED; /* Return value */
- FUNC_ENTER_NOAPI_NOINIT
+ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
/*
* Check arguments.
*/
HDassert(f);
HDassert(f->shared);
- HDassert(type != H5FD_MEM_NOLIST);
+ if(H5F_PAGED_AGGR(f))
+ HDassert(type < H5F_MEM_PAGE_NTYPES);
+ else {
+ HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES);
+ HDassert((H5FD_mem_t)type != H5FD_MEM_NOLIST);
+ } /* end else */
/* Check if the free space manager exists already */
if(H5F_addr_defined(f->shared->fs_addr[type])) {
/* Open existing free space manager */
- if(H5MF__alloc_open(f, dxpl_id, type) < 0)
+ if(H5MF_open_fstype(f, dxpl_id, type) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTOPENOBJ, FAIL, "can't initialize file free space")
} /* end if */
else {
/* Create new free space manager */
- if(H5MF_alloc_create(f, dxpl_id, type) < 0)
+ if(H5MF__create_fstype(f, dxpl_id, type) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCREATE, FAIL, "can't initialize file free space")
} /* end else */
done:
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5MF__alloc_start() */
+ FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
+} /* end H5MF_start_fstype() */
/*-------------------------------------------------------------------------
- * Function: H5MF__alloc_close
+ * Function: H5MF__delete_fstype
*
- * Purpose: Close an existing free space manager of TYPE for file
+ * Purpose: Delete the free-space manager as specified by TYPE.
+ * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types.
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ * Programmer: Vailin Choi; April 2013
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5MF__delete_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type)
+{
+ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
+ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
+ H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */
+ hbool_t reset_ring = FALSE; /* Whether the ring was set */
+ haddr_t tmp_fs_addr; /* Temporary holder for free space manager address */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
+
+ /* check args */
+ HDassert(f);
+ if(H5F_PAGED_AGGR(f))
+ HDassert(type < H5F_MEM_PAGE_NTYPES);
+ else
+ HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES);
+ HDassert(H5F_addr_defined(f->shared->fs_addr[type]));
+
+ /* Put address into temporary variable and reset it */
+ /* (Avoids loopback in file space freeing routine) */
+ tmp_fs_addr = f->shared->fs_addr[type];
+ f->shared->fs_addr[type] = HADDR_UNDEF;
+
+ /* Shift to "deleting" state, to make certain we don't track any
+ * file space freed as a result of deleting the free space manager.
+ */
+ f->shared->fs_state[type] = H5F_FS_STATE_DELETING;
+
+ /* Set the ring type in the DXPL */
+ if(H5MF__fsm_type_is_self_referential(f, type))
+ fsm_ring = H5AC_RING_MDFSM;
+ else
+ fsm_ring = H5AC_RING_RDFSM;
+ if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+ reset_ring = TRUE;
+
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: Before deleting free space manager\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+
+ /* Delete free space manager for this type */
+ if(H5FS_delete(f, dxpl_id, tmp_fs_addr) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't delete free space manager")
+
+ /* Shift [back] to closed state */
+ HDassert(f->shared->fs_state[type] == H5F_FS_STATE_DELETING);
+ f->shared->fs_state[type] = H5F_FS_STATE_CLOSED;
+
+ /* Sanity check that the free space manager for this type wasn't started up again */
+ HDassert(!H5F_addr_defined(f->shared->fs_addr[type]));
+
+done:
+ /* Reset the ring in the DXPL */
+ if(reset_ring)
+ if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+ HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+
+ FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
+} /* end H5MF__delete_fstype() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF__close_fstype
+ *
+ * Purpose: Close the free space manager of TYPE for file
+ * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types.
*
* Return: Success: non-negative
* Failure: negative
@@ -402,7 +594,7 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
-H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type)
+H5MF__close_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type)
{
herr_t ret_value = SUCCEED; /* Return value */
@@ -412,11 +604,18 @@ H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type)
* Check arguments.
*/
HDassert(f);
+ if(H5F_PAGED_AGGR(f))
+ HDassert(type < H5F_MEM_PAGE_NTYPES);
+ else
+ HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES);
HDassert(f->shared);
- HDassert(type != H5FD_MEM_NOLIST);
HDassert(f->shared->fs_man[type]);
HDassert(f->shared->fs_state[type] != H5F_FS_STATE_CLOSED);
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: Before closing free space manager\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+
/* Close an existing free space structure for the file */
if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free space info")
@@ -425,7 +624,158 @@ H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type)
done:
FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
-} /* end H5MF__alloc_close() */
+} /* end H5MF__close_fstype() */
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF_add_sect
+ *
+ * Purpose: To add a section to the specified free-space manager.
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ * Programmer: Vailin Choi; April 2013
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5MF_add_sect(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, H5FS_t *fspace, H5MF_free_section_t *node)
+{
+ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
+ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
+ H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */
+ H5MF_sect_ud_t udata; /* User data for callback */
+ H5F_mem_page_t fs_type; /* Free space type (mapped from allocation type) */
+
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
+
+ HDassert(f);
+ HDassert(fspace);
+ HDassert(node);
+
+ H5MF_alloc_to_fs_type(f, alloc_type, node->sect_info.size, &fs_type);
+
+ /* Construct user data for callbacks */
+ udata.f = f;
+ udata.dxpl_id = dxpl_id;
+ udata.alloc_type = alloc_type;
+ udata.allow_sect_absorb = TRUE;
+ udata.allow_eoa_shrink_only = FALSE;
+
+ /* Set the ring type in the DXPL */
+ if(H5MF__fsm_is_self_referential(f, fspace))
+ fsm_ring = H5AC_RING_MDFSM;
+ else
+ fsm_ring = H5AC_RING_RDFSM;
+ if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: adding node, node->sect_info.addr = %a, node->sect_info.size = %Hu\n", FUNC, node->sect_info.addr, node->sect_info.size);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+ /* Add the section */
+ if(H5FS_sect_add(f, dxpl_id, fspace, (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, &udata) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't re-add section to file free space")
+
+done:
+ /* Reset the ring in the DXPL */
+ if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+ HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+ FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
+} /* end H5MF_add_sect() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF_find_sect
+ *
+ * Purpose: To find a section from the specified free-space manager to fulfill the request.
+ * If found, re-add the left-over space back to the manager.
+ *
+ * Return: TRUE if a section is found to fulfill the request
+ * FALSE if not
+ *
+ * Programmer: Vailin Choi; April 2013
+ *
+ *-------------------------------------------------------------------------
+ */
+htri_t
+H5MF_find_sect(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size, H5FS_t *fspace, haddr_t *addr)
+{
+ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
+ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
+ H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */
+ H5MF_free_section_t *node; /* Free space section pointer */
+ hbool_t reset_ring = FALSE; /* Whether the ring was set */
+ htri_t ret_value = FAIL; /* Whether an existing free list node was found */
+
+ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
+
+ HDassert(f);
+ HDassert(fspace);
+
+ /* Set the ring type in the DXPL */
+ if(H5MF__fsm_is_self_referential(f, fspace))
+ fsm_ring = H5AC_RING_MDFSM;
+ else
+ fsm_ring = H5AC_RING_RDFSM;
+ if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+ reset_ring = TRUE;
+
+ /* Try to get a section from the free space manager */
+ if((ret_value = H5FS_sect_find(f, dxpl_id, fspace, size, (H5FS_section_info_t **)&node)) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "error locating free space in file")
+
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: section found = %t\n", FUNC, ret_value);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+
+ /* Check for actually finding section */
+ if(ret_value) {
+ /* Sanity check */
+ HDassert(node);
+
+ /* Retrieve return value */
+ if(addr)
+ *addr = node->sect_info.addr;
+
+ /* Check for eliminating the section */
+ if(node->sect_info.size == size) {
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: freeing node\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+
+ /* Free section node */
+ if(H5MF_sect_free((H5FS_section_info_t *)node) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node")
+ } /* end if */
+ else {
+ /* Adjust information for section */
+ node->sect_info.addr += size;
+ node->sect_info.size -= size;
+
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: re-adding node, node->sect_info.size = %Hu\n", FUNC, node->sect_info.size);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+
+ /* Re-add the section to the free-space manager */
+ if(H5MF_add_sect(f, alloc_type, dxpl_id, fspace, node) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't re-add section to file free space")
+ } /* end else */
+ } /* end if */
+
+done:
+ /* Reset the ring in the DXPL */
+ if(reset_ring)
+ if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+ HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+
+ FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
+} /* end H5MF_find_sect() */
/*-------------------------------------------------------------------------
@@ -451,7 +801,7 @@ H5MF_alloc(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size)
H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
H5AC_ring_t fsm_ring = H5AC_RING_INV; /* free space manager ring */
H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
- H5FD_mem_t fs_type; /* Free space type (mapped from allocation type) */
+ H5F_mem_page_t fs_type; /* Free space type (mapped from allocation type) */
hbool_t reset_ring = FALSE; /* Whether the ring was set */
haddr_t ret_value = HADDR_UNDEF; /* Return value */
@@ -466,12 +816,20 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ
HDassert(f->shared->lf);
HDassert(size > 0);
- /* Get free space type from allocation type */
- fs_type = H5MF_ALLOC_TO_FS_TYPE(f, alloc_type);
+ if(f->shared->first_alloc_dealloc) {
+ HDassert(! H5AC_cache_image_pending(f));
+ if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "tidy of self referential fsm hack failed")
+ } /* end if */
+
+ H5MF_alloc_to_fs_type(f, alloc_type, size, &fs_type);
+
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: Check 1.0\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
/* Set the ring type in the DXPL */
- if((fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
- || (fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+ if(H5MF__fsm_type_is_self_referential(f, fs_type))
fsm_ring = H5AC_RING_MDFSM;
else
fsm_ring = H5AC_RING_RDFSM;
@@ -481,74 +839,46 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ
/* Check if we are using the free space manager for this file */
if(H5F_HAVE_FREE_SPACE_MANAGER(f)) {
+ /* We are about to change the contents of the free space manager --
+ * notify metadata cache that the associated fsm ring is
+ * unsettled
+ */
+ if(H5AC_unsettle_ring(f, fsm_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_SYSTEM, HADDR_UNDEF, "attempt to notify cache that ring is unsettled failed")
+
/* Check if the free space manager for the file has been initialized */
- if(!f->shared->fs_man[fs_type] && H5F_addr_defined(f->shared->fs_addr[fs_type]))
- if(H5MF__alloc_open(f, dxpl_id, fs_type) < 0)
+ if(!f->shared->fs_man[fs_type] && H5F_addr_defined(f->shared->fs_addr[fs_type])) {
+ /* Open the free-space manager */
+ if(H5MF_open_fstype(f, dxpl_id, fs_type) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTOPENOBJ, HADDR_UNDEF, "can't initialize file free space")
+ HDassert(f->shared->fs_man[fs_type]);
+ } /* end if */
/* Search for large enough space in the free space manager */
- if(f->shared->fs_man[fs_type]) {
- H5MF_free_section_t *node; /* Free space section pointer */
- htri_t node_found = FALSE; /* Whether an existing free list node was found */
-
- /* Try to get a section from the free space manager */
- if((node_found = H5FS_sect_find(f, dxpl_id, f->shared->fs_man[fs_type], size, (H5FS_section_info_t **)&node)) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "error locating free space in file")
-#ifdef H5MF_ALLOC_DEBUG_MORE
-HDfprintf(stderr, "%s: Check 1.5, node_found = %t\n", FUNC, node_found);
-#endif /* H5MF_ALLOC_DEBUG_MORE */
-
- /* Check for actually finding section */
- if(node_found) {
- /* Sanity check */
- HDassert(node);
-
- /* Retrieve return value */
- ret_value = node->sect_info.addr;
-
- /* Check for eliminating the section */
- if(node->sect_info.size == size) {
-#ifdef H5MF_ALLOC_DEBUG_MORE
-HDfprintf(stderr, "%s: Check 1.6, freeing node\n", FUNC);
-#endif /* H5MF_ALLOC_DEBUG_MORE */
- /* Free section node */
- if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, HADDR_UNDEF, "can't free simple section node")
- } /* end if */
- else {
- H5MF_sect_ud_t udata; /* User data for callback */
-
- /* Adjust information for section */
- node->sect_info.addr += size;
- node->sect_info.size -= size;
-
- /* Construct user data for callbacks */
- udata.f = f;
- udata.dxpl_id = dxpl_id;
- udata.alloc_type = alloc_type;
- udata.allow_sect_absorb = TRUE;
- udata.allow_eoa_shrink_only = FALSE;
+ if(f->shared->fs_man[fs_type])
+ if(H5MF_find_sect(f, alloc_type, dxpl_id, size, f->shared->fs_man[fs_type], &ret_value) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "error locating a node")
+ } /* end if */
+ /* If no space is found from the free-space manager, continue further action */
+ if(!H5F_addr_defined(ret_value)) {
#ifdef H5MF_ALLOC_DEBUG_MORE
-HDfprintf(stderr, "%s: Check 1.7, re-adding node, node->sect_info.size = %Hu\n", FUNC, node->sect_info.size);
+HDfprintf(stderr, "%s: Check 2.0\n", FUNC);
#endif /* H5MF_ALLOC_DEBUG_MORE */
- /* Re-insert section node into file's free space */
- if(H5FS_sect_add(f, dxpl_id, f->shared->fs_man[fs_type], (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, &udata) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, HADDR_UNDEF, "can't re-add section to file free space")
- } /* end else */
-
- /* Leave now */
- HGOTO_DONE(ret_value)
- } /* end if */
+ if(f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE) {
+ HDassert(f->shared->fs_page_size >= H5F_FILE_SPACE_PAGE_SIZE_MIN);
+ if(HADDR_UNDEF == (ret_value = H5MF__alloc_pagefs(f, alloc_type, dxpl_id, size)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "allocation failed from paged aggregation")
} /* end if */
+ else { /* For non-paged aggregation, continue further action */
+ if(HADDR_UNDEF == (ret_value = H5MF_aggr_vfd_alloc(f, alloc_type, dxpl_id, size)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "allocation failed from aggr/vfd")
+ } /* end else */
+ } /* end if */
+ HDassert(H5F_addr_defined(ret_value));
#ifdef H5MF_ALLOC_DEBUG_MORE
-HDfprintf(stderr, "%s: Check 2.0\n", FUNC);
+HDfprintf(stderr, "%s: Check 3.0\n", FUNC);
#endif /* H5MF_ALLOC_DEBUG_MORE */
- } /* end if */
-
- /* Allocate from the metadata aggregator (or the VFD) */
- if(HADDR_UNDEF == (ret_value = H5MF_aggr_vfd_alloc(f, alloc_type, dxpl_id, size)))
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "allocation failed from aggr/vfd")
done:
/* Reset the ring in the DXPL */
@@ -568,6 +898,146 @@ H5MF_sects_dump(f, dxpl_id, stderr);
/*-------------------------------------------------------------------------
+ * Function: H5MF__alloc_pagefs
+ *
+ * Purpose: Allocate space from either the large or small free-space manager.
+ * For "large" request:
+ * Allocate request from VFD
+ * Determine mis-aligned fragment and return the fragment to the
+ * appropriate manager
+ * For "small" request:
+ * Allocate a page from the large manager
+ * Determine whether space is available from a mis-aligned fragment
+ * being returned to the manager
+ * Return left-over space to the manager after fulfilling request
+ *
+ * Return: Success: The file address of new chunk.
+ * Failure: HADDR_UNDEF
+ *
+ * Programmer: Vailin Choi; Dec 2012
+ *
+ *-------------------------------------------------------------------------
+ */
+static haddr_t
+H5MF__alloc_pagefs(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size)
+{
+ H5F_mem_page_t ptype; /* Free-space mananger type */
+ H5MF_free_section_t *node = NULL; /* Free space section pointer */
+ haddr_t ret_value = HADDR_UNDEF; /* Return value */
+
+ FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, HADDR_UNDEF)
+
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_type, size);
+#endif /* H5MF_ALLOC_DEBUG */
+
+ H5MF_alloc_to_fs_type(f, alloc_type, size, &ptype);
+
+ switch(ptype) {
+ case H5F_MEM_PAGE_GENERIC:
+ case H5F_MEM_PAGE_LARGE_BTREE:
+ case H5F_MEM_PAGE_LARGE_DRAW:
+ case H5F_MEM_PAGE_LARGE_GHEAP:
+ case H5F_MEM_PAGE_LARGE_LHEAP:
+ case H5F_MEM_PAGE_LARGE_OHDR:
+ {
+ haddr_t eoa; /* EOA for the file */
+ hsize_t frag_size = 0; /* Fragment size */
+
+ /* Get the EOA for the file */
+ if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, alloc_type)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, HADDR_UNDEF, "Unable to get eoa")
+ HDassert(!(eoa % f->shared->fs_page_size));
+
+ H5MF_EOA_MISALIGN(f, (eoa+size), f->shared->fs_page_size, frag_size);
+
+ /* Allocate from VFD */
+ if(HADDR_UNDEF == (ret_value = H5F_alloc(f, dxpl_id, alloc_type, size + frag_size, NULL, NULL)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space")
+
+ /* If there is a mis-aligned fragment at EOA */
+ if(frag_size) {
+
+ /* Start up the free-space manager */
+ if(!(f->shared->fs_man[ptype]))
+ if(H5MF_start_fstype(f, dxpl_id, ptype) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, HADDR_UNDEF, "can't initialize file free space")
+
+ /* Create free space section for the fragment */
+ if(NULL == (node = H5MF_sect_new(H5MF_FSPACE_SECT_LARGE, ret_value + size, frag_size)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, HADDR_UNDEF, "can't initialize free space section")
+
+ /* Add the fragment to the large free-space manager */
+ if(H5MF_add_sect(f, alloc_type, dxpl_id, f->shared->fs_man[ptype], node) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, HADDR_UNDEF, "can't re-add section to file free space")
+
+ node = NULL;
+ } /* end if */
+ }
+ break;
+
+ case H5F_MEM_PAGE_META:
+ case H5F_MEM_PAGE_DRAW:
+ case H5F_MEM_PAGE_BTREE:
+ case H5F_MEM_PAGE_GHEAP:
+ case H5F_MEM_PAGE_LHEAP:
+ case H5F_MEM_PAGE_OHDR:
+ {
+ haddr_t new_page; /* The address for the new file size page */
+
+ /* Allocate one file space page */
+ if(HADDR_UNDEF == (new_page = H5MF_alloc(f, alloc_type, dxpl_id, f->shared->fs_page_size)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space")
+
+ /* Start up the free-space manager */
+ if(!(f->shared->fs_man[ptype]))
+ if(H5MF_start_fstype(f, dxpl_id, ptype) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, HADDR_UNDEF, "can't initialize file free space")
+ HDassert(f->shared->fs_man[ptype]);
+
+ if(NULL == (node = H5MF_sect_new(H5MF_FSPACE_SECT_SMALL, (new_page + size), (f->shared->fs_page_size - size))))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, HADDR_UNDEF, "can't initialize free space section")
+
+ /* Add the remaining space in the page to the manager */
+ if(H5MF_add_sect(f, alloc_type, dxpl_id, f->shared->fs_man[ptype], node) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, HADDR_UNDEF, "can't re-add section to file free space")
+
+ node = NULL;
+
+ /* Insert the new page into the Page Buffer list of new pages so
+ we don't read an empty page from disk */
+ if(f->shared->page_buf != NULL && H5PB_add_new_page(f, alloc_type, new_page) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, HADDR_UNDEF, "can't add new page to Page Buffer new page list")
+
+ ret_value = new_page;
+ }
+ break;
+
+ case H5F_MEM_PAGE_NTYPES:
+ case H5F_MEM_PAGE_DEFAULT:
+ default:
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space: unrecognized type")
+ break;
+ } /* end switch */
+
+done:
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", FUNC, ret_value, size);
+#endif /* H5MF_ALLOC_DEBUG */
+#ifdef H5MF_ALLOC_DEBUG_DUMP
+H5MF_sects_dump(f, dxpl_id, stderr);
+#endif /* H5MF_ALLOC_DEBUG_DUMP */
+
+ /* Release section node, if allocated and not added to section list or merged */
+ if(node)
+ if(H5MF_sect_free((H5FS_section_info_t *)node) < 0)
+ HDONE_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, HADDR_UNDEF, "can't free section node")
+
+ FUNC_LEAVE_NOAPI_TAG(ret_value, HADDR_UNDEF)
+} /* end H5MF__alloc_pagefs() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5MF_alloc_tmp
*
* Purpose: Allocate temporary space in the file
@@ -645,14 +1115,14 @@ herr_t
H5MF_xfree(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr,
hsize_t size)
{
- H5F_io_info_t fio_info; /* I/O info for operation */
+ H5F_io_info2_t fio_info; /* I/O info for operation */
+ H5F_mem_page_t fs_type; /* Free space type (mapped from allocation type) */
H5MF_free_section_t *node = NULL; /* Free space section pointer */
- H5MF_sect_ud_t udata; /* User data for callback */
+ unsigned ctype; /* section class type */
H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
- H5AC_ring_t fsm_ring; /* Free space manager ring */
- H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
+ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
+ H5AC_ring_t fsm_ring = H5AC_RING_INV; /* Ring of fsm */
hbool_t reset_ring = FALSE; /* Whether the ring was set */
- H5FD_mem_t fs_type; /* Free space type (mapped from allocation type) */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
@@ -666,19 +1136,16 @@ HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUN
HGOTO_DONE(SUCCEED)
HDassert(addr != 0); /* Can't deallocate the superblock :-) */
- /* Check for attempting to free space that's a 'temporary' file address */
- if(H5F_addr_le(f->shared->tmp_addr, addr))
- HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, FAIL, "attempting to free temporary file space")
+ if(f->shared->first_alloc_dealloc) {
+ HDassert(!H5AC_cache_image_pending(f));
+ if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed")
+ } /* end if */
- /* Get free space type from allocation type */
- fs_type = H5MF_ALLOC_TO_FS_TYPE(f, alloc_type);
-#ifdef H5MF_ALLOC_DEBUG_MORE
-HDfprintf(stderr, "%s: fs_type = %u\n", FUNC, (unsigned)fs_type);
-#endif /* H5MF_ALLOC_DEBUG_MORE */
+ H5MF_alloc_to_fs_type(f, alloc_type, size, &fs_type);
/* Set the ring type in the DXPL */
- if((fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
- || (fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+ if(H5MF__fsm_type_is_self_referential(f, fs_type))
fsm_ring = H5AC_RING_MDFSM;
else
fsm_ring = H5AC_RING_RDFSM;
@@ -686,10 +1153,33 @@ HDfprintf(stderr, "%s: fs_type = %u\n", FUNC, (unsigned)fs_type);
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
reset_ring = TRUE;
+ /* we are about to change the contents of the free space manager --
+ * notify metadata cache that the associated fsm ring is
+ * unsettled
+ */
+ /* Only do so for strategies that use free-space managers */
+ if(H5F_HAVE_FREE_SPACE_MANAGER(f))
+ if(H5AC_unsettle_ring(f, fsm_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_SYSTEM, FAIL, "attempt to notify cache that ring is unsettled failed")
+
+ /* Check for attempting to free space that's a 'temporary' file address */
+ if(H5F_addr_le(f->shared->tmp_addr, addr))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, FAIL, "attempting to free temporary file space")
+
/* Set up I/O info for operation */
fio_info.f = f;
- if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
- HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ if(H5FD_MEM_DRAW == alloc_type) {
+ if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ } /* end if */
+ else {
+ if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list")
+ } /* end else */
/* Check if the space to free intersects with the file's metadata accumulator */
if(H5F__accum_free(&fio_info, alloc_type, addr, size) < 0)
@@ -702,7 +1192,7 @@ HDfprintf(stderr, "%s: fs_type = %u\n", FUNC, (unsigned)fs_type);
* space is at the end of the file
*/
#ifdef H5MF_ALLOC_DEBUG_MORE
-HDfprintf(stderr, "%s: f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)fs_type, f->shared->fs_addr[fs_type]);
+HDfprintf(stderr, "%s: fs_addr = %a\n", FUNC, f->shared->fs_addr[fs_type]);
#endif /* H5MF_ALLOC_DEBUG_MORE */
if(!H5F_addr_defined(f->shared->fs_addr[fs_type])) {
htri_t status; /* "can absorb" status for section into */
@@ -726,8 +1216,8 @@ HDfprintf(stderr, "%s: dropping addr = %a, size = %Hu, on the floor!\n", FUNC, a
/* If we are deleting the free space manager, leave now, to avoid
* [re-]starting it.
- * or if file space strategy type is not using a free space manager
- * (H5F_FILE_SPACE_AGGR_VFD or H5F_FILE_SPACE_VFD), drop free space
+ * or if file space strategy type is not using a free space manager
+ * (H5F_FSPACE_STRATEGY_AGGR or H5F_FSPACE_STRATEGY_NONE), drop free space
* section on the floor.
*
* Note: this drops the space to free on the floor...
@@ -745,38 +1235,42 @@ HDfprintf(stderr, "%s: dropping addr = %a, size = %Hu, on the floor!\n", FUNC, a
* space isn't at the end of the file, so start up (or create)
* the file space manager
*/
- if(H5MF__alloc_start(f, dxpl_id, fs_type) < 0)
+ if(H5MF_start_fstype(f, dxpl_id, fs_type) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space")
} /* end if */
- /* Create free space section for block */
- if(NULL == (node = H5MF_sect_simple_new(addr, size)))
+ /* Create the free-space section for the freed section */
+ ctype = H5MF_SECT_CLASS_TYPE(f, size);
+ if(NULL == (node = H5MF_sect_new(ctype, addr, size)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section")
- /* Construct user data for callbacks */
- udata.f = f;
- udata.dxpl_id = dxpl_id;
- udata.alloc_type = alloc_type;
- udata.allow_sect_absorb = TRUE;
- udata.allow_eoa_shrink_only = FALSE;
-
- /* If size of section freed is larger than threshold, add it to the free space manager */
+ /* If size of the freed section is larger than threshold, add it to the free space manager */
if(size >= f->shared->fs_threshold) {
HDassert(f->shared->fs_man[fs_type]);
#ifdef H5MF_ALLOC_DEBUG_MORE
HDfprintf(stderr, "%s: Before H5FS_sect_add()\n", FUNC);
#endif /* H5MF_ALLOC_DEBUG_MORE */
+
/* Add to the free space for the file */
- if(H5FS_sect_add(f, dxpl_id, f->shared->fs_man[fs_type], (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, &udata) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't add section to file free space")
- node = NULL;
+ if(H5MF_add_sect(f, alloc_type, dxpl_id, f->shared->fs_man[fs_type], node) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't add section to file free space")
+ node = NULL;
+
#ifdef H5MF_ALLOC_DEBUG_MORE
HDfprintf(stderr, "%s: After H5FS_sect_add()\n", FUNC);
#endif /* H5MF_ALLOC_DEBUG_MORE */
} /* end if */
else {
htri_t merged; /* Whether node was merged */
+ H5MF_sect_ud_t udata; /* User data for callback */
+
+ /* Construct user data for callbacks */
+ udata.f = f;
+ udata.dxpl_id = dxpl_id;
+ udata.alloc_type = alloc_type;
+ udata.allow_sect_absorb = TRUE;
+ udata.allow_eoa_shrink_only = FALSE;
/* Try to merge the section that is smaller than threshold */
if((merged = H5FS_sect_try_merge(f, dxpl_id, f->shared->fs_man[fs_type], (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, &udata)) < 0)
@@ -794,7 +1288,7 @@ done:
/* Release section node, if allocated and not added to section list or merged */
if(node)
- if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0)
+ if(H5MF_sect_free((H5FS_section_info_t *)node) < 0)
HDONE_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node")
#ifdef H5MF_ALLOC_DEBUG
@@ -811,6 +1305,14 @@ H5MF_sects_dump(f, dxpl_id, stderr);
* Function: H5MF_try_extend
*
* Purpose: Extend a block in the file if possible.
+ * For non-paged aggregation:
+ * --try to extend at EOA
+ * --try to extend into the aggregators
+ * --try to extend into a free-space section if adjoined
+ * For paged aggregation:
+ * --try to extend at EOA
+ * --try to extend into a free-space section if adjoined
+ * --try to extend into the page end threshold if a metadata block
*
* Return: Success: TRUE(1) - Block was extended
* FALSE(0) - Block could not be extended
@@ -825,14 +1327,16 @@ htri_t
H5MF_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type, haddr_t addr,
hsize_t size, hsize_t extra_requested)
{
- H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
- H5AC_ring_t fsm_ring; /* Free space manager ring */
- H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
- haddr_t end; /* End of block to extend */
- H5FD_mem_t fs_type; /* Memory type of the free space manager */
- H5FD_mem_t map_type; /* Mapped type */
- hbool_t reset_ring = FALSE; /* Whether the ring was set */
- htri_t ret_value = FAIL; /* Return value */
+ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
+ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
+ H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */
+ haddr_t end; /* End of block to extend */
+ H5FD_mem_t map_type; /* Mapped type */
+ H5F_mem_page_t fs_type; /* free space type */
+ htri_t allow_extend = TRUE; /* Possible to extend the block */
+ hsize_t frag_size = 0; /* Size of mis-aligned fragment */
+ hbool_t reset_ring = FALSE; /* Whether the ring was set */
+ htri_t ret_value = FALSE; /* Return value */
FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
#ifdef H5MF_ALLOC_DEBUG
@@ -849,12 +1353,33 @@ HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_r
/* Compute end of block to extend */
end = addr + size;
+ /* For paged aggregation:
+ * To extend a small block: can only extend if not crossing page boundary
+ * To extend a large block at EOA: calculate in advance mis-aligned fragment so EOA will still end at page boundary
+ */
+ if(H5F_PAGED_AGGR(f)) {
+ if(size < f->shared->fs_page_size) {
+ /* To extend a small block: cannot cross page boundary */
+ if((addr / f->shared->fs_page_size) != (((end + extra_requested) - 1) / f->shared->fs_page_size))
+ allow_extend = FALSE;
+ } /* end if */
+ else {
+ haddr_t eoa; /* EOA for the file */
+
+ /* To extend a large block: calculate in advance the mis-aligned fragment so EOA will end at page boundary if extended */
+ if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, alloc_type)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "Unable to get eoa")
+ HDassert(!(eoa % f->shared->fs_page_size));
+
+ H5MF_EOA_MISALIGN(f, (eoa+extra_requested), f->shared->fs_page_size, frag_size);
+ } /* end else */
+ } /* end if */
+
/* Get free space type from allocation type */
- fs_type = H5MF_ALLOC_TO_FS_TYPE(f, alloc_type);
+ H5MF_alloc_to_fs_type(f, alloc_type, size, &fs_type);
/* Set the ring type in the DXPL */
- if((fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
- || (fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+ if(H5MF__fsm_type_is_self_referential(f, fs_type))
fsm_ring = H5AC_RING_MDFSM;
else
fsm_ring = H5AC_RING_RDFSM;
@@ -862,29 +1387,90 @@ HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_r
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
reset_ring = TRUE;
- /* Check if the block is exactly at the end of the file */
- if((ret_value = H5FD_try_extend(f->shared->lf, map_type, f, dxpl_id, end, extra_requested)) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file")
- else if(ret_value == FALSE) {
- H5F_blk_aggr_t *aggr; /* Aggregator to use */
+ if(allow_extend) {
+ /* Try extending the block at EOA */
+ if((ret_value = H5F_try_extend(f, dxpl_id, map_type, end, extra_requested + frag_size)) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file")
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: extended = %t\n", FUNC, ret_value);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+
+ /* If extending at EOA succeeds: */
+ /* for paged aggregation, put the fragment into the large-sized free-space manager */
+ if(ret_value == TRUE && H5F_PAGED_AGGR(f) && frag_size) {
+ H5MF_free_section_t *node = NULL; /* Free space section pointer */
- /* Check for test block able to extend aggregation block */
- aggr = (map_type == H5FD_MEM_DRAW) ? &(f->shared->sdata_aggr) : &(f->shared->meta_aggr);
- if((ret_value = H5MF_aggr_try_extend(f, dxpl_id, aggr, map_type, end, extra_requested)) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending aggregation block")
- else if(ret_value == FALSE) {
+ /* Should be large-sized block */
+ HDassert(size >= f->shared->fs_page_size);
+
+ /* Start up the free-space manager */
+ if(!(f->shared->fs_man[fs_type]))
+ if(H5MF_start_fstype(f, dxpl_id, fs_type) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space")
+
+ /* Create free space section for the fragment */
+ if(NULL == (node = H5MF_sect_new(H5MF_FSPACE_SECT_LARGE, end + extra_requested, frag_size)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section")
+
+ /* Add the fragment to the large-sized free-space manager */
+ if(H5MF_add_sect(f, alloc_type, dxpl_id, f->shared->fs_man[fs_type], node) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't re-add section to file free space")
+
+ node = NULL;
+ } /* end if */
+
+ /* For non-paged aggregation: try to extend into the aggregators */
+ if(ret_value == FALSE && (f->shared->fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR ||
+ f->shared->fs_strategy == H5F_FSPACE_STRATEGY_AGGR) ) {
+ H5F_blk_aggr_t *aggr; /* Aggregator to use */
+
+ /* Check if the block is able to extend into aggregation block */
+ aggr = (map_type == H5FD_MEM_DRAW) ? &(f->shared->sdata_aggr) : &(f->shared->meta_aggr);
+ if((ret_value = H5MF_aggr_try_extend(f, dxpl_id, aggr, map_type, end, extra_requested)) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending aggregation block")
+
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: H5MF_aggr_try_extend = %t\n", FUNC, ret_value);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+ } /* end if */
+
+ /* If no extension so far, try to extend into a free-space section */
+ if(ret_value == FALSE && ((f->shared->fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR) ||
+ (H5F_PAGED_AGGR(f))) ) {
+ H5MF_sect_ud_t udata; /* User data */
+
+ /* Construct user data for callbacks */
+ udata.f = f;
+ udata.dxpl_id = dxpl_id;
+ udata.alloc_type = alloc_type;
/* Check if the free space for the file has been initialized */
if(!f->shared->fs_man[fs_type] && H5F_addr_defined(f->shared->fs_addr[fs_type]))
- if(H5MF__alloc_open(f, dxpl_id, fs_type) < 0)
+ /* Open the free-space manager */
+ if(H5MF_open_fstype(f, dxpl_id, fs_type) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space")
- /* Check for test block able to block in free space manager */
- if(f->shared->fs_man[fs_type])
- if((ret_value = H5FS_sect_try_extend(f, dxpl_id, f->shared->fs_man[fs_type], addr, size, extra_requested)) < 0)
+ /* Try to extend the block into a free-space section */
+ if(f->shared->fs_man[fs_type]) {
+ if((ret_value = H5FS_sect_try_extend(f, dxpl_id, f->shared->fs_man[fs_type], addr, size, extra_requested, H5FS_ADD_RETURNED_SPACE, &udata)) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending block in free space manager")
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: Try to H5FS_sect_try_extend = %t\n", FUNC, ret_value);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+ } /* end if */
+
+ /* For paged aggregation and a metadata block: try to extend into page end threshold */
+ if(ret_value == FALSE && H5F_PAGED_AGGR(f) && map_type != H5FD_MEM_DRAW) {
+ H5MF_EOA_MISALIGN(f, end, f->shared->fs_page_size, frag_size);
+
+ if(frag_size <= H5F_PGEND_META_THRES(f) && extra_requested <= frag_size)
+ ret_value = TRUE;
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: Try to extend into the page end threshold = %t\n", FUNC, ret_value);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+ } /* end if */
} /* end if */
- } /* end else-if */
+ } /* allow_extend */
done:
/* Reset the ring in the DXPL */
@@ -904,174 +1490,316 @@ H5MF_sects_dump(f, dxpl_id, stderr);
/*-------------------------------------------------------------------------
- * Function: H5MF_get_freespace
+ * Function: H5MF_try_shrink
*
- * Purpose: Retrieve the amount of free space in a file.
+ * Purpose: Try to shrink the size of a file with a block or absorb it
+ * into a block aggregator.
*
- * Return: Success: Amount of free space in file
- * Failure: Negative
+ * Return: Non-negative on success/Negative on failure
*
* Programmer: Quincey Koziol
- * Monday, October 6, 2003
+ * koziol@hdfgroup.org
+ * Feb 14 2008
*
- * Modifications:
- * Vailin Choi; July 2012
- * As the default free-list mapping is changed to H5FD_FLMAP_DICHOTOMY,
- * checks are added to account for the last section of each free-space manager
- * and the remaining space in the two aggregators are at EOF.
*-------------------------------------------------------------------------
*/
-herr_t
-H5MF_get_freespace(H5F_t *f, hid_t dxpl_id, hsize_t *tot_space, hsize_t *meta_size)
+htri_t
+H5MF_try_shrink(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr,
+ hsize_t size)
{
- H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
- H5AC_ring_t curr_ring; /* Current ring value */
- H5AC_ring_t needed_ring; /* Ring value needed for loop iteration */
- H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
- haddr_t eoa; /* End of allocated space in the file */
- haddr_t ma_addr = HADDR_UNDEF; /* Base "metadata aggregator" address */
- hsize_t ma_size = 0; /* Size of "metadata aggregator" */
- haddr_t sda_addr = HADDR_UNDEF; /* Base "small data aggregator" address */
- hsize_t sda_size = 0; /* Size of "small data aggregator" */
- hsize_t tot_fs_size = 0; /* Amount of all free space managed */
- hsize_t tot_meta_size = 0; /* Amount of metadata for free space managers */
- H5FD_mem_t type; /* Memory type for iteration */
- hbool_t fs_started[H5FD_MEM_NTYPES]; /* Indicate whether the free-space manager has been started */
- hbool_t eoa_shrank; /* Whether an EOA shrink occurs */
- hbool_t reset_ring = FALSE; /* Whether the ring was set */
- herr_t ret_value = SUCCEED; /* Return value */
+ H5MF_free_section_t *node = NULL; /* Free space section pointer */
+ H5MF_sect_ud_t udata; /* User data for callback */
+ H5FS_section_class_t *sect_cls; /* Section class */
+ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
+ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
+ H5AC_ring_t fsm_ring = H5AC_RING_INV; /* Ring of fsm */
+ H5F_mem_page_t fs_type; /* Free space type */
+ hbool_t reset_ring = FALSE; /* Whether the ring was set */
+ htri_t ret_value = FAIL; /* Return value */
FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUNC, (unsigned)alloc_type, addr, size);
+#endif /* H5MF_ALLOC_DEBUG */
- /* check args */
+ /* check arguments */
HDassert(f);
HDassert(f->shared);
HDassert(f->shared->lf);
+ HDassert(H5F_addr_defined(addr));
+ HDassert(size > 0);
- /* Retrieve the 'eoa' for the file */
- if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_DEFAULT)))
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed")
+ /* Set up free-space section class information */
+ sect_cls = H5MF_SECT_CLS_TYPE(f, size);
+ HDassert(sect_cls);
+
+ /* Get free space type from allocation type */
+ H5MF_alloc_to_fs_type(f, alloc_type, size, &fs_type);
/* Set the ring type in the DXPL */
- if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0)
+ if(H5MF__fsm_type_is_self_referential(f, fs_type))
+ fsm_ring = H5AC_RING_MDFSM;
+ else
+ fsm_ring = H5AC_RING_RDFSM;
+ if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
reset_ring = TRUE;
- curr_ring = H5AC_RING_RDFSM;
- /* Retrieve metadata aggregator info, if available */
- if(H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query metadata aggregator stats")
+ /* Create free-space section for block */
+ if(NULL == (node = H5MF_sect_new(sect_cls->type, addr, size)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section")
- /* Retrieve 'small data' aggregator info, if available */
- if(H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sda_addr, &sda_size) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query small data aggregator stats")
+ /* Construct user data for callbacks */
+ udata.f = f;
+ udata.dxpl_id = dxpl_id;
+ udata.alloc_type = alloc_type;
+ udata.allow_sect_absorb = FALSE; /* Force section to be absorbed into aggregator */
+ udata.allow_eoa_shrink_only = FALSE;
- /* Iterate over all the free space types that have managers and get each free list's space */
- for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
+ /* Check if the block can shrink the container */
+ if(sect_cls->can_shrink) {
+ if((ret_value = (*sect_cls->can_shrink)((const H5FS_section_info_t *)node, &udata)) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTMERGE, FAIL, "can't check if section can shrink container")
+ if(ret_value > 0) {
+ HDassert(sect_cls->shrink);
- fs_started[type] = FALSE;
+ if((*sect_cls->shrink)((H5FS_section_info_t **)&node, &udata) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink container")
+ } /* end if */
+ } /* end if */
- /* test to see if we need to switch rings -- do so if required */
- if((type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
- || (type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
- needed_ring = H5AC_RING_MDFSM;
- else
- needed_ring = H5AC_RING_RDFSM;
+done:
+ /* Reset the ring in the DXPL */
+ if(reset_ring)
+ if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+ HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
- if(needed_ring != curr_ring) {
- if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (2)")
- curr_ring = needed_ring;
- } /* end if */
+ /* Free section node allocated */
+ if(node && H5MF_sect_free((H5FS_section_info_t *)node) < 0)
+ HDONE_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node")
- /* Check if the free space for the file has been initialized */
- if(!f->shared->fs_man[type] && H5F_addr_defined(f->shared->fs_addr[type])) {
- if(H5MF__alloc_open(f, dxpl_id, type) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space")
- HDassert(f->shared->fs_man[type]);
- fs_started[type] = TRUE;
- } /* end if */
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value);
+#endif /* H5MF_ALLOC_DEBUG */
+ FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
+} /* end H5MF_try_shrink() */
- /* Check if there's free space of this type */
- if(f->shared->fs_man[type]) {
- hsize_t type_fs_size = 0; /* Amount of free space managed for each type */
- hsize_t type_meta_size = 0; /* Amount of free space metadata for each type */
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF_close
+ *
+ * Purpose: Close the free space tracker(s) for a file:
+ * paged or non-paged aggregation
+ *
+ * Return: SUCCEED/FAIL
+ *
+ * Programmer: Vailin Choi; Dec 2012
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5MF_close(H5F_t *f, hid_t dxpl_id)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
- /* Retrieve free space size from free space manager */
- if(H5FS_sect_stats(f->shared->fs_man[type], &type_fs_size, NULL) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space stats")
- if(H5FS_size(f, f->shared->fs_man[type], &type_meta_size) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space metadata stats")
+ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: Entering\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG */
- /* Increment total free space for types */
- tot_fs_size += type_fs_size;
- tot_meta_size += type_meta_size;
- } /* end if */
- } /* end for */
+ /* check args */
+ HDassert(f);
+ HDassert(f->shared);
- /* Iterate until no more EOA shrink occurs */
- do {
- eoa_shrank = FALSE;
+ if(H5F_PAGED_AGGR(f)) {
+ if((ret_value = H5MF__close_pagefs(f, dxpl_id)) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't close free-space managers for 'page' file space")
+ } /* end if */
+ else {
+ if((ret_value = H5MF__close_aggrfs(f, dxpl_id)) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't close free-space managers for 'aggr' file space")
+ } /* end else */
- /* Check the last section of each free-space manager */
- for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
- haddr_t sect_addr = HADDR_UNDEF;
- hsize_t sect_size = 0;
-
- if(f->shared->fs_man[type]) {
- if(H5FS_sect_query_last_sect(f->shared->fs_man[type], &sect_addr, &sect_size) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query last section on merge list")
-
- /* Deduct space from previous accumulation if the section is at EOA */
- if(H5F_addr_eq(sect_addr + sect_size, eoa)) {
- eoa = sect_addr;
- eoa_shrank = TRUE;
- tot_fs_size -= sect_size;
- } /* end if */
- } /* end if */
- } /* end for */
-
- /* Check the metadata and raw data aggregators */
- if(ma_size > 0 && H5F_addr_eq(ma_addr + ma_size, eoa)) {
- eoa = ma_addr;
- eoa_shrank = TRUE;
- ma_size = 0;
- } /* end if */
- if(sda_size > 0 && H5F_addr_eq(sda_addr + sda_size, eoa)) {
- eoa = sda_addr;
- eoa_shrank = TRUE;
- sda_size = 0;
- } /* end if */
- } while(eoa_shrank);
+done:
- /* Close the free-space managers if they were opened earlier in this routine */
- for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
- if(fs_started[type]) {
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: Leaving\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG */
+ FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
+} /* end H5MF_close() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF__close_delete_fstype
+ *
+ * Purpose: Common code for closing and deleting the freespace manager
+ * of TYPE for file.
+ * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ * Programmer: Vailin Choi
+ * Jan 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5MF__close_delete_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: Entering\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG */
+
+ /* check args */
+ HDassert(f);
+ HDassert(f->shared);
+ if(H5F_PAGED_AGGR(f))
+ HDassert(type < H5F_MEM_PAGE_NTYPES);
+ else
+ HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES);
+
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: Check 1.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+
+ /* If the free space manager for this type is open, close it */
+ if(f->shared->fs_man[type])
+ if(H5MF__close_fstype(f, dxpl_id, type) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close the free space manager")
+
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: Check 2.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+
+ /* If there is free space manager info for this type, delete it */
+ if(H5F_addr_defined(f->shared->fs_addr[type]))
+ if(H5MF__delete_fstype(f, dxpl_id, type) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't delete the free space manager")
+
+done:
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: Leaving\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG */
+ FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
+} /* H5MF__close_delete() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF_try_close
+ *
+ * Purpose: This is called by H5Fformat_convert() to close and delete
+ * free-space managers when downgrading persistent free-space
+ * to non-persistent.
+ *
+ * Return: SUCCEED/FAIL
+ *
+ * Programmer: Vailin Choi
+ * Jan 2016
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5MF_try_close(H5F_t *f, hid_t dxpl_id)
+{
+ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
+ H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */
+ H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration */
+ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
+ hbool_t reset_ring = FALSE; /* Whether the ring was set */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: Entering\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG */
+
+ /* check args */
+ HDassert(f);
+
+ /* If there have been no file space allocations / deallocation so
+ * far, must call H5MF_tidy_self_referential_fsm_hack() to float
+ * all self referential FSMs and release file space allocated to
+ * them. Otherwise, the function will be called after the format
+ * conversion, and will become very confused.
+ *
+ * The situation is further complicated if a cache image exists
+ * and had not yet been loaded into the metadata cache. In this
+ * case, call H5AC_force_cache_image_load() instead of
+ * H5MF_tidy_self_referential_fsm_hack(). H5AC_force_cache_image_load()
+ * will load the cache image, and then call
+ * H5MF_tidy_self_referential_fsm_hack() to discard the cache image
+ * block.
+ */
+ if(f->shared->first_alloc_dealloc) {
+ if(H5AC_cache_image_pending(f)) {
+ if(H5AC_force_cache_image_load(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "forced cache image load failed")
+ } /* end if */
+ else {
+ if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed")
+ } /* end else */
+ } /* end if */
+
+ /* Set the ring type in the DXPL. In most cases, we will
+ * need H5AC_RING_RDFSM, so initialy set the ring in
+ * the DXPL to that value. We will alter this later if
+ * needed.
+ */
+ if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+ reset_ring = TRUE;
+ curr_ring = H5AC_RING_RDFSM;
+
+ if(H5F_PAGED_AGGR(f)) {
+ H5F_mem_page_t ptype; /* Memory type for iteration */
+ /* Iterate over all the free space types that have managers and
+ * get each free list's space
+ */
+ for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) {
+ /* Test to see if we need to switch rings -- do so if required */
+ if(H5MF__fsm_type_is_self_referential(f, ptype))
+ needed_ring = H5AC_RING_MDFSM;
+ else
+ needed_ring = H5AC_RING_RDFSM;
+
+ if(needed_ring != curr_ring ) {
+ if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)")
+ curr_ring = needed_ring;
+ } /* end if */
+
+ if(H5MF__close_delete_fstype(f, dxpl_id, ptype) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close the free space manager")
+ } /* end for */
+ } /* end if */
+ else {
+ H5FD_mem_t type; /* Memory type for iteration */
+
+ /* Iterate over all the free space types that have managers and
+ * get each free list's space
+ */
+ for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
/* test to see if we need to switch rings -- do so if required */
- if((type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
- || (type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
+ if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type))
needed_ring = H5AC_RING_MDFSM;
else
needed_ring = H5AC_RING_RDFSM;
if(needed_ring != curr_ring) {
if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (3)")
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
curr_ring = needed_ring;
} /* end if */
- if(H5MF__alloc_close(f, dxpl_id, type) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't close file free space")
- } /* end if */
- } /* end for */
-
- /* Set the value(s) to return */
- /* (The metadata & small data aggregators count as free space now, since they aren't at EOA) */
- if(tot_space)
- *tot_space = tot_fs_size + ma_size + sda_size;
- if(meta_size)
- *meta_size = tot_meta_size;
+ if(H5MF__close_delete_fstype(f, dxpl_id, (H5F_mem_page_t)type) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close the free space manager")
+ } /* end for */
+ } /* end else */
done:
/* Reset the ring in the DXPL */
@@ -1079,75 +1807,370 @@ done:
if(H5AC_reset_ring(dxpl, orig_ring) < 0)
HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: Leaving\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG */
FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
-} /* end H5MF_get_freespace() */
+} /* H5MF_try_close() */
/*-------------------------------------------------------------------------
- * Function: H5MF_try_shrink
+ * Function: H5MF__close_aggrfs
*
- * Purpose: Try to shrink the size of a file with a block or absorb it
- * into a block aggregator.
+ * Purpose: Close the free space tracker(s) for a file: non-paged aggregation
*
- * Return: Non-negative on success/Negative on failure
+ * Return: SUCCEED/FAIL
*
* Programmer: Quincey Koziol
- * koziol@hdfgroup.org
- * Feb 14 2008
+ * Tuesday, January 22, 2008
*
*-------------------------------------------------------------------------
*/
-htri_t
-H5MF_try_shrink(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr,
- hsize_t size)
+static herr_t
+H5MF__close_aggrfs(H5F_t *f, hid_t dxpl_id)
{
- H5MF_free_section_t *node = NULL; /* Free space section pointer */
- H5MF_sect_ud_t udata; /* User data for callback */
- htri_t ret_value = FAIL; /* Return value */
+ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
+ H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */
+ H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration. */
+ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
+ H5FD_mem_t type; /* Memory type for iteration */
+ hbool_t reset_ring = FALSE; /* Whether the ring was set */
+ herr_t ret_value = SUCCEED; /* Return value */
- FUNC_ENTER_NOAPI(FAIL)
+ FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
#ifdef H5MF_ALLOC_DEBUG
-HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUNC, (unsigned)alloc_type, addr, size);
+HDfprintf(stderr, "%s: Entering\n", FUNC);
#endif /* H5MF_ALLOC_DEBUG */
- /* check arguments */
+ /* check args */
HDassert(f);
HDassert(f->shared);
HDassert(f->shared->lf);
- HDassert(H5F_addr_defined(addr));
- HDassert(size > 0);
+ HDassert(f->shared->sblock);
- /* Create free space section for block */
- if(NULL == (node = H5MF_sect_simple_new(addr, size)))
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section")
+ /* Set the ring type in the DXPL. In most cases, we will
+ * need H5AC_RING_RDFSM, so initialy set the ring in
+ * the DXPL to that value. We will alter this later if
+ * needed.
+ */
+ if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+ reset_ring = TRUE;
+ curr_ring = H5AC_RING_RDFSM;
- /* Construct user data for callbacks */
- udata.f = f;
- udata.dxpl_id = dxpl_id;
- udata.alloc_type = alloc_type;
- udata.allow_sect_absorb = FALSE; /* Force section to be absorbed into aggregator */
- udata.allow_eoa_shrink_only = FALSE;
-
- /* Call the "can shrink" callback for the section */
- if((ret_value = H5MF_sect_simple_can_shrink((const H5FS_section_info_t *)node, &udata)) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTMERGE, FAIL, "can't check if section can shrink container")
- else if(ret_value > 0) {
- /* Shrink or absorb the section */
- if(H5MF_sect_simple_shrink((H5FS_section_info_t **)&node, &udata) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink container")
+ /* Free the space in aggregators */
+ /* (for space not at EOA, it may be put into free space managers) */
+ if(H5MF_free_aggrs(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators")
+
+ /* Trying shrinking the EOA for the file */
+ if(H5MF__close_shrink_eoa(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
+
+ /* Making free-space managers persistent for superblock version >= 2 */
+ if(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2
+ && f->shared->fs_persist) {
+ H5O_fsinfo_t fsinfo; /* File space info message */
+ haddr_t final_eoa; /* Final eoa -- for sanity check */
+ H5F_mem_page_t ptype; /* Memory type for iteration */
+
+ /* superblock extension and free space manager message should
+ * exist at this point -- verify at least the former.
+ */
+ HDassert(H5F_addr_defined(f->shared->sblock->ext_addr));
+
+ /* file space for all non-empty free space managers should be
+ * allocated at this point, and these free space managers should
+ * be written to file and thus their headers and section info
+ * entries in the metadata cache should be clean.
+ */
+
+ /* gather data for the free space manager superblock extension message.
+ *
+ * In passing, verify that all the free space managers are closed.
+ */
+ for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype))
+ fsinfo.fs_addr[ptype - 1] = HADDR_UNDEF;
+ for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
+ fsinfo.fs_addr[type-1] = f->shared->fs_addr[type];
+ fsinfo.strategy = f->shared->fs_strategy;
+ fsinfo.persist = f->shared->fs_persist;
+ fsinfo.threshold = f->shared->fs_threshold;
+ fsinfo.page_size = f->shared->fs_page_size;
+ fsinfo.pgend_meta_thres = f->shared->pgend_meta_thres;
+ fsinfo.eoa_pre_fsm_fsalloc = f->shared->eoa_pre_fsm_fsalloc;
+
+ /* Write the free space manager message -- message must already exist */
+ if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension")
+
+ /* Close the free space managers */
+ for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
+ if(f->shared->fs_man[type]) {
+ /* test to see if we need to switch rings -- do
+ * so if required
+ */
+ if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type))
+ needed_ring = H5AC_RING_MDFSM;
+ else
+ needed_ring = H5AC_RING_RDFSM;
+
+ if(needed_ring != curr_ring) {
+ if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)")
+ curr_ring = needed_ring;
+ } /* end if */
+
+ HDassert(f->shared->fs_state[type] == H5F_FS_STATE_OPEN);
+
+ if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close free space manager")
+ f->shared->fs_man[type] = NULL;
+ f->shared->fs_state[type] = H5F_FS_STATE_CLOSED;
+ } /* end if */
+ f->shared->fs_addr[type] = HADDR_UNDEF;
+ } /* end for */
+
+ /* verify that we haven't dirtied any metadata cache entries
+ * from the metadata free space manager ring out.
+ */
+ HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM));
+
+ /* verify that the aggregators are still shutdown. */
+ HDassert(f->shared->sdata_aggr.tot_size == 0);
+ HDassert(f->shared->sdata_aggr.addr == 0);
+ HDassert(f->shared->sdata_aggr.size == 0);
+
+ HDassert(f->shared->meta_aggr.tot_size == 0);
+ HDassert(f->shared->meta_aggr.addr == 0);
+ HDassert(f->shared->meta_aggr.size == 0);
+
+ /* Trying shrinking the EOA for the file */
+ /* (in case any free space is now at the EOA) */
+ if(H5MF__close_shrink_eoa(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
+
+ /* get the eoa, and verify that it has the expected value */
+ if(HADDR_UNDEF == (final_eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)) )
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file size")
+
+ /* f->shared->eoa_post_fsm_fsalloc is undefined if there has
+ * been no file space allocation or deallocation since file
+ * open.
+ */
+ HDassert((f->shared->first_alloc_dealloc) || (final_eoa == f->shared->eoa_post_fsm_fsalloc));
} /* end if */
+ else { /* super_vers can be 0, 1, 2 */
+ for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
+ if(H5MF__close_delete_fstype(f, dxpl_id, (H5F_mem_page_t)type) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space")
+ } /* end else */
+
+ /* Free the space in aggregators (again) */
+ /* (in case any free space information re-started them) */
+ if(H5MF_free_aggrs(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators")
+
+ /* Trying shrinking the EOA for the file */
+ /* (in case any free space is now at the EOA) */
+ if(H5MF__close_shrink_eoa(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
done:
+ /* Reset the ring in the DXPL */
+ if(reset_ring)
+ if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+ HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
- /* Free section node allocated */
- if(node && H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0)
- HDONE_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node")
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: Leaving\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG */
+ FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
+} /* end H5MF__close_aggrfs() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF__close_pagefs
+ *
+ * Purpose: Close the free space tracker(s) for a file: paged aggregation
+ *
+ * Return: SUCCEED/FAIL
+ *
+ * Programmer: Vailin Choi; Dec 2012
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5MF__close_pagefs(H5F_t *f, hid_t dxpl_id)
+{
+ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
+ H5AC_ring_t curr_ring = H5AC_RING_INV; /* current ring value */
+ H5AC_ring_t needed_ring = H5AC_RING_INV; /* ring value needed for this
+ * iteration.
+ */
+ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
+ H5F_mem_page_t ptype; /* Memory type for iteration */
+ H5O_fsinfo_t fsinfo; /* File space info message */
+ hbool_t reset_ring = FALSE; /* Whether the ring was set */
+ herr_t ret_value = SUCCEED; /* Return value */
+ FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
#ifdef H5MF_ALLOC_DEBUG
-HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value);
+HDfprintf(stderr, "%s: Entering\n", FUNC);
#endif /* H5MF_ALLOC_DEBUG */
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5MF_try_shrink() */
+
+ /* check args */
+ HDassert(f);
+ HDassert(f->shared);
+ HDassert(f->shared->lf);
+ HDassert(f->shared->sblock);
+ HDassert(f->shared->fs_page_size);
+ HDassert(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2);
+
+ /* Set the ring type in the DXPL. In most cases, we will
+ * need H5AC_RING_RDFSM, so initialy set the ring in
+ * the DXPL to that value. We will alter this later if
+ * needed.
+ */
+ if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+ reset_ring = TRUE;
+ curr_ring = H5AC_RING_RDFSM;
+
+ /* Trying shrinking the EOA for the file */
+ if(H5MF__close_shrink_eoa(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
+
+ /* Set up file space info message */
+ fsinfo.strategy = f->shared->fs_strategy;
+ fsinfo.persist = f->shared->fs_persist;
+ fsinfo.threshold = f->shared->fs_threshold;
+ fsinfo.page_size = f->shared->fs_page_size;
+ fsinfo.pgend_meta_thres = f->shared->pgend_meta_thres;
+ fsinfo.eoa_pre_fsm_fsalloc = HADDR_UNDEF;
+ for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype))
+ fsinfo.fs_addr[ptype - 1] = HADDR_UNDEF;
+
+ if(f->shared->fs_persist) {
+ haddr_t final_eoa; /* final eoa -- for sanity check */
+
+ /* superblock extension and free space manager message should
+ * exist at this point -- verify at least the former.
+ */
+ HDassert(H5F_addr_defined(f->shared->sblock->ext_addr));
+
+ /* file space for all non-empty free space managers should be
+ * allocated at this point, and these free space managers should
+ * be written to file and thus their headers and section info
+ * entries in the metadata cache should be clean.
+ */
+
+ /* gather data for the free space manager superblock extension message.
+ * Only need addresses of FSMs and eoa prior to allocation of
+ * file space for the self referential free space managers. Other
+ * data was gathered above.
+ */
+ for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype))
+ fsinfo.fs_addr[ptype-1] = f->shared->fs_addr[ptype];
+ fsinfo.eoa_pre_fsm_fsalloc = f->shared->eoa_pre_fsm_fsalloc;
+
+ /* Write the free space manager message -- message must already exist */
+ if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension")
+
+ /* Close the free space managers */
+ /* use H5MF__close_fstype() for this? */
+ for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) {
+ if(f->shared->fs_man[ptype]) {
+ /* test to see if we need to switch rings -- do
+ * so if required
+ */
+ if(H5MF__fsm_type_is_self_referential(f, ptype))
+ needed_ring = H5AC_RING_MDFSM;
+ else
+ needed_ring = H5AC_RING_RDFSM;
+
+ if(needed_ring != curr_ring) {
+ if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)")
+ curr_ring = needed_ring;
+ } /* end if */
+
+ HDassert(f->shared->fs_state[ptype] == H5F_FS_STATE_OPEN);
+
+ if(H5FS_close(f, dxpl_id, f->shared->fs_man[ptype]) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close free space manager")
+ f->shared->fs_man[ptype] = NULL;
+ f->shared->fs_state[ptype] = H5F_FS_STATE_CLOSED;
+ } /* end if */
+ f->shared->fs_addr[ptype] = HADDR_UNDEF;
+ } /* end for */
+
+ /* verify that we haven't dirtied any metadata cache entries
+ * from the metadata free space manager ring out.
+ */
+ HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM));
+
+ /* Trying shrinking the EOA for the file */
+ /* (in case any free space is now at the EOA) */
+ if(H5MF__close_shrink_eoa(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
+
+ /* get the eoa, and verify that it has the expected value */
+ if(HADDR_UNDEF == (final_eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)) )
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file size")
+
+ /* f->shared->eoa_post_fsm_fsalloc is undefined if there has
+ * been no file space allocation or deallocation since file
+ * open.
+ *
+ * If there is a cache image in the file at file open,
+ * f->shared->first_alloc_dealloc will always be FALSE unless
+ * the file is opened R/O, as otherwise, the image will have been
+ * read and discarded by this point.
+ *
+ * If a cache image was created on file close, the actual EOA
+ * should be in f->shared->eoa_post_mdci_fsalloc. Note that in
+ * this case, it is conceivable that f->shared->first_alloc_dealloc
+ * will still be TRUE, as the cache image is allocated directly from
+ * the file driver layer. However, as this possibility seems remote,
+ * it is ignored in the following assert.
+ */
+ HDassert((f->shared->first_alloc_dealloc) ||
+ (final_eoa == f->shared->eoa_post_fsm_fsalloc) ||
+ ((H5F_addr_defined(f->shared->eoa_post_mdci_fsalloc)) &&
+ (final_eoa == f->shared->eoa_post_mdci_fsalloc)));
+ } /* end if */
+ else {
+ /* Iterate over all the free space types that have managers
+ * and get each free list's space
+ */
+ for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype))
+ if(H5MF__close_delete_fstype(f, dxpl_id, ptype) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close the free space manager")
+
+ /* Write file space info message to superblock extension object header */
+ /* Create the superblock extension object header in advance if needed */
+ if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension")
+ } /* end else */
+
+ /* Trying shrinking the EOA for the file */
+ /* (in case any free space is now at the EOA) */
+ if(H5MF__close_shrink_eoa(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
+
+done:
+ /* Reset the ring in the DXPL */
+ if(reset_ring)
+ if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+ HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: Leaving\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG */
+ FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
+} /* end H5MF__close_pagefs() */
/*-------------------------------------------------------------------------
@@ -1165,17 +2188,16 @@ HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value);
static herr_t
H5MF__close_shrink_eoa(H5F_t *f, hid_t dxpl_id)
{
- H5FD_mem_t type; /* Memory type for iteration */
H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
+ H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */
+ H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration. */
H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
- H5AC_ring_t curr_ring = H5AC_RING_INV; /* current ring value */
- H5AC_ring_t needed_ring = H5AC_RING_INV; /* ring value needed for this
- * iteration.
- */
+ H5F_mem_t type;
+ H5F_mem_page_t ptype; /* Memory type for iteration */
hbool_t eoa_shrank; /* Whether an EOA shrink occurs */
- hbool_t reset_ring = FALSE; /* Whether the ring was set */
htri_t status; /* Status value */
H5MF_sect_ud_t udata; /* User data for callback */
+ hbool_t reset_ring = FALSE; /* Whether the ring was set */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
@@ -1187,13 +2209,10 @@ H5MF__close_shrink_eoa(H5F_t *f, hid_t dxpl_id)
/* Construct user data for callbacks */
udata.f = f;
udata.dxpl_id = dxpl_id;
- udata.allow_sect_absorb = FALSE;
- udata.allow_eoa_shrink_only = TRUE;
+ udata.allow_sect_absorb = FALSE;
+ udata.allow_eoa_shrink_only = TRUE;
- /* Set the ring type in the DXPL. In most cases, we will
- * need H5AC_RING_RDFSM, so initialy set the ring type in
- * the DXPL to that value. We will alter this later if needed.
- */
+ /* Set the ring type in the DXPL */
if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value(1)")
reset_ring = TRUE;
@@ -1203,36 +2222,62 @@ H5MF__close_shrink_eoa(H5F_t *f, hid_t dxpl_id)
do {
eoa_shrank = FALSE;
- /* Check the last section of each free-space manager */
- for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
+ if(H5F_PAGED_AGGR(f)) {
+ /* Check the last section of each free-space manager */
+ for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) {
+ if(f->shared->fs_man[ptype]) {
+ /* Test to see if we need to switch rings -- do so if required */
+ if(H5MF__fsm_type_is_self_referential(f, ptype))
+ needed_ring = H5AC_RING_MDFSM;
+ else
+ needed_ring = H5AC_RING_RDFSM;
- /* test to see if we need to switch rings -- do so if required */
- if((H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
- || (H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
- needed_ring = H5AC_RING_MDFSM;
- else
- needed_ring = H5AC_RING_RDFSM;
+ if(needed_ring != curr_ring) {
+ if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)")
+ curr_ring = needed_ring;
+ } /* end if */
- if(needed_ring != curr_ring) {
- if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (2)")
- curr_ring = needed_ring;
- } /* end if */
+ udata.alloc_type = (H5FD_mem_t)((H5FD_mem_t)ptype < H5FD_MEM_NTYPES ? ptype : ((ptype % H5FD_MEM_NTYPES) + 1));
- if(f->shared->fs_man[type]) {
- udata.alloc_type = type;
- if((status = H5FS_sect_try_shrink_eoa(f, dxpl_id, f->shared->fs_man[type], &udata)) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa")
- else if(status > 0)
- eoa_shrank = TRUE;
- } /* end if */
- } /* end for */
+ if((status = H5FS_sect_try_shrink_eoa(f, dxpl_id, f->shared->fs_man[ptype], &udata)) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa")
+ else if(status > 0)
+ eoa_shrank = TRUE;
+ } /* end if */
+ } /* end for */
+ } /* end if */
+ else {
+ /* Check the last section of each free-space manager */
+ for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
+ if(f->shared->fs_man[type]) {
+ /* Test to see if we need to switch rings -- do so if required */
+ if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type))
+ needed_ring = H5AC_RING_MDFSM;
+ else
+ needed_ring = H5AC_RING_RDFSM;
+
+ if(needed_ring != curr_ring) {
+ if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)")
+ curr_ring = needed_ring;
+ } /* end if */
- /* check the two aggregators */
- if((status = H5MF_aggrs_try_shrink_eoa(f, dxpl_id)) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa")
- else if(status > 0)
- eoa_shrank = TRUE;
+ udata.alloc_type = type;
+
+ if((status = H5FS_sect_try_shrink_eoa(f, dxpl_id, f->shared->fs_man[type], &udata)) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa")
+ else if(status > 0)
+ eoa_shrank = TRUE;
+ } /* end if */
+ } /* end for */
+
+ /* check the two aggregators */
+ if((status = H5MF_aggrs_try_shrink_eoa(f, dxpl_id)) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa")
+ else if(status > 0)
+ eoa_shrank = TRUE;
+ } /* end else */
} while(eoa_shrank);
done:
@@ -1246,6 +2291,370 @@ done:
/*-------------------------------------------------------------------------
+ * Function: H5MF_get_freespace
+ *
+ * Purpose: Retrieve the amount of free space in the file
+ *
+ * Return: Success: Amount of free space in file
+ * Failure: Negative
+ *
+ * Programmer: Quincey Koziol
+ * Monday, October 6, 2003
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5MF_get_freespace(H5F_t *f, hid_t dxpl_id, hsize_t *tot_space, hsize_t *meta_size)
+{
+ haddr_t ma_addr = HADDR_UNDEF; /* Base "metadata aggregator" address */
+ hsize_t ma_size = 0; /* Size of "metadata aggregator" */
+ haddr_t sda_addr = HADDR_UNDEF; /* Base "small data aggregator" address */
+ hsize_t sda_size = 0; /* Size of "small data aggregator" */
+ hsize_t tot_fs_size = 0; /* Amount of all free space managed */
+ hsize_t tot_meta_size = 0; /* Amount of metadata for free space managers */
+ H5FD_mem_t tt; /* Memory type for iteration */
+ H5F_mem_page_t type; /* Memory type for iteration */
+ H5F_mem_page_t start_type; /* Memory type for iteration */
+ H5F_mem_page_t end_type; /* Memory type for iteration */
+ htri_t fs_started[H5F_MEM_PAGE_NTYPES]; /* Indicate whether the free-space manager has been started */
+ haddr_t fs_eoa[H5FD_MEM_NTYPES]; /* EAO for each free-space manager */
+ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
+ H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */
+ H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration. */
+ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
+ hbool_t reset_ring = FALSE; /* Whether the ring was set */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
+
+ /* check args */
+ HDassert(f);
+ HDassert(f->shared);
+ HDassert(f->shared->lf);
+
+ /* Set the ring type in the DXPL. In most cases, we will
+ * need H5AC_RING_RDFSM, so initialy set the ring in
+ * the DXPL to that value. We will alter this later if
+ * needed.
+ */
+ if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+ reset_ring = TRUE;
+ curr_ring = H5AC_RING_RDFSM;
+
+ /* Determine start/end points for loop */
+ if(H5F_PAGED_AGGR(f)) {
+ start_type = H5F_MEM_PAGE_META;
+ end_type = H5F_MEM_PAGE_NTYPES;
+ } /* end if */
+ else {
+ start_type = (H5F_mem_page_t)H5FD_MEM_SUPER;
+ end_type = (H5F_mem_page_t)H5FD_MEM_NTYPES;
+ } /* end else */
+
+ for(tt = H5FD_MEM_SUPER; tt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, tt))
+ if(HADDR_UNDEF == (fs_eoa[tt] = H5F_get_eoa(f, tt)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed")
+
+ if(!H5F_PAGED_AGGR(f)) {
+ /* Retrieve metadata aggregator info, if available */
+ if(H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query metadata aggregator stats")
+
+ /* Retrieve 'small data' aggregator info, if available */
+ if(H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sda_addr, &sda_size) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query small data aggregator stats")
+ } /* end if */
+
+ /* Iterate over all the free space types that have managers and get each free list's space */
+ for(type = start_type; type < end_type; H5_INC_ENUM(H5F_mem_page_t, type)) {
+ fs_started[type] = FALSE;
+
+ /* Check if the free space for the file has been initialized */
+ if(!f->shared->fs_man[type] && H5F_addr_defined(f->shared->fs_addr[type])) {
+ if(H5MF_open_fstype(f, dxpl_id, type) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space")
+ HDassert(f->shared->fs_man[type]);
+ fs_started[type] = TRUE;
+ } /* end if */
+
+ /* test to see if we need to switch rings -- do
+ * so if required
+ */
+ if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type))
+ needed_ring = H5AC_RING_MDFSM;
+ else
+ needed_ring = H5AC_RING_RDFSM;
+
+ if(needed_ring != curr_ring) {
+ if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+ curr_ring = needed_ring;
+ } /* end if */
+
+ /* Check if there's free space of this type */
+ if(f->shared->fs_man[type]) {
+ hsize_t type_fs_size = 0; /* Amount of free space managed for each type */
+ hsize_t type_meta_size = 0; /* Amount of free space metadata for each type */
+
+ /* Retrieve free space size from free space manager */
+ if(H5FS_sect_stats(f->shared->fs_man[type], &type_fs_size, NULL) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space stats")
+ if(H5FS_size(f, f->shared->fs_man[type], &type_meta_size) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space metadata stats")
+
+ /* Increment total free space for types */
+ tot_fs_size += type_fs_size;
+ tot_meta_size += type_meta_size;
+ } /* end if */
+ } /* end for */
+
+ /* Close the free-space managers if they were opened earlier in this routine */
+ for(type = start_type; type < end_type; H5_INC_ENUM(H5F_mem_page_t, type)) {
+ /* Test to see if we need to switch rings -- do so if required */
+ if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type))
+ needed_ring = H5AC_RING_MDFSM;
+ else
+ needed_ring = H5AC_RING_RDFSM;
+
+ if(needed_ring != curr_ring) {
+ if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+ curr_ring = needed_ring;
+ } /* end if */
+
+ if(fs_started[type])
+ if(H5MF__close_fstype(f, dxpl_id, type) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't close file free space")
+ } /* end for */
+
+ /* Set the value(s) to return */
+ /* (The metadata & small data aggregators count as free space now, since they aren't at EOA) */
+ if(tot_space)
+ *tot_space = tot_fs_size + ma_size + sda_size;
+ if(meta_size)
+ *meta_size = tot_meta_size;
+
+done:
+ /* Reset the ring in the DXPL */
+ if(reset_ring)
+ if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+ HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+
+ FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
+} /* end H5MF_get_freespace() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF_get_free_sections()
+ *
+ * Purpose: To retrieve free-space section information for
+ * paged or non-paged aggregation
+ *
+ * Return: SUCCEED/FAIL
+ *
+ * Programmer: Vailin Choi; Dec 2012
+ *
+ *-------------------------------------------------------------------------
+ */
+ssize_t
+H5MF_get_free_sections(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, size_t nsects, H5F_sect_info_t *sect_info)
+{
+ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
+ H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */
+ H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration. */
+ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
+ size_t total_sects = 0; /* Total number of sections */
+ H5MF_sect_iter_ud_t sect_udata; /* User data for callback */
+ H5F_mem_page_t start_type, end_type; /* Memory types to iterate over */
+ H5F_mem_page_t ty; /* Memory type for iteration */
+ hbool_t reset_ring = FALSE; /* Whether the ring was set */
+ ssize_t ret_value = -1; /* Return value */
+
+ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, -1)
+
+ /* check args */
+ HDassert(f);
+ HDassert(f->shared);
+ HDassert(f->shared->lf);
+
+ /* H5MF_tidy_self_referential_fsm_hack() will fail if any self
+ * referential FSM is opened prior to the call to it. Thus call
+ * it here if necessary and if it hasn't been called already.
+ *
+ * The situation is further complicated if a cache image exists
+ * and had not yet been loaded into the metadata cache. In this
+ * case, call H5AC_force_cache_image_load() instead of
+ * H5MF_tidy_self_referential_fsm_hack(). H5AC_force_cache_image_load()
+ * will load the cache image, and then call
+ * H5MF_tidy_self_referential_fsm_hack() to discard the cache image
+ * block.
+ */
+ if(f->shared->first_alloc_dealloc) {
+ if(H5AC_cache_image_pending(f)) {
+ if(H5AC_force_cache_image_load(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "forced cache image load failed")
+ } /* end if */
+ else {
+ if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed")
+ } /* end else */
+ } /* end if */
+
+ if(type == H5FD_MEM_DEFAULT) {
+ start_type = H5F_MEM_PAGE_SUPER;
+ end_type = H5F_MEM_PAGE_NTYPES;
+ } /* end if */
+ else {
+ start_type = end_type = (H5F_mem_page_t)type;
+ if(H5F_PAGED_AGGR(f)) /* set to the corresponding LARGE free-space manager */
+ end_type = (H5F_mem_page_t)(end_type + H5FD_MEM_NTYPES);
+ else
+ H5_INC_ENUM(H5F_mem_page_t, end_type);
+ } /* end else */
+
+ /* Set up user data for section iteration */
+ sect_udata.sects = sect_info;
+ sect_udata.sect_count = nsects;
+ sect_udata.sect_idx = 0;
+
+ /* Set the ring type in the DXPL. In most cases, we will
+ * need H5AC_RING_RDFSM, so initialy set the ring in
+ * the DXPL to that value. We will alter this later if
+ * needed.
+ */
+ if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value(0)")
+ reset_ring = TRUE;
+ curr_ring = H5AC_RING_RDFSM;
+
+ /* Iterate over memory types, retrieving the number of sections of each type */
+ for(ty = start_type; ty < end_type; H5_INC_ENUM(H5F_mem_page_t, ty)) {
+ hbool_t fs_started = FALSE; /* The free-space manager is opened or not */
+ size_t nums = 0; /* The number of free-space sections */
+
+ /* Test to see if we need to switch rings -- do so if required */
+ if(H5MF__fsm_type_is_self_referential(f, ty))
+ needed_ring = H5AC_RING_MDFSM;
+ else
+ needed_ring = H5AC_RING_RDFSM;
+
+ if(needed_ring != curr_ring) {
+ if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)")
+ curr_ring = needed_ring;
+ } /* end if */
+
+ if(!f->shared->fs_man[ty] && H5F_addr_defined(f->shared->fs_addr[ty])) {
+ if(H5MF_open_fstype(f, dxpl_id, ty) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't open the free space manager")
+ HDassert(f->shared->fs_man[ty]);
+ fs_started = TRUE;
+ } /* end if */
+
+ /* Check if there's free space sections of this type */
+ if(f->shared->fs_man[ty])
+ if(H5MF__get_free_sects(f, dxpl_id, f->shared->fs_man[ty], &sect_udata, &nums) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get section info for the free space manager")
+
+ /* Increment total # of sections */
+ total_sects += nums;
+
+ /* Close the free space manager of this type, if we started it here */
+ if(fs_started)
+ if(H5MF__close_fstype(f, dxpl_id, ty) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCLOSEOBJ, FAIL, "can't close file free space")
+ if((H5F_PAGED_AGGR(f)) && (type != H5FD_MEM_DEFAULT))
+ ty = (H5F_mem_page_t)(ty + H5FD_MEM_NTYPES - 2);
+ } /* end for */
+
+ /* Set return value */
+ ret_value = (ssize_t)total_sects;
+
+done:
+ /* Reset the ring in the DXPL */
+ if(reset_ring)
+ if(H5AC_reset_ring(dxpl, orig_ring) < 0)
+ HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+
+ FUNC_LEAVE_NOAPI_TAG(ret_value, -1)
+} /* H5MF_get_free_sections() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF_sects_cb()
+ *
+ * Purpose: Iterator callback for each free-space section
+ * Retrieve address and size into user data
+ *
+ * Return: Always succeed
+ *
+ * Programmer: Vailin Choi
+ * July 1st, 2009
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5MF_sects_cb(H5FS_section_info_t *_sect, void *_udata)
+{
+ H5MF_free_section_t *sect = (H5MF_free_section_t *)_sect;
+ H5MF_sect_iter_ud_t *udata = (H5MF_sect_iter_ud_t *)_udata;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ if(udata->sect_idx < udata->sect_count) {
+ udata->sects[udata->sect_idx].addr = sect->sect_info.addr;
+ udata->sects[udata->sect_idx].size = sect->sect_info.size;
+ udata->sect_idx++;
+ } /* end if */
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5MF_sects_cb() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF__get_free_sects
+ *
+ * Purpose: Retrieve section information for the specified free-space manager.
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ * Programmer: Vailin Choi; Dec 2012
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5MF__get_free_sects(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5MF_sect_iter_ud_t *sect_udata, size_t *nums)
+{
+ hsize_t hnums = 0; /* # of sections */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* check args */
+ HDassert(f);
+ HDassert(sect_udata);
+ HDassert(nums);
+ HDassert(fspace);
+
+ /* Query how many sections of this type */
+ if(H5FS_sect_stats(fspace, NULL, &hnums) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space stats")
+ H5_CHECKED_ASSIGN(*nums, size_t, hnums, hsize_t);
+
+ /* Check if we should retrieve the section info */
+ if(sect_udata->sects && *nums > 0)
+ /* Iterate over all the free space sections of this type, adding them to the user's section info */
+ if(H5FS_sect_iterate(f, dxpl_id, fspace, H5MF_sects_cb, sect_udata) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_BADITER, FAIL, "can't iterate over sections")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5MF__get_free_sects() */
+
+
+/*-------------------------------------------------------------------------
* Function: H5MF_settle_raw_data_fsm()
*
* Purpose: Handle any tasks required before the metadata cache
@@ -1253,19 +2662,48 @@ done:
* and any metadata free space managers that reside in the
* raw data free space manager ring.
*
- * Specifically, any metadata managers that DON'T handle
- * space allocation for free space manager header or section
- * info will reside in the raw data free space manager ring.
- * As of this writing, the plan is to move to only two free space
- * managers, one for raw data and one for metadata -- which
- * means that only the raw data free space manager will reside
- * in the free space manager ring. However, this has not been
- * fully implemented yet, so this code must support the
- * possibilty of multiple metadata free space managers, at most
- * two of which handle free space manager header or section info,
- * and thus reside in the metadata free space manager ring.
- *
- * At present, the task list is:
+ * Specifically, this means any metadata managers that DON'T
+ * handle space allocation for free space manager header or
+ * section info will reside in the raw data free space manager
+ * ring.
+ *
+ * In the absence of page allocation, there is at most one
+ * free space manager per memory type defined in H5F_mem_t.
+ * Of these, the one that allocates H5FD_MEM_DRAW will
+ * always reside in the raw data free space manager ring.
+ * If there is more than one metadata free space manager,
+ * all that don't handle H5FD_MEM_FSPACE_HDR or
+ * H5FD_MEM_FSPACE_SINFO (which map to H5FD_MEM_OHDR and
+ * H5FD_MEM_LHEAP respectively) will reside in the raw
+ * data free space manager ring as well
+ *
+ * With page allocation, the situation is conceptually
+ * identical, but more complex in practice.
+ *
+ * In the worst case (multi file driver) page allocation
+ * can result in two free space managers for each memory
+ * type -- one for small (less than on equal to one page)
+ * allocations, and one for large (greater than one page)
+ * allocations.
+ *
+ * In the more common one file case, page allocation will
+ * result in a total of three free space managers -- one for
+ * small (<= one page) raw data allocations, one for small
+ * metadata allocations (i.e, all memory types other than
+ * H5FD_MEM_DRAW), and one for all large (> one page)
+ * allocations.
+ *
+ * Despite these complications, the solution is the same in
+ * the page allocation case -- free space managers (be they
+ * small data or large) are assigned to the raw data free
+ * space manager ring if they don't allocate file space for
+ * free space managers. Note that in the one file case, the
+ * large free space manager must be assigned to the metadata
+ * free space manager ring, as it both allocates pages for
+ * the metadata free space manager, and allocates space for
+ * large (> 1 page) metadata cache entries.
+ *
+ * At present, the task list for this routine is:
*
* 1) Reduce the EOA to the extent possible. To do this:
*
@@ -1320,26 +2758,43 @@ done:
herr_t
H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled)
{
+ int pass_count;
+ hsize_t alloc_size;
+ H5F_mem_t mem_type; /* Memory type for iteration */
+ H5F_mem_page_t fsm_type; /* FSM type for iteration */
+ H5O_fsinfo_t fsinfo; /* Free space manager info message */
+ H5FS_stat_t fs_stat; /* Information for free-space manager */
H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
+ H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */
+ H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration */
hbool_t reset_ring = FALSE; /* Whether the ring was set */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
- /* check args */
+ /* Check args */
HDassert(f);
HDassert(f->shared);
HDassert(fsm_settled);
- /* Only need to settle things if we are persisting the free space info */
- if(f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) {
- H5O_fsinfo_t fsinfo; /* Free space manager info message */
- H5FD_mem_t type; /* Memory type for iteration */
- H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */
- H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration */
- hbool_t fsm_opened[H5FD_MEM_NTYPES]; /* State of FSM */
- hbool_t fsm_visited[H5FD_MEM_NTYPES]; /* State of FSM */
+ /* Only need to settle things if we are persisting the free space info
+ * and allocation/deallocation has occurred.
+ */
+ if(f->shared->fs_persist && !f->shared->first_alloc_dealloc) {
+ hbool_t fsm_opened[H5F_MEM_PAGE_NTYPES]; /* State of FSM */
+ hbool_t fsm_visited[H5F_MEM_PAGE_NTYPES]; /* State of FSM */
+
+ /* Sanity check */
+ HDassert(f->shared->sblock);
+
+ /* should only be called if file is opened R/W */
+ HDassert(H5F_INTENT(f) & H5F_ACC_RDWR);
+
+ /* shouldn't be called unless we have a superblock supporting the
+ * superblock extension.
+ */
+ HDassert(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2);
/* Initialize fsm_opened and fsm_visited */
HDmemset(fsm_opened, 0, sizeof(fsm_opened));
@@ -1356,8 +2811,12 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled)
*
* Note that while the raw data aggregator should not be restarted during
* the close process, this need not be the case for the metadata aggregator.
+ *
+ * Note also that the aggregators will not exist if page aggregation
+ * is enabled -- skip this if so.
*/
- if(H5MF_free_aggrs(f, dxpl_id) < 0)
+ /* Vailin -- is this correct? */
+ if(!H5F_PAGED_AGGR(f) && (H5MF_free_aggrs(f, dxpl_id) < 0))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free aggregators")
/* Set the ring type in the DXPL. In most cases, we will
@@ -1379,7 +2838,7 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled)
* In the case of the raw data free space manager, and any other free
* space manager that does not allocate space for free space managers,
* allocations should be complete at this point, as all raw data should
- * have space allocated and be flushed to file at this point. Thus we
+ * have space allocated and be flushed to file by now. Thus we
* can examine such free space managers and only re-allocate space for
* them if they contain free space. Do this later in this function after
* the EOA has been reduced to the extent possible.
@@ -1390,79 +2849,100 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled)
* are possible), the matter is more ticklish due to the self-
* referential nature of the problem. These FSMs are dealt with in
* H5MF_settle_meta_data_fsm().
+ *
+ * Since paged allocation may be enabled, there may be up to two
+ * free space managers per memory type -- one for small and one for
+ * large allocation. Hence we must loop over the memory types twice
+ * setting the allocation size accordingly if paged allocation is
+ * enabled.
*/
- for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
- H5FS_stat_t fs_stat; /* Information for free-space manager */
- H5FD_mem_t fsm_type; /* File memory type for FSM */
-
- /* There is potentially a many-to-one mapping from memory types to
- * free space managers. Use the fsm_visited[] array to avoid visiting
- * a given FSM more than once. Use fsm_opened[] to track which FSMs
- * must be closed at the end of this function.
- */
- fsm_type = H5MF_ALLOC_TO_FS_TYPE(f, type);
- if(!fsm_visited[fsm_type]) {
- fsm_visited[fsm_type] = TRUE;
-
- /* If there is no active FSM for this type, but such a FSM has
- * space allocated in file, open it so that we can free its file
- * space.
- */
- if(NULL == f->shared->fs_man[fsm_type]) {
- if(H5F_addr_defined(f->shared->fs_addr[fsm_type])) {
- /* Sanity check */
- HDassert(fsm_opened[fsm_type] == FALSE);
-
- /* Start up FSM for the file memory type */
- if(H5MF__alloc_open(f, dxpl_id, fsm_type) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager")
- fsm_opened[fsm_type] = TRUE;
- } /* end if */
+ for(pass_count = 0; pass_count <= 1; pass_count++) {
+ if(pass_count == 0)
+ alloc_size = 1;
+ else if ( H5F_PAGED_AGGR(f) )
+ alloc_size = f->shared->fs_page_size + 1;
+ else /* no need for a second pass */
+ break;
+
+ for(mem_type = H5FD_MEM_SUPER; mem_type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5F_mem_t, mem_type)) {
+ H5MF_alloc_to_fs_type(f, mem_type, alloc_size, &fsm_type);
+
+ if(pass_count == 0) { /* this is the first pass */
+ HDassert(fsm_type > H5F_MEM_PAGE_DEFAULT);
+ HDassert(fsm_type < H5F_MEM_PAGE_LARGE_SUPER);
} /* end if */
+ else if(H5F_PAGED_AGGR(f)) { /* page alloc active */
+ HDassert(fsm_type >= H5F_MEM_PAGE_LARGE_SUPER);
+ HDassert(fsm_type < H5F_MEM_PAGE_NTYPES);
+ } /* end else-if */
+ else /* paged allocation disabled -- should be unreachable */
+ HDassert(FALSE);
+
+ if(!fsm_visited[fsm_type]) {
+ fsm_visited[fsm_type] = TRUE;
+
+ /* If there is no active FSM for this type, but such a FSM has
+ * space allocated in file, open it so that we can free its file
+ * space.
+ */
+ if(NULL == f->shared->fs_man[fsm_type]) {
+ if(H5F_addr_defined(f->shared->fs_addr[fsm_type])) {
+ /* Sanity check */
+ HDassert(fsm_opened[fsm_type] == FALSE);
- /* Check for an actual FSM for this type now */
- /* (Possibly opened in previous step) */
- if(f->shared->fs_man[fsm_type]) {
- /* Test to see if we need to switch rings -- do so if required */
- if((fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
- || (fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
- needed_ring = H5AC_RING_MDFSM;
- else
- needed_ring = H5AC_RING_RDFSM;
- if(needed_ring != curr_ring) {
- if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
- curr_ring = needed_ring;
+ if(H5MF_open_fstype(f, dxpl_id, fsm_type) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager")
+ fsm_opened[fsm_type] = TRUE;
+ } /* end if */
} /* end if */
- /* Query free space manager info for this type */
- if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
+ if(f->shared->fs_man[fsm_type]) {
+ /* Test to see if we need to switch rings -- do so if required */
+ if(H5MF__fsm_type_is_self_referential(f, fsm_type))
+ needed_ring = H5AC_RING_MDFSM;
+ else
+ needed_ring = H5AC_RING_RDFSM;
+
+ if(needed_ring != curr_ring) {
+ if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+ curr_ring = needed_ring;
+ } /* end if */
- /* Check if the free space manager has space in the file */
- if(H5F_addr_defined(fs_stat.addr) || H5F_addr_defined(fs_stat.sect_addr)) {
- /* Delete the free space manager in the file. Will
- * reallocate later if the free space manager contains
- * any free space.
- */
- if(H5FS_free(f, f->shared->fs_man[fsm_type], dxpl_id) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers")
- f->shared->fs_addr[fsm_type] = HADDR_UNDEF;
+ /* Query free space manager info for this type */
+ if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
+
+ /* Check if the free space manager has space in the file */
+ if(H5F_addr_defined(fs_stat.addr) || H5F_addr_defined(fs_stat.sect_addr)) {
+ /* Delete the free space manager in the file. Will
+ * reallocate later if the free space manager contains
+ * any free space.
+ */
+ if(H5FS_free(f, f->shared->fs_man[fsm_type], dxpl_id, TRUE) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers")
+ f->shared->fs_addr[fsm_type] = HADDR_UNDEF;
+ } /* end if */
} /* end if */
- } /* end if */
- /* note that we are tracking opened FSM -- we will close them
- * at the end of the function.
- */
- } /* end if */
+ /* note that we are tracking opened FSM -- we will close them
+ * at the end of the function.
+ */
+ } /* end if */
+ } /* end for */
} /* end for */
+
/* c) Delete the free space manager superblock extension message
* if allocated.
*
* Must do this since the routine that writes / creates superblock
* extension messages will choke if the target message is
* unexpectedly either absent or present.
+ *
+ * Update: This is probably unecessary, as I gather that the
+ * file space manager info message is guaranteed to exist.
+ * Leave it in for now, but consider removing it.
*/
if(H5F_addr_defined(f->shared->sblock->ext_addr))
if(H5F_super_ext_remove_msg(f, dxpl_id, H5O_FSINFO_ID) < 0)
@@ -1486,11 +2966,16 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled)
* those addresses are unknown. This is OK -- we will write the correct
* values to the message at free space manager shutdown.
*/
- for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
- fsinfo.fs_addr[type - 1] = HADDR_UNDEF;
+ for(fsm_type = H5F_MEM_PAGE_SUPER; fsm_type < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, fsm_type))
+ fsinfo.fs_addr[fsm_type - 1] = HADDR_UNDEF;
fsinfo.strategy = f->shared->fs_strategy;
+ fsinfo.persist = f->shared->fs_persist;
fsinfo.threshold = f->shared->fs_threshold;
- if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE, H5O_MSG_NO_FLAGS_SET) < 0)
+ fsinfo.page_size = f->shared->fs_page_size;
+ fsinfo.pgend_meta_thres = f->shared->pgend_meta_thres;
+ fsinfo.eoa_pre_fsm_fsalloc = HADDR_UNDEF;
+
+ if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing fsinfo message to superblock extension")
@@ -1514,102 +2999,117 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled)
*/
/* Reinitialize fsm_visited */
- HDmemset(fsm_visited, 0, sizeof(fsm_visited));
-
- for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
- H5FD_mem_t fsm_type; /* File memory type for FSM */
-
- fsm_type = H5MF_ALLOC_TO_FS_TYPE(f, type);
-
- /* test to see if we need to switch rings -- do so if required */
- if((fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
- || (fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
- needed_ring = H5AC_RING_MDFSM;
- else
- needed_ring = H5AC_RING_RDFSM;
-
- if(needed_ring != curr_ring) {
- if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring)< 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
- curr_ring = needed_ring;
- } /* end if */
-
- /* Since there can be a many-to-one mapping from memory types
- * to free space managers, ensure that we don't visit any FSM
- * more than once.
- */
- if(!fsm_visited[fsm_type]) {
- fsm_visited[fsm_type] = TRUE;
-
- if(f->shared->fs_man[fsm_type]) {
- /* Only allocate file space if the target free space manager
- * doesn't allocate file space for free space managers. Note
- * that this is also the deciding factor as to whether a FSM
- * in in the raw data FSM ring.
- */
- if((fsm_type != H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
- && (fsm_type != H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) {
- H5FS_stat_t fs_stat; /* Information for free-space manager */
+ for(fsm_type = H5F_MEM_PAGE_SUPER; fsm_type < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, fsm_type))
+ fsm_visited[fsm_type] = FALSE;
+
+ for(pass_count = 0; pass_count <= 1; pass_count++) {
+ if(pass_count == 0)
+ alloc_size = 1;
+ else if(H5F_PAGED_AGGR(f))
+ alloc_size = f->shared->fs_page_size + 1;
+ else /* no need for a second pass */
+ break;
+
+ for(mem_type = H5FD_MEM_SUPER; mem_type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5F_mem_t, mem_type)) {
+ H5MF_alloc_to_fs_type(f, mem_type, alloc_size, &fsm_type);
+
+ if(pass_count == 0) { /* this is the first pass */
+ HDassert(fsm_type > H5F_MEM_PAGE_DEFAULT);
+ HDassert(fsm_type < H5F_MEM_PAGE_LARGE_SUPER);
+ } /* end if */
+ else if(H5F_PAGED_AGGR(f)) { /* page alloc active */
+ HDassert(fsm_type >= H5F_MEM_PAGE_LARGE_SUPER);
+ HDassert(fsm_type < H5F_MEM_PAGE_NTYPES);
+ } /* end else-if */
+ else /* paged allocation disabled -- should be unreachable */
+ HDassert(FALSE);
- /* The current ring should be H5AC_RING_RDFSM */
- HDassert(curr_ring == H5AC_RING_RDFSM);
+ /* test to see if we need to switch rings -- do so if required */
+ if(H5MF__fsm_type_is_self_referential(f, fsm_type))
+ needed_ring = H5AC_RING_MDFSM;
+ else
+ needed_ring = H5AC_RING_RDFSM;
- /* Query free space manager info for this type */
- if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0 )
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info")
+ if(needed_ring != curr_ring) {
+ if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring)< 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
+ curr_ring = needed_ring;
+ } /* end if */
- /* If the free space manager contains section info,
- * allocate space for the header and sinfo (note that
- * space must not be allocated at present -- verify
- * verify this with assertions).
+ /* Since there can be a many-to-one mapping from memory types
+ * to free space managers, ensure that we don't visit any FSM
+ * more than once.
+ */
+ if(!fsm_visited[fsm_type]) {
+ fsm_visited[fsm_type] = TRUE;
+
+ if(f->shared->fs_man[fsm_type]) {
+ /* Only allocate file space if the target free space manager
+ * doesn't allocate file space for free space managers. Note
+ * that this is also the deciding factor as to whether a FSM
+ * in in the raw data FSM ring.
*/
- if(fs_stat.serial_sect_count > 0) {
- /* Sanity check */
- HDassert(!H5F_addr_defined(fs_stat.addr));
-
- /* Allocate FSM header */
- if(H5FS_alloc_hdr(f, f->shared->fs_man[fsm_type], &f->shared->fs_addr[fsm_type], dxpl_id) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocated free-space header")
-
- /* Allocate FSM section info */
- HDassert(!H5F_addr_defined(fs_stat.sect_addr));
- HDassert(fs_stat.alloc_sect_size == 0);
- if(H5FS_alloc_sect(f, f->shared->fs_man[fsm_type], dxpl_id) < 0 )
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate free-space section info")
+ if(!H5MF__fsm_type_is_self_referential(f, fsm_type)) {
+ /* The current ring should be H5AC_RING_RDFSM */
+ HDassert(curr_ring == H5AC_RING_RDFSM);
+
+ /* Query free space manager info for this type */
+ if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0 )
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info")
+
+ /* If the free space manager contains section info,
+ * allocate space for the header and sinfo (note that
+ * space must not be allocated at present -- verify
+ * verify this with assertions).
+ */
+ if(fs_stat.serial_sect_count > 0) {
+ /* Sanity check */
+ HDassert(!H5F_addr_defined(fs_stat.addr));
+
+ /* Allocate FSM header */
+ if(H5FS_alloc_hdr(f, f->shared->fs_man[fsm_type], &f->shared->fs_addr[fsm_type], dxpl_id) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocated free-space header")
+
+ /* Allocate FSM section info */
+ HDassert(!H5F_addr_defined(fs_stat.sect_addr));
+ HDassert(fs_stat.alloc_sect_size == 0);
+ if(H5FS_alloc_sect(f, f->shared->fs_man[fsm_type], dxpl_id) < 0 )
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate free-space section info")
#ifndef NDEBUG
- /* Re-Query free space manager info for this type */
- if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
-
- HDassert(H5F_addr_defined(fs_stat.addr));
- HDassert(H5F_addr_defined(fs_stat.sect_addr));
- HDassert(fs_stat.serial_sect_count > 0);
- HDassert(fs_stat.alloc_sect_size > 0);
- HDassert(fs_stat.alloc_sect_size == fs_stat.sect_size);
+ /* Re-Query free space manager info for this type */
+ if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
+
+ HDassert(H5F_addr_defined(fs_stat.addr));
+ HDassert(H5F_addr_defined(fs_stat.sect_addr));
+ HDassert(fs_stat.serial_sect_count > 0);
+ HDassert(fs_stat.alloc_sect_size > 0);
+ HDassert(fs_stat.alloc_sect_size == fs_stat.sect_size);
#endif /* NDEBUG */
+ } /* end if */
+ else {
+ HDassert(!H5F_addr_defined(fs_stat.addr));
+ HDassert(!H5F_addr_defined(fs_stat.sect_addr));
+ HDassert(fs_stat.serial_sect_count == 0);
+ HDassert(fs_stat.alloc_sect_size == 0);
+ } /* end else */
} /* end if */
- else {
- HDassert(!H5F_addr_defined(fs_stat.addr));
- HDassert(!H5F_addr_defined(fs_stat.sect_addr));
- HDassert(fs_stat.serial_sect_count == 0);
- HDassert(fs_stat.alloc_sect_size == 0);
- } /* end else */
} /* end if */
- } /* end if */
- /* Close any opened FSMs */
- if(fsm_opened[fsm_type]) {
- if(H5MF__alloc_close(f, dxpl_id, fsm_type) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't close file free space manager")
- fsm_opened[fsm_type] = FALSE;
+ /* Close any opened FSMs */
+ if(fsm_opened[fsm_type]) {
+ if(H5MF__close_fstype(f, dxpl_id, fsm_type) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't close file free space manager")
+ fsm_opened[fsm_type] = FALSE;
+ } /* end if */
} /* end if */
- } /* end if */
+ } /* end for */
} /* end for */
/* verify that all opened FSMs were closed */
- for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
- HDassert(!fsm_opened[type]);
+ for(fsm_type = H5F_MEM_PAGE_SUPER; fsm_type < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, fsm_type))
+ HDassert(!fsm_opened[fsm_type]);
/* Indicate that the FSM was settled successfully */
*fsm_settled = TRUE;
@@ -1630,11 +3130,11 @@ done:
*
* Purpose: If the free space manager is persistent, handle any tasks
* required before the metadata cache can serialize or flush
- * the metadata free space manager(sI) that handle file space
+ * the metadata free space manager(s) that handle file space
* allocation for free space managers.
*
* In most cases, there will be only one manager assigned
- * to this role. However, since for reason or reason unknown,
+ * to this role. However, since for reasons unknown,
* free space manager headers and section info blocks are
* different classes of memory, it is possible that two free
* space managers will be involved.
@@ -1642,7 +3142,7 @@ done:
* On entry to this function, the raw data settle routine
* (H5MF_settle_raw_data_fsm()) should have:
*
- * 1) Freed the accumulators.
+ * 1) Freed the aggregators.
*
* 2) Freed all file space allocated to the free space managers.
*
@@ -1678,9 +3178,12 @@ done:
* 1) Verify that the free space manager(s) involved in file
* space allocation for free space managers are still floating.
*
- * 2) Free the accumulators.
+ * 2) Free the aggregators.
*
- * 3) Reduce the EOA to the extent possible.
+ * 3) Reduce the EOA to the extent possible, and make note
+ * of the resulting value. This value will be stored
+ * in the fsinfo superblock extension message and be used
+ * in the subsequent file open.
*
* 4) Re-allocate space for any free space manager(s) that:
*
@@ -1714,6 +3217,9 @@ done:
* severe time pressure at the moment, the above brute
* force solution is attractive.
*
+ * 5) Make note of the EOA -- used for sanity checking on
+ * FSM shutdown.
+ *
* Return: SUCCEED/FAIL
*
* Programmer: John Mainzer
@@ -1724,10 +3230,23 @@ done:
herr_t
H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled)
{
+ H5F_mem_page_t sm_fshdr_fs_type; /* small fs hdr fsm */
+ H5F_mem_page_t sm_fssinfo_fs_type; /* small fs sinfo fsm */
+ H5F_mem_page_t lg_fshdr_fs_type; /* large fs hdr fsm */
+ H5F_mem_page_t lg_fssinfo_fs_type; /* large fs sinfo fsm */
+ H5FS_t *sm_hdr_fspace = NULL; /* ptr to sm FSM hdr alloc FSM */
+ H5FS_t *sm_sinfo_fspace = NULL; /* ptr to sm FSM sinfo alloc FSM */
+ H5FS_t *lg_hdr_fspace = NULL; /* ptr to lg FSM hdr alloc FSM */
+ H5FS_t *lg_sinfo_fspace = NULL; /* ptr to lg FSM sinfo alloc FSM */
+ haddr_t eoa_pre_fsm_fsalloc; /* eoa pre file space allocation */
+ /* for self referential FSMs */
+ haddr_t eoa_post_fsm_fsalloc; /* eoa post file space allocation */
+ /* for self referential FSMs */
+ H5FS_stat_t fs_stat; /* Information for hdr FSM */
H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
hbool_t reset_ring = FALSE; /* Whether we set the ring */
- herr_t ret_value = SUCCEED; /* Return value */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
@@ -1736,29 +3255,49 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled)
HDassert(f->shared);
HDassert(fsm_settled);
- /* Only need to settle things if we are persisting the free space info */
- if(f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) {
- H5FS_t *hdr_fspace; /* Ptr to FSM hdr alloc FSM */
- H5FS_t *sinfo_fspace; /* Ptr to FSM sinfo alloc FSM */
- H5FS_stat_t fs_stat; /* Information for FSM */
- H5FD_mem_t hdr_fsm_alloc_type; /* FSM hdr alloc type */
- H5FD_mem_t sinfo_fsm_alloc_type; /* FSM info alloc type */
-
- /* Get the file memory types for the FSM header & section info */
- hdr_fsm_alloc_type = H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR);
- sinfo_fsm_alloc_type = H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO);
-
- /* Sanity checks */
- HDassert(hdr_fsm_alloc_type > H5FD_MEM_NOLIST);
- HDassert(hdr_fsm_alloc_type < H5FD_MEM_NTYPES);
- HDassert(sinfo_fsm_alloc_type > H5FD_MEM_NOLIST);
- HDassert(sinfo_fsm_alloc_type < H5FD_MEM_NTYPES);
- HDassert(!H5F_addr_defined(f->shared->fs_addr[hdr_fsm_alloc_type]));
- HDassert(!H5F_addr_defined(f->shared->fs_addr[sinfo_fsm_alloc_type]));
-
- /* Note that in most cases, hdr_fspace will equal sinfo_fspace. */
- hdr_fspace = f->shared->fs_man[hdr_fsm_alloc_type];
- sinfo_fspace = f->shared->fs_man[sinfo_fsm_alloc_type];
+ /* Only need to settle things if we are persisting the free space info
+ * and allocation/deallocation has occurred.
+ */
+ if(f->shared->fs_persist && !f->shared->first_alloc_dealloc) {
+ /* Sanity check */
+ HDassert(f->shared->lf);
+
+ /* should only be called if file is opened R/W */
+ HDassert(H5F_INTENT(f) & H5F_ACC_RDWR);
+
+ H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fs_type);
+ H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fs_type);
+
+ HDassert(sm_fshdr_fs_type > H5F_MEM_PAGE_DEFAULT);
+ HDassert(sm_fshdr_fs_type < H5F_MEM_PAGE_LARGE_SUPER);
+
+ HDassert(sm_fssinfo_fs_type > H5F_MEM_PAGE_DEFAULT);
+ HDassert(sm_fssinfo_fs_type < H5F_MEM_PAGE_LARGE_SUPER);
+
+ HDassert(!H5F_addr_defined(f->shared->fs_addr[sm_fshdr_fs_type]));
+ HDassert(!H5F_addr_defined(f->shared->fs_addr[sm_fssinfo_fs_type]));
+
+ /* Note that in most cases, sm_hdr_fspace will equal sm_sinfo_fspace. */
+ sm_hdr_fspace = f->shared->fs_man[sm_fshdr_fs_type];
+ sm_sinfo_fspace = f->shared->fs_man[sm_fssinfo_fs_type];
+
+ if(H5F_PAGED_AGGR(f)) {
+ H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fs_type);
+ H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fs_type);
+
+ HDassert(lg_fshdr_fs_type >= H5F_MEM_PAGE_LARGE_SUPER);
+ HDassert(lg_fshdr_fs_type < H5F_MEM_PAGE_NTYPES);
+
+ HDassert(lg_fssinfo_fs_type >= H5F_MEM_PAGE_LARGE_SUPER);
+ HDassert(lg_fssinfo_fs_type < H5F_MEM_PAGE_NTYPES);
+
+ HDassert(!H5F_addr_defined(f->shared->fs_addr[lg_fshdr_fs_type]));
+ HDassert(!H5F_addr_defined(f->shared->fs_addr[lg_fssinfo_fs_type]));
+
+ /* Note that in most cases, lg_hdr_fspace will equal lg_sinfo_fspace. */
+ lg_hdr_fspace = f->shared->fs_man[lg_fshdr_fs_type];
+ lg_sinfo_fspace = f->shared->fs_man[lg_fssinfo_fs_type];
+ } /* end if */
/* Set the ring in the dxpl appropriately for subsequent calls */
if(H5AC_set_ring(dxpl_id, H5AC_RING_MDFSM, &dxpl, &orig_ring) < 0)
@@ -1766,41 +3305,109 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled)
reset_ring = TRUE;
#ifndef NDEBUG
- /* Verify that hdr_fspace is floating if it exists */
- if(hdr_fspace) {
+ /* Verify that sm_hdr_fspace is floating if it exists */
+ if(sm_hdr_fspace) {
/* Query free space manager info for this type */
- if(H5FS_stat_info(f, hdr_fspace, &fs_stat) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info")
+ if(H5FS_stat_info(f, sm_hdr_fspace, &fs_stat) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info")
HDassert(!H5F_addr_defined(fs_stat.addr));
HDassert(!H5F_addr_defined(fs_stat.sect_addr));
HDassert(fs_stat.alloc_sect_size == 0);
} /* end if */
- /* Verify that sinfo_fspace is floating if it exists */
- if((sinfo_fspace) && (hdr_fspace != sinfo_fspace)) {
+ /* Verify that sm_sinfo_fspace is floating if it exists and is distinct */
+ if((sm_sinfo_fspace) && (sm_hdr_fspace != sm_sinfo_fspace)) {
/* Query free space manager info for this type */
- if(H5FS_stat_info(f, sinfo_fspace, &fs_stat) < 0)
+ if(H5FS_stat_info(f, sm_sinfo_fspace, &fs_stat) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info")
HDassert(!H5F_addr_defined(fs_stat.addr));
HDassert(!H5F_addr_defined(fs_stat.sect_addr));
HDassert(fs_stat.alloc_sect_size == 0);
} /* end if */
+
+ if(H5F_PAGED_AGGR(f)) {
+ /* Verify that lg_hdr_fspace is floating if it exists */
+ if(lg_hdr_fspace) {
+ /* Query free space manager info for this type */
+ if(H5FS_stat_info(f, lg_hdr_fspace, &fs_stat) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info (3)")
+
+ HDassert(!H5F_addr_defined(fs_stat.addr));
+ HDassert(!H5F_addr_defined(fs_stat.sect_addr));
+ HDassert(fs_stat.alloc_sect_size == 0);
+ } /* end if */
+
+ /* Verify that lg_sinfo_fspace is floating if it
+ * exists and is distinct
+ */
+ if((lg_sinfo_fspace) && (lg_hdr_fspace != lg_sinfo_fspace)) {
+ /* Query free space manager info for this type */
+ if(H5FS_stat_info(f, lg_sinfo_fspace, &fs_stat) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info (4)")
+
+ HDassert(!H5F_addr_defined(fs_stat.addr));
+ HDassert(!H5F_addr_defined(fs_stat.sect_addr));
+ HDassert(fs_stat.alloc_sect_size == 0);
+ } /* end if */
+ } /* end if */
#endif /* NDEBUG */
/* Free the space in the metadata aggregator. Do this via the
* H5MF_free_aggrs() call. Note that the raw data aggregator must
* have already been freed. Sanity checks for this?
+ *
+ * Note that the aggregators will not exist if paged aggregation
+ * is enabled -- don't attempt to free if this is the case.
*/
+ /* Vailin -- is this correct? */
/* (for space not at EOF, it may be put into free space managers) */
- if(H5MF_free_aggrs(f, dxpl_id) < 0)
+ if((!H5F_PAGED_AGGR(f)) && (H5MF_free_aggrs(f, dxpl_id) < 0))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free aggregators")
/* Trying shrinking the EOA for the file */
if(H5MF__close_shrink_eoa(f, dxpl_id) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
+ /* At this point, the EOA should be set to a value that contains
+ * the allocation for all user data, all non self referential FSMs,
+ * the superblock and all superblock extension messages.
+ *
+ * Make note of the current EOA. We will store this value in the
+ * free space manager superblock extension message. Since space for
+ * everything other than the self referential FSMs (and possibly the
+ * cache image) has been allocated at this point, this allows us to
+ * to float the self referential FSMs on the first file space allocation /
+ * deallocaiton and then set the EOA to this value before we handle
+ * the allocation / deallocation. (If a cache image exists, the
+ * first allocation / deallocation will be the deallocation of space
+ * for the cache image).
+ *
+ * WARNING: This approach settling the self referential free space
+ * managers and allocating space for them in the file will
+ * not work as currently implemented with the split and
+ * multi file drivers, as the self referential free space
+ * manager header and section info can be stored in up to
+ * two different files -- requiring that up to two EOA's
+ * be stored in the the free space managers super block
+ * extension message.
+ *
+ * As of this writing, we are solving this problem by
+ * simply not supporting persistant FSMs with the split
+ * and multi file drivers.
+ *
+ * Current plans are to do away with the multi file
+ * driver, so this should be a non-issue in this case.
+ *
+ * We should be able to support the split file driver
+ * without a file format change. However, the code to
+ * do so does not exist at present.
+ */
+ if(HADDR_UNDEF == (eoa_pre_fsm_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)))
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get EOA")
+
+
/* ******************* PROBLEM: ********************
*
* If the file has an alignement other than 1, and if
@@ -1849,15 +3456,39 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled)
* This is isn't good, but due to schedule pressure, we will just drop
* the fragment on the floor for now.
*/
- if(hdr_fspace)
- if(H5FS_alloc_vfd_alloc_hdr_and_section_info(f, dxpl_id, hdr_fspace,
- &(f->shared->fs_addr[hdr_fsm_alloc_type])) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate hdr FSM file space")
+ if(sm_hdr_fspace)
+ if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, dxpl_id, sm_hdr_fspace, &(f->shared->fs_addr[sm_fshdr_fs_type])) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sm hdr FSM file space")
+
+ if(sm_sinfo_fspace && (sm_sinfo_fspace != sm_hdr_fspace))
+ if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, dxpl_id, sm_sinfo_fspace, &(f->shared->fs_addr[sm_fssinfo_fs_type])) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sm sinfo FSM file space")
+
+ if(H5F_PAGED_AGGR(f)) {
+ if(lg_hdr_fspace)
+ if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, dxpl_id, lg_hdr_fspace, &(f->shared->fs_addr[lg_fshdr_fs_type])) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg hdr FSM file space")
+
+ if(lg_sinfo_fspace && (lg_sinfo_fspace != lg_hdr_fspace))
+ if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, dxpl_id, lg_sinfo_fspace, &(f->shared->fs_addr[lg_fssinfo_fs_type])) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg sinfo FSM file space")
+ } /* end if */
- if(sinfo_fspace && (sinfo_fspace != hdr_fspace))
- if(H5FS_alloc_vfd_alloc_hdr_and_section_info(f, dxpl_id, sinfo_fspace,
- &(f->shared->fs_addr[sinfo_fsm_alloc_type])) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sinfo FSM file space")
+ /* Get the eoa after allocation of file space for the self referential
+ * free space managers. Assuming no cache image, this should be the
+ * final EOA of the file.
+ */
+ if(HADDR_UNDEF == (eoa_post_fsm_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)))
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file size")
+
+ /* All free space managers should have file space allocated for them
+ * now, and should see no further allocations / deallocations. Store
+ * the pre and post file space allocaton for self referential FSMs EOA
+ * for use when we actually write the free space manager superblock
+ * extension message.
+ */
+ f->shared->eoa_pre_fsm_fsalloc = eoa_pre_fsm_fsalloc;
+ f->shared->eoa_post_fsm_fsalloc = eoa_post_fsm_fsalloc;
/* Indicate that the FSM was settled successfully */
*fsm_settled = TRUE;
@@ -1868,479 +3499,410 @@ done:
if(H5AC_reset_ring(dxpl, orig_ring) < 0)
HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
- FUNC_LEAVE_NOAPI(ret_value)
+ FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
} /* H5MF_settle_meta_data_fsm() */
/*-------------------------------------------------------------------------
- * Function: H5MF__close_delete
+ * Function: H5MF__fsm_type_is_self_referential()
*
- * Purpose: Common code for closing and deleting freespace managers from
- * the file.
+ * Purpose: Return TRUE if the indicated free space manager allocates
+ * file space for free space managers. Return FALSE otherwise.
*
- * Return: SUCCEED/FAIL
+ * Return: TRUE/FALSE
*
- * Programmer: Vailin Choi
- * Jan 2016
+ * Programmer: John Mainzer
+ * 12/6/16
*
*-------------------------------------------------------------------------
*/
-static herr_t
-H5MF__close_delete(H5F_t *f, hid_t dxpl_id, H5P_genplist_t **dxpl)
+hbool_t
+H5MF__fsm_type_is_self_referential(H5F_t *f, H5F_mem_page_t fsm_type)
{
- H5FD_mem_t type; /* Memory type for iteration */
- H5AC_ring_t curr_ring = H5AC_RING_INV; /* current ring value */
- herr_t ret_value = SUCCEED; /* Return value */
+ H5F_mem_page_t sm_fshdr_fsm;
+ H5F_mem_page_t sm_fssinfo_fsm;
+ H5F_mem_page_t lg_fshdr_fsm;
+ H5F_mem_page_t lg_fssinfo_fsm;
+ hbool_t result = FALSE;
- FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
-#ifdef H5MF_ALLOC_DEBUG
-HDfprintf(stderr, "%s: Entering\n", FUNC);
-#endif /* H5MF_ALLOC_DEBUG */
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
- /* check args */
+ /* Sanity check */
HDassert(f);
HDassert(f->shared);
+ HDassert(fsm_type >= H5F_MEM_PAGE_DEFAULT);
+ HDassert(fsm_type < H5F_MEM_PAGE_NTYPES);
- /* Iterate over all the free space types that have managers and get each free list's space */
- for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
- H5AC_ring_t needed_ring; /* Ring value needed for this iteration */
+ H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fsm);
+ H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fsm);
-#ifdef H5MF_ALLOC_DEBUG_MORE
-HDfprintf(stderr, "%s: Check 1.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]);
-#endif /* H5MF_ALLOC_DEBUG_MORE */
+ if(H5F_PAGED_AGGR(f)) {
+ H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fsm);
+ H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fsm);
- /* test to see if we need to switch rings -- do so if required */
- if((H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
- || (H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
- needed_ring = H5AC_RING_MDFSM;
+ result = (fsm_type == sm_fshdr_fsm) || (fsm_type == sm_fssinfo_fsm)
+ || (fsm_type == lg_fshdr_fsm) || (fsm_type == lg_fssinfo_fsm);
+ } /* end if */
+ else {
+ /* In principle, fsm_type should always be less than
+ * H5F_MEM_PAGE_LARGE_SUPER whenever paged aggregation
+ * is not enabled. However, since there is code that does
+ * not observe this prinicple, force the result to FALSE if
+ * fsm_type is greater than or equal to H5F_MEM_PAGE_LARGE_SUPER.
+ */
+ if(fsm_type >= H5F_MEM_PAGE_LARGE_SUPER)
+ result = FALSE;
else
- needed_ring = H5AC_RING_RDFSM;
-
- if(needed_ring != curr_ring) {
- if(H5AC_set_ring(dxpl_id, needed_ring, dxpl, &curr_ring) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (4)")
- curr_ring = needed_ring;
- } /* end if */
-
- /* If the free space manager for this type is open, close it */
- if(f->shared->fs_man[type]) {
-#ifdef H5MF_ALLOC_DEBUG_MORE
-HDfprintf(stderr, "%s: Before closing free space manager\n", FUNC);
-#endif /* H5MF_ALLOC_DEBUG_MORE */
- if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free space info")
- f->shared->fs_man[type] = NULL;
- f->shared->fs_state[type] = H5F_FS_STATE_CLOSED;
- } /* end if */
-#ifdef H5MF_ALLOC_DEBUG_MORE
-HDfprintf(stderr, "%s: Check 2.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]);
-#endif /* H5MF_ALLOC_DEBUG_MORE */
-
- /* If there is free space manager info for this type, delete it */
- if(H5F_addr_defined(f->shared->fs_addr[type])) {
- haddr_t tmp_fs_addr; /* Temporary holder for free space manager address */
-
- /* Put address into temporary variable and reset it */
- /* (Avoids loopback in file space freeing routine) */
- tmp_fs_addr = f->shared->fs_addr[type];
- f->shared->fs_addr[type] = HADDR_UNDEF;
-
- /* Shift to "deleting" state, to make certain we don't track any
- * file space freed as a result of deleting the free space manager.
- */
- f->shared->fs_state[type] = H5F_FS_STATE_DELETING;
-
-#ifdef H5MF_ALLOC_DEBUG_MORE
-HDfprintf(stderr, "%s: Before deleting free space manager\n", FUNC);
-#endif /* H5MF_ALLOC_DEBUG_MORE */
-
- /* Delete free space manager for this type */
- if(H5FS_delete(f, dxpl_id, tmp_fs_addr) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't delete free space manager")
-
- /* Shift [back] to closed state */
- HDassert(f->shared->fs_state[type] == H5F_FS_STATE_DELETING);
- f->shared->fs_state[type] = H5F_FS_STATE_CLOSED;
-
- /* Sanity check that the free space manager for this type wasn't started up again */
- HDassert(!H5F_addr_defined(f->shared->fs_addr[type]));
- } /* end if */
- } /* end for */
+ result = (fsm_type == sm_fshdr_fsm) || (fsm_type == sm_fssinfo_fsm);
+ } /* end else */
-done:
-#ifdef H5MF_ALLOC_DEBUG
-HDfprintf(stderr, "%s: Leaving\n", FUNC);
-#endif /* H5MF_ALLOC_DEBUG */
- FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
-} /* H5MF__close_delete() */
+ FUNC_LEAVE_NOAPI(result)
+} /* H5MF__fsm_type_is_self_referential() */
/*-------------------------------------------------------------------------
- * Function: H5MF_try_close
+ * Function: H5MF__fsm_is_self_referential()
*
- * Purpose: This is called by H5Fformat_convert() to close and delete
- * free-space managers when downgrading persistent free-space
- * to non-persistent.
+ * Purpose: Return TRUE if the indicated free space manager allocates
+ * file space for free space managers. Return FALSE otherwise.
*
- * Return: SUCCEED/FAIL
+ * Return: TRUE/FALSE
*
- * Programmer: Vailin Choi
- * Jan 2016
+ * Programmer: John Mainzer
+ * 12/6/16
*
*-------------------------------------------------------------------------
*/
-herr_t
-H5MF_try_close(H5F_t *f, hid_t dxpl_id)
+static hbool_t
+H5MF__fsm_is_self_referential(H5F_t *f, H5FS_t *fspace)
{
- H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
- H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
- hbool_t reset_ring = FALSE; /* Whether the ring was set */
- herr_t ret_value = SUCCEED; /* Return value */
+ H5F_mem_page_t sm_fshdr_fsm;
+ H5F_mem_page_t sm_fssinfo_fsm;
+ hbool_t result = FALSE;
- FUNC_ENTER_NOAPI(FAIL)
-#ifdef H5MF_ALLOC_DEBUG
-HDfprintf(stderr, "%s: Entering\n", FUNC);
-#endif /* H5MF_ALLOC_DEBUG */
+ FUNC_ENTER_STATIC_NOERR
- /* check args */
+ /* Sanity check */
HDassert(f);
+ HDassert(f->shared);
+ HDassert(fspace);
- /* Set the ring type in the DXPL */
- if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
- reset_ring = TRUE;
+ H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fsm);
+ H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fsm);
- /* Close and delete freespace managers from the file */
- if(H5MF__close_delete(f, dxpl_id, &dxpl) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to close delete free-space managers")
+ if(H5F_PAGED_AGGR(f)) {
+ H5F_mem_page_t lg_fshdr_fsm;
+ H5F_mem_page_t lg_fssinfo_fsm;
-done:
- /* Reset the ring in the DXPL */
- if(reset_ring)
- if(H5AC_reset_ring(dxpl, orig_ring) < 0)
- HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+ H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fsm);
+ H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fsm);
-#ifdef H5MF_ALLOC_DEBUG
-HDfprintf(stderr, "%s: Leaving\n", FUNC);
-#endif /* H5MF_ALLOC_DEBUG */
- FUNC_LEAVE_NOAPI(ret_value)
-} /* H5MF_try_close() */
+ result = (fspace == f->shared->fs_man[sm_fshdr_fsm]) ||
+ (fspace == f->shared->fs_man[sm_fssinfo_fsm]) ||
+ (fspace == f->shared->fs_man[lg_fshdr_fsm]) ||
+ (fspace == f->shared->fs_man[lg_fssinfo_fsm]);
+ } /* end if */
+ else
+ result = (fspace == f->shared->fs_man[sm_fshdr_fsm]) ||
+ (fspace == f->shared->fs_man[sm_fssinfo_fsm]);
+
+ FUNC_LEAVE_NOAPI(result)
+} /* H5MF__fsm_is_self_referential() */
/*-------------------------------------------------------------------------
- * Function: H5MF_close
- *
- * Purpose: Close the free space tracker(s) for a file
- *
- * Return: SUCCEED/FAIL
+ * Function: H5MF_tidy_self_referential_fsm_hack
+ *
+ * Purpose: As discussed in the comments of the settle routines above,
+ * the existence of self referential free space managers
+ * as currently implemented creates the possibility of
+ * infinite loops at file close.
+ *
+ * As a hack to avoid this, we have added code to settle
+ * self referential free space managers, and then allocate
+ * space for them directly from the file driver.
+ *
+ * To avoid dropping ever increasing amounts of file space
+ * on the floor with each subsequent file close/open cycle,
+ * we need to clean this up on file open. To avoid this,
+ * this function is called on the first file space allocation
+ * or deallocation after file open to float the self referential
+ * free space managers and reduce the EOA to the value it
+ * had before the direct allocation of space for the self
+ * referential free space managers.
+ *
+ * The function proceeds as follows:
+ *
+ * 1) Verify that f->shared->first_alloc_dealloc is TRUE,
+ * and then set it to FALSE.
+ *
+ * 2) Get the current EOA. Verify that it is greater than
+ * or equal to f->shared->eoa_pre_fsm_fsalloc. If the
+ * current eoa is equal to f->shared->eoa_pre_fsm_fsalloc,
+ * no self referential FSMs were stored, and we are done.
+ *
+ * NOTE: This will have to be reworked somewhat for
+ * cache image.
+ *
+ * 3) Load the self referential FSMs. In passing verify that
+ * the lowest address of a FSM header is equal to
+ * f->shared->eoa_pre_fsm_fsalloc.'
+ *
+ * Note that we don't have to use any special I/O for
+ * this -- we can use the regular I/O methods even if
+ * paged aggregation and page buffering is enabled.
+ *
+ * 4) Float the FSMs. Ensure that the file space is NOT
+ * released.
+ *
+ * 5) Set EOA equal to f->shared->eoa_pre_fsm_fsalloc,
+ * and then set f->shared->eoa_pre_fsm_fsalloc to
+ * HADDR_UNDEF.
+ *
+ * If page buffering, verify that the new EOA is
+ * on a page boundary, and expunge any pages in the
+ * page buffer after the new EOA.
+ *
+ * Note that this function is also called from test code
+ * when it is necessary to startup a self referential
+ * free space manager prior to the first file space
+ * allocation / deallocation. Failure to do so will
+ * result in assertion failures in this function on
+ * the first file space allocation / deallocation.
+ *
+ * Return: SUCCEED/FAIL
*
- * Programmer: Quincey Koziol
- * Tuesday, January 22, 2008
+ * Programmer: John Mainzer
+ * 12/11/16
*
- * Modifications:
- * Vailin Choi; July 2012
- * As the default free-list mapping is changed to H5FD_FLMAP_DICHOTOMY,
- * modifications are needed to shrink EOA if the last section of each free-space manager
- * and the remaining space in the two aggregators are at EOA.
-
*-------------------------------------------------------------------------
*/
herr_t
-H5MF_close(H5F_t *f, hid_t dxpl_id)
+H5MF_tidy_self_referential_fsm_hack(H5F_t *f, hid_t dxpl_id)
{
+ haddr_t eoa; /* EOA of file */
+ hsize_t tail_size = 0; /* Size of chunk to free */
H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
- H5AC_ring_t curr_ring; /* Current ring value */
- H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
- H5FD_mem_t type; /* Memory type for iteration */
+ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
+ haddr_t first_srfsm_hdr = HADDR_UNDEF; /* Addr of first self referential */
+ /* fsm header in file */
+ H5FS_stat_t fs_stat; /* Information for hdr FSM */
+ H5F_mem_page_t sm_fshdr_fs_type; /* Small fs hdr fsm */
+ H5F_mem_page_t sm_fssinfo_fs_type; /* Small fs sinfo fsm */
+ H5F_mem_page_t lg_fshdr_fs_type; /* Large fs hdr fsm */
+ H5F_mem_page_t lg_fssinfo_fs_type; /* Large fs sinfo fsm */
hbool_t reset_ring = FALSE; /* Whether the ring was set */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
-#ifdef H5MF_ALLOC_DEBUG
-HDfprintf(stderr, "%s: Entering\n", FUNC);
-#endif /* H5MF_ALLOC_DEBUG */
/* check args */
HDassert(f);
HDassert(f->shared);
- HDassert(f->shared->lf);
- HDassert(f->shared->sblock);
+ HDassert(f->shared->fs_persist);
+ HDassert(f->shared->first_alloc_dealloc);
- /* Set the ring type in the DXPL. In most cases, we will
- * need H5AC_RING_RDFSM, so initialy set the ring in
- * the DXPL to that value. We will alter this later if
- * needed.
+ /* Set the ring type in the DXPL. Since we are only dealing with
+ * self referential FSMs, we will only need H5AC_RING_MDFSM.
*/
- if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0)
+ if(H5AC_set_ring(dxpl_id, H5AC_RING_MDFSM, &dxpl, &orig_ring) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
reset_ring = TRUE;
- curr_ring = H5AC_RING_RDFSM;
- /* Free the space in aggregators */
- /* (for space not at EOF, it may be put into free space managers) */
- if(H5MF_free_aggrs(f, dxpl_id) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators")
+ /* 1) Verify that f->shared->first_alloc_dealloc is TRUE,
+ * and then set it to FALSE.
+ */
+ HDassert(f->shared->first_alloc_dealloc);
+ f->shared->first_alloc_dealloc = FALSE;
- /* Trying shrinking the EOA for the file */
- if(H5MF__close_shrink_eoa(f, dxpl_id) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
- /* Making free-space managers persistent for superblock version >= 2 */
- if(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2
- && f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) {
- H5O_fsinfo_t fsinfo; /* Free space manager info message */
+ /* 2) Get the current EOA. Verify that it is greater than
+ * or equal to f->shared->eoa_pre_fsm_fsalloc. If the
+ * current eoa is equal to f->shared->eoa_pre_fsm_fsalloc,
+ * no self referential FSMs were stored, and we are done.
+ *
+ * NOTE: This will have to be reworked somewhat for
+ * cache image.
+ */
+ if(HADDR_UNDEF == (eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)))
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get EOA")
+ HDassert(H5F_addr_le(f->shared->eoa_pre_fsm_fsalloc, eoa));
- /* Superblock extension and free space manager message should
- * exist at this point -- verify at least the former.
- */
- HDassert(H5F_addr_defined(f->shared->sblock->ext_addr));
+ if(H5F_addr_eq(f->shared->eoa_pre_fsm_fsalloc, eoa))
+ HGOTO_DONE(SUCCEED)
- /* Note that unlike the previous version of this code, we do not
- * delete free space managers that have no section to store.
- *
- * Can't do this, as that would involve freeing file space, which would
- * dirty the free space manager in question.
- *
- * Fortunately, the code doesn't seem to care about this.
- */
- /* Gather data for the free space manager superblock extension message.
- *
- * In passing, verify that all the free space managers are closed.
- */
- for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
- fsinfo.fs_addr[type - 1] = f->shared->fs_addr[type];
- fsinfo.strategy = f->shared->fs_strategy;
- fsinfo.threshold = f->shared->fs_threshold;
+ /* 3) Load the self referential FSMs. In passing verify that
+ * the lowest address of a FSM header is equal to
+ * f->shared->eoa_pre_fsm_fsalloc.'
+ *
+ * Note that we don't have to use any special I/O for
+ * this -- we can use the regular I/O methods even if
+ * paged aggregation and page buffering is enabled.
+ */
+ H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fs_type);
+ H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fs_type);
+ HDassert(sm_fshdr_fs_type > H5F_MEM_PAGE_DEFAULT);
+ HDassert(sm_fshdr_fs_type < H5F_MEM_PAGE_LARGE_SUPER);
- /* Write the free space manager message -- message must already exist */
- if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_NO_FLAGS_SET) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension")
+ HDassert(sm_fssinfo_fs_type > H5F_MEM_PAGE_DEFAULT);
+ HDassert(sm_fssinfo_fs_type < H5F_MEM_PAGE_LARGE_SUPER);
- /* Final close of free-space managers */
- for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
- if(f->shared->fs_man[type]) {
- H5AC_ring_t needed_ring; /* Ring value needed for this iteration */
+ HDassert(NULL == f->shared->fs_man[sm_fshdr_fs_type]);
+ HDassert(NULL == f->shared->fs_man[sm_fssinfo_fs_type]);
- /* test to see if we need to switch rings -- do so if required */
- if((H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
- || (H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
- needed_ring = H5AC_RING_MDFSM;
- else
- needed_ring = H5AC_RING_RDFSM;
+ if(H5F_addr_defined(f->shared->fs_addr[sm_fshdr_fs_type])) {
+ first_srfsm_hdr = f->shared->fs_addr[sm_fshdr_fs_type];
- if(needed_ring != curr_ring) {
- if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (3)")
- curr_ring = needed_ring;
- } /* end if */
+ /* open the FSM */
+ if(H5MF_open_fstype(f, dxpl_id, sm_fshdr_fs_type) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager")
- HDassert(f->shared->fs_state[type] == H5F_FS_STATE_OPEN);
- if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close free space manager")
- f->shared->fs_man[type] = NULL;
- f->shared->fs_state[type] = H5F_FS_STATE_CLOSED;
- } /* end if */
- f->shared->fs_addr[type] = HADDR_UNDEF;
- } /* end for */
+ HDassert(f->shared->fs_man[sm_fshdr_fs_type]);
+ } /* end if */
- /* Verify that we haven't dirtied any metadata cache entries
- * from the metadata free space manager ring out.
- */
- HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM));
+ if((sm_fshdr_fs_type != sm_fssinfo_fs_type) &&
+ (H5F_addr_defined(f->shared->fs_addr[sm_fssinfo_fs_type]))) {
- /* Verify that the aggregators are still shutdown. */
- HDassert(f->shared->sdata_aggr.tot_size == 0);
- HDassert(f->shared->sdata_aggr.addr == 0);
- HDassert(f->shared->sdata_aggr.size == 0);
- HDassert(f->shared->meta_aggr.tot_size == 0);
- HDassert(f->shared->meta_aggr.addr == 0);
- HDassert(f->shared->meta_aggr.size == 0);
- } /* end if */
- else { /* super_vers can be 0, 1, 2 */
- /* Close and delete freespace managers from the file */
- if(H5MF__close_delete(f, dxpl_id, &dxpl) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space")
+ if(!H5F_addr_defined(first_srfsm_hdr) ||
+ (H5F_addr_defined(first_srfsm_hdr) &&
+ H5F_addr_lt(f->shared->fs_addr[sm_fssinfo_fs_type], first_srfsm_hdr)))
+ first_srfsm_hdr = f->shared->fs_addr[sm_fssinfo_fs_type];
- /* moved code that was for both the persistant and non persistant free
- * space managers to the non-persistant case. In the persistant
- * case, the EOA should already be as shrunked as it can be, and the
- * aggregators should alread be shut down.
- */
+ HDassert(NULL == f->shared->fs_man[sm_fssinfo_fs_type]);
- /* Free the space in aggregators (again) */
- /* (in case any free space information re-started them) */
- if(H5MF_free_aggrs(f, dxpl_id) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators")
+ /* open the FSM */
+ if(H5MF_open_fstype(f, dxpl_id, sm_fssinfo_fs_type) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager")
- /* Trying shrinking the EOA for the file */
- /* (in case any free space is now at the EOA) */
- if(H5MF__close_shrink_eoa(f, dxpl_id) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa")
- } /* end else */
+ HDassert(f->shared->fs_man[sm_fssinfo_fs_type]);
+ } /* end if */
-done:
- /* Reset the ring in the DXPL */
- if(reset_ring)
- if(H5AC_reset_ring(dxpl, orig_ring) < 0)
- HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
+ if(H5F_PAGED_AGGR(f)) {
+ H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fs_type);
+ H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fs_type);
-#ifdef H5MF_ALLOC_DEBUG
-HDfprintf(stderr, "%s: Leaving\n", FUNC);
-#endif /* H5MF_ALLOC_DEBUG */
- FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
-} /* end H5MF_close() */
+ HDassert(lg_fshdr_fs_type >= H5F_MEM_PAGE_LARGE_SUPER);
+ HDassert(lg_fshdr_fs_type < H5F_MEM_PAGE_NTYPES);
-
-/*-------------------------------------------------------------------------
- * Function: H5MF_sects_cb()
- *
- * Purpose: Iterator callback for each free-space section
- * Retrieve address and size into user data
- *
- * Return: Always succeed
- *
- * Programmer: Vailin Choi
- * July 1st, 2009
- *
- *-------------------------------------------------------------------------
- */
-static herr_t
-H5MF_sects_cb(H5FS_section_info_t *_sect, void *_udata)
-{
- H5MF_free_section_t *sect = (H5MF_free_section_t *)_sect;
- H5MF_sect_iter_ud_t *udata = (H5MF_sect_iter_ud_t *)_udata;
+ HDassert(lg_fssinfo_fs_type >= H5F_MEM_PAGE_LARGE_SUPER);
+ HDassert(lg_fssinfo_fs_type < H5F_MEM_PAGE_NTYPES);
- FUNC_ENTER_NOAPI_NOINIT_NOERR
+ HDassert(NULL == f->shared->fs_man[lg_fshdr_fs_type]);
+ HDassert(NULL == f->shared->fs_man[lg_fssinfo_fs_type]);
- if(udata->sect_idx < udata->sect_count) {
- udata->sects[udata->sect_idx].addr = sect->sect_info.addr;
- udata->sects[udata->sect_idx].size = sect->sect_info.size;
- udata->sect_idx++;
- } /* end if */
+ if(H5F_addr_defined(f->shared->fs_addr[lg_fshdr_fs_type])) {
+ if(!H5F_addr_defined(first_srfsm_hdr) ||
+ (H5F_addr_defined(first_srfsm_hdr) &&
+ H5F_addr_lt(f->shared->fs_addr[lg_fshdr_fs_type], first_srfsm_hdr)))
+ first_srfsm_hdr = f->shared->fs_addr[lg_fshdr_fs_type];
- FUNC_LEAVE_NOAPI(SUCCEED)
-} /* H5MF_sects_cb() */
+ HDassert(NULL == f->shared->fs_man[lg_fshdr_fs_type]);
-
-/*-------------------------------------------------------------------------
- * Function: H5MF_get_free_sections()
- *
- * Purpose: To iterate over one or all free-space managers for:
- * # of sections
- * section info as defined in H5F_sect_info_t
- *
- * Return: SUCCEED/FAIL
- *
- * Programmer: Vailin Choi
- * July 1st, 2009
- *
- *-------------------------------------------------------------------------
- */
-ssize_t
-H5MF_get_free_sections(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, size_t nsects, H5F_sect_info_t *sect_info)
-{
- size_t total_sects = 0; /* total number of sections */
- H5MF_sect_iter_ud_t sect_udata; /* User data for callback */
- H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */
- H5AC_ring_t curr_ring; /* Current ring value */
- H5AC_ring_t needed_ring; /* Ring value needed for this iteration */
- H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */
- H5FD_mem_t start_type, end_type; /* Memory types to iterate over */
- H5FD_mem_t ty; /* Memory type for iteration */
- hbool_t reset_ring = FALSE; /* Whether the ring was set */
- ssize_t ret_value = -1; /* Return value */
+ /* open the FSM */
+ if(H5MF_open_fstype(f, dxpl_id, lg_fshdr_fs_type) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager")
+ HDassert(f->shared->fs_man[lg_fshdr_fs_type]);
+ } /* end if */
- FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL)
+ if(lg_fshdr_fs_type != lg_fssinfo_fs_type && H5F_addr_defined(f->shared->fs_addr[lg_fssinfo_fs_type])) {
+ if(!H5F_addr_defined(first_srfsm_hdr) ||
+ (H5F_addr_defined(first_srfsm_hdr) &&
+ H5F_addr_lt(f->shared->fs_addr[lg_fssinfo_fs_type], first_srfsm_hdr)))
+ first_srfsm_hdr = f->shared->fs_addr[lg_fssinfo_fs_type];
- /* check args */
- HDassert(f);
- HDassert(f->shared);
- HDassert(f->shared->lf);
+ HDassert(NULL == f->shared->fs_man[lg_fssinfo_fs_type]);
- /* Determine start/end points for loop */
- if(type == H5FD_MEM_DEFAULT) {
- start_type = H5FD_MEM_SUPER;
- end_type = H5FD_MEM_NTYPES;
+ /* open the FSM */
+ if(H5MF_open_fstype(f, dxpl_id, lg_fssinfo_fs_type) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager")
+ HDassert(f->shared->fs_man[lg_fssinfo_fs_type]);
+ } /* end if */
+ } /* end if */
+ HDassert(H5F_addr_eq(first_srfsm_hdr, f->shared->eoa_pre_fsm_fsalloc));
+
+ /* 4) Float the FSMs. Ensure that the file space is NOT released. */
+ if(f->shared->fs_man[sm_fshdr_fs_type]) {
+ /* Sanity check: Query free space manager info for this type */
+ if(H5FS_stat_info(f, f->shared->fs_man[sm_fshdr_fs_type], &fs_stat) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
+
+ HDassert(H5F_addr_defined(fs_stat.addr));
+ HDassert(H5F_addr_defined(fs_stat.sect_addr));
+ if(H5FS_free(f, f->shared->fs_man[sm_fshdr_fs_type], dxpl_id, FALSE) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers")
+ f->shared->fs_addr[sm_fshdr_fs_type] = HADDR_UNDEF;
} /* end if */
- else {
- start_type = end_type = type;
- H5_INC_ENUM(H5FD_mem_t, end_type);
- } /* end else */
- /* Set up user data for section iteration */
- sect_udata.sects = sect_info;
- sect_udata.sect_count = nsects;
- sect_udata.sect_idx = 0;
+ if(sm_fshdr_fs_type != sm_fssinfo_fs_type && f->shared->fs_man[sm_fssinfo_fs_type]) {
+ /* Sanity check: Query free space manager info for this type */
+ if(H5FS_stat_info(f, f->shared->fs_man[sm_fssinfo_fs_type], &fs_stat) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info")
- /* Set the ring type in the DXPL. Note that if we are
- * scanning a number of different free space managers,
- * we may have to change the ring
- */
- if((H5MF_ALLOC_TO_FS_TYPE(f, start_type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
- || (H5MF_ALLOC_TO_FS_TYPE(f, start_type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
- needed_ring = H5AC_RING_MDFSM;
- else
- needed_ring = H5AC_RING_RDFSM;
- curr_ring = needed_ring;
- if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &orig_ring) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value")
- reset_ring = TRUE;
-
- /* Iterate over memory types, retrieving the number of sections of each type */
- for(ty = start_type; ty < end_type; H5_INC_ENUM(H5FD_mem_t, ty)) {
- hbool_t fs_started = FALSE;
+ HDassert(H5F_addr_defined(fs_stat.addr));
+ HDassert(H5F_addr_defined(fs_stat.sect_addr));
+ if(H5FS_free(f, f->shared->fs_man[sm_fssinfo_fs_type], dxpl_id, FALSE) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers")
+ f->shared->fs_addr[sm_fssinfo_fs_type] = HADDR_UNDEF;
+ } /* end if */
- /* test to see if we need to switch rings -- do so if required */
- if((H5MF_ALLOC_TO_FS_TYPE(f, ty) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR))
- || (H5MF_ALLOC_TO_FS_TYPE(f, ty) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO)))
- needed_ring = H5AC_RING_MDFSM;
- else
- needed_ring = H5AC_RING_RDFSM;
+ if(H5F_PAGED_AGGR(f)) {
+ if(f->shared->fs_man[lg_fshdr_fs_type]) {
+ /* Sanity check: Query free space manager info for this type */
+ if(H5FS_stat_info(f, f->shared->fs_man[lg_fshdr_fs_type], &fs_stat) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info")
- if(needed_ring != curr_ring) {
- if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (2)")
- curr_ring = needed_ring;
+ HDassert(H5F_addr_defined(fs_stat.addr));
+ HDassert(H5F_addr_defined(fs_stat.sect_addr));
+ if(H5FS_free(f, f->shared->fs_man[lg_fshdr_fs_type], dxpl_id, FALSE) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't float free-space headers")
+ f->shared->fs_addr[lg_fshdr_fs_type] = HADDR_UNDEF;
} /* end if */
- /* Open free space manager of this type, if it isn't already */
- if(!f->shared->fs_man[ty] && H5F_addr_defined(f->shared->fs_addr[ty])) {
- if(H5MF__alloc_open(f, dxpl_id, ty) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTOPENOBJ, FAIL, "can't initialize file free space")
- HDassert(f->shared->fs_man[ty]);
- fs_started = TRUE;
- } /* end if */
-
- /* Check if f there's free space sections of this type */
- if(f->shared->fs_man[ty]) {
- hsize_t hnums = 0; /* Total # of sections */
- size_t nums; /* Total # of sections, cast to a size_t */
-
- /* Query how many sections of this type */
- if(H5FS_sect_stats(f->shared->fs_man[ty], NULL, &hnums) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space stats")
- H5_CHECKED_ASSIGN(nums, size_t, hnums, hsize_t);
-
- /* Increment total # of sections */
- total_sects += nums;
-
- /* Check if we should retrieve the section info */
- if(sect_info && nums > 0) {
- /* Iterate over all the free space sections of this type, adding them to the user's section info */
- if(H5FS_sect_iterate(f, dxpl_id, f->shared->fs_man[ty], H5MF_sects_cb, &sect_udata) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_BADITER, FAIL, "can't iterate over sections")
- } /* end if */
+ if(lg_fshdr_fs_type != lg_fssinfo_fs_type && f->shared->fs_man[lg_fssinfo_fs_type]) {
+ /* Sanity check: Query free space manager info for this type */
+ if(H5FS_stat_info(f, f->shared->fs_man[lg_fssinfo_fs_type], &fs_stat) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info")
+
+ HDassert(H5F_addr_defined(fs_stat.addr));
+ HDassert(H5F_addr_defined(fs_stat.sect_addr));
+ if(H5FS_free(f, f->shared->fs_man[lg_fssinfo_fs_type], dxpl_id, FALSE) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't float free-space headers")
+ f->shared->fs_addr[lg_fssinfo_fs_type] = HADDR_UNDEF;
} /* end if */
+ } /* end if */
- /* Close the free space manager of this type, if we started it here */
- if(fs_started)
- if(H5MF__alloc_close(f, dxpl_id, ty) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCLOSEOBJ, FAIL, "can't close file free space")
- } /* end for */
+ /* 5) Set EOA equal to f->shared->eoa_pre_fsm_fsalloc,
+ * and then set f->shared->eoa_pre_fsm_fsalloc to
+ * HADDR_UNDEF.
+ *
+ * If page buffering, verify that the new EOA is
+ * on a page boundary, and expunge any pages in the
+ * page buffer after the new EOA.
+ */
+ if(!H5F_PAGED_AGGR(f)) {
+ /* Verify that the aggregators are still shutdown. */
+ HDassert(f->shared->sdata_aggr.tot_size == 0);
+ HDassert(f->shared->sdata_aggr.addr == 0);
+ HDassert(f->shared->sdata_aggr.size == 0);
- /* Set return value */
- ret_value = (ssize_t)total_sects;
+ HDassert(f->shared->meta_aggr.tot_size == 0);
+ HDassert(f->shared->meta_aggr.addr == 0);
+ HDassert(f->shared->meta_aggr.size == 0);
+ } /* end if */
+
+ tail_size = (hsize_t)(eoa - f->shared->eoa_pre_fsm_fsalloc);
+
+ /* Release file space allocated to self referential FSMs */
+ if(H5F_free(f, dxpl_id, H5FD_MEM_DEFAULT, f->shared->eoa_pre_fsm_fsalloc, tail_size) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "driver free request failed")
+ if(HADDR_UNDEF == (eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)))
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get EOA")
+ HDassert(H5F_addr_eq(f->shared->eoa_pre_fsm_fsalloc, eoa));
+
+ f->shared->eoa_pre_fsm_fsalloc = HADDR_UNDEF;
+
+ HDassert((!H5F_PAGED_AGGR(f)) || (0 == (eoa % f->shared->fs_page_size)));
done:
/* Reset the ring in the DXPL */
@@ -2349,5 +3911,5 @@ done:
HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value")
FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
-} /* H5MF_get_free_sections() */
+} /* H5MF_tidy_self_referential_fsm_hack() */
diff --git a/src/H5MFaggr.c b/src/H5MFaggr.c
index 1510645..de98bf1 100644
--- a/src/H5MFaggr.c
+++ b/src/H5MFaggr.c
@@ -57,8 +57,10 @@
/********************/
/* Local Prototypes */
/********************/
-static herr_t H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type,
+static herr_t H5MF__aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type,
H5F_blk_aggr_t *aggr);
+static haddr_t H5MF__aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr,
+ H5F_blk_aggr_t *other_aggr, H5FD_mem_t type, hsize_t size);
/*********************/
@@ -78,86 +80,9 @@ static herr_t H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type,
/*-------------------------------------------------------------------------
- * Function: H5MF_vfd_alloc
- *
- * Purpose: Allocate SIZE bytes of file memory via H5FD_alloc() and return
- * the relative address where that contiguous chunk of file memory
- * exists.
- * The TYPE argument describes the purpose for which the storage
- * is being requested.
- *
- * Return: Success: The file address of new chunk.
- * Failure: HADDR_UNDEF
- *
- * Programmer: Quincey Koziol
- * January 2, 2017
- *
- *-------------------------------------------------------------------------
- */
-haddr_t
-H5MF_vfd_alloc(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type, hsize_t size,
- hbool_t keep_fragment)
-{
- haddr_t eoa; /* Initial EOA for the file */
- haddr_t eoa_frag_addr = HADDR_UNDEF; /* Address of fragment at EOA */
- hsize_t eoa_frag_size = 0; /* Size of fragment at EOA */
- haddr_t ret_value = HADDR_UNDEF; /* Return value */
-
- FUNC_ENTER_NOAPI(HADDR_UNDEF)
-#ifdef H5MF_ALLOC_DEBUG
-HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_type, size);
-#endif /* H5MF_ALLOC_DEBUG */
-
- /* check arguments */
- HDassert(f);
- HDassert(f->shared);
- HDassert(f->shared->lf);
- HDassert(size > 0);
-
- /* Get the EOA for the file -- need for sanity check below */
- if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, alloc_type)))
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, HADDR_UNDEF, "Unable to get eoa")
-
- /* Check for overlap into temporary allocation space */
- if(H5F_addr_gt((eoa + size), f->shared->tmp_addr))
- HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "hdr file space alloc will overlap into 'temporary' file space")
-
- /* Allocate space for the header */
- if(HADDR_UNDEF == (ret_value = H5FD_alloc(f->shared->lf, dxpl_id, alloc_type, f, size, &eoa_frag_addr, &eoa_frag_size)))
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space for hdr")
-
- /* Sanity check for overlapping into file's temporary allocation space */
- HDassert(H5F_addr_le((ret_value + size), f->shared->tmp_addr));
-
- /* If the file alignment is 1, there should be no eoa fragment */
- HDassert((eoa_frag_size == 0) || (f->shared->alignment != 1));
-
- /* Check if fragment was generated and we want to keep it */
- if(keep_fragment && eoa_frag_size > 0) {
- /* Sanity check */
- HDassert(H5F_addr_defined(eoa_frag_addr));
-
- /* Put fragment on the free list */
- if(H5MF_xfree(f, alloc_type, dxpl_id, eoa_frag_addr, eoa_frag_size) < 0)
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free eoa fragment")
- } /* end if */
-
-done:
-#ifdef H5MF_ALLOC_DEBUG
-HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", FUNC, ret_value, size);
-#endif /* H5MF_ALLOC_DEBUG */
-#ifdef H5MF_ALLOC_DEBUG_DUMP
-H5MF_sects_dump(f, dxpl_id, stderr);
-#endif /* H5MF_ALLOC_DEBUG_DUMP */
-
- FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5MF_vfd_alloc() */
-
-
-/*-------------------------------------------------------------------------
* Function: H5MF_aggr_vfd_alloc
*
- * Purpose: Allocate SIZE bytes of file memory via H5MF_aggr_alloc()
+ * Purpose: Allocate SIZE bytes of file memory via H5MF__aggr_alloc()
* and return the relative address where that contiguous chunk
* of file memory exists.
* The TYPE argument describes the purpose for which the storage
@@ -177,9 +102,9 @@ H5MF_aggr_vfd_alloc(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size
haddr_t ret_value = HADDR_UNDEF; /* Return value */
FUNC_ENTER_NOAPI(HADDR_UNDEF)
-#ifdef H5MF_ALLOC_DEBUG
+#ifdef H5MF_AGGR_DEBUG
HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_type, size);
-#endif /* H5MF_ALLOC_DEBUG */
+#endif /* H5MF_AGGR_DEBUG */
/* check arguments */
HDassert(f);
@@ -190,12 +115,12 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ
/* Couldn't find anything from the free space manager, go allocate some */
if(alloc_type != H5FD_MEM_DRAW && alloc_type != H5FD_MEM_GHEAP) {
/* Handle metadata differently from "raw" data */
- if(HADDR_UNDEF == (ret_value = H5MF_aggr_alloc(f, dxpl_id, &(f->shared->meta_aggr), &(f->shared->sdata_aggr), alloc_type, size)))
+ if(HADDR_UNDEF == (ret_value = H5MF__aggr_alloc(f, dxpl_id, &(f->shared->meta_aggr), &(f->shared->sdata_aggr), alloc_type, size)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate metadata")
} /* end if */
else {
/* Allocate "raw" data: H5FD_MEM_DRAW and H5FD_MEM_GHEAP */
- if(HADDR_UNDEF == (ret_value = H5MF_aggr_alloc(f, dxpl_id, &(f->shared->sdata_aggr), &(f->shared->meta_aggr), H5FD_MEM_DRAW, size)))
+ if(HADDR_UNDEF == (ret_value = H5MF__aggr_alloc(f, dxpl_id, &(f->shared->sdata_aggr), &(f->shared->meta_aggr), H5FD_MEM_DRAW, size)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate raw data")
} /* end else */
@@ -203,19 +128,16 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ
HDassert(H5F_addr_le((ret_value + size), f->shared->tmp_addr));
done:
-#ifdef H5MF_ALLOC_DEBUG
+#ifdef H5MF_AGGR_DEBUG
HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", FUNC, ret_value, size);
-#endif /* H5MF_ALLOC_DEBUG */
-#ifdef H5MF_ALLOC_DEBUG_DUMP
-H5MF_sects_dump(f, dxpl_id, stderr);
-#endif /* H5MF_ALLOC_DEBUG_DUMP */
+#endif /* H5MF_AGGR_DEBUG */
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5MF_aggr_vfd_alloc() */
/*-------------------------------------------------------------------------
- * Function: H5MF_aggr_alloc
+ * Function: H5MF__aggr_alloc
*
* Purpose: Try to allocate SIZE bytes of memory from an aggregator
* block if possible.
@@ -228,8 +150,8 @@ H5MF_sects_dump(f, dxpl_id, stderr);
*
*-------------------------------------------------------------------------
*/
-haddr_t
-H5MF_aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr,
+static haddr_t
+H5MF__aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr,
H5F_blk_aggr_t *other_aggr, H5FD_mem_t type, hsize_t size)
{
haddr_t eoa_frag_addr = HADDR_UNDEF; /* Address of fragment at EOA */
@@ -237,7 +159,7 @@ H5MF_aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr,
haddr_t eoa = HADDR_UNDEF; /* Initial EOA for the file */
haddr_t ret_value = HADDR_UNDEF; /* Return value */
- FUNC_ENTER_NOAPI(HADDR_UNDEF)
+ FUNC_ENTER_STATIC
#ifdef H5MF_AGGR_DEBUG
HDfprintf(stderr, "%s: type = %u, size = %Hu\n", FUNC, (unsigned)type, size);
#endif /* H5MF_AGGR_DEBUG */
@@ -257,11 +179,11 @@ HDfprintf(stderr, "%s: type = %u, size = %Hu\n", FUNC, (unsigned)type, size);
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, HADDR_UNDEF, "Unable to get eoa")
/*
- * If the aggregation feature is enabled for this file and strategy is not H5F_FILE_SPACE_VFD,
+ * If the aggregation feature is enabled for this file and strategy is not H5F_FILE_SPACE_NONE,
* allocate "generic" space and sub-allocate out of that, if possible.
- * Otherwise just allocate through H5FD_alloc().
+ * Otherwise just allocate through H5F_alloc().
*/
- if((f->shared->feature_flags & aggr->feature_flag) && f->shared->fs_strategy != H5F_FILE_SPACE_VFD) {
+ if((f->shared->feature_flags & aggr->feature_flag) && f->shared->fs_strategy != H5F_FSPACE_STRATEGY_NONE) {
haddr_t aggr_frag_addr = HADDR_UNDEF; /* Address of aggregrator fragment */
hsize_t aggr_frag_size = 0; /* Size of aggregator fragment */
hsize_t alignment; /* Alignment of this section */
@@ -273,12 +195,12 @@ HDfprintf(stderr, "%s: aggr = {%a, %Hu, %Hu}\n", FUNC, aggr->addr, aggr->tot_siz
#endif /* H5MF_AGGR_DEBUG */
/* Turn off alignment if allocation < threshold */
- alignment = f->shared->alignment;
- if(!((alignment > 1) && (size >= f->shared->threshold)))
+ alignment = H5F_ALIGNMENT(f);
+ if(!((alignment > 1) && (size >= H5F_THRESHOLD(f))))
alignment = 0; /* no alignment */
/* Generate fragment if aggregator is mis-aligned */
- if(alignment && aggr->addr > 0 && (aggr_mis_align = (aggr->addr + H5FD_get_base_addr(f->shared->lf)) % alignment)) {
+ if(alignment && H5F_addr_gt(aggr->addr, 0) && (aggr_mis_align = (aggr->addr + H5F_BASE_ADDR(f)) % alignment)) {
aggr_frag_addr = aggr->addr;
aggr_frag_size = alignment - aggr_mis_align;
} /* end if */
@@ -288,7 +210,7 @@ HDfprintf(stderr, "%s: aggr = {%a, %Hu, %Hu}\n", FUNC, aggr->addr, aggr->tot_siz
/* Check if the space requested is larger than the space left in the block */
if((size + aggr_frag_size) > aggr->size) {
- htri_t was_extended = FALSE; /* Whether the file was extended */
+ htri_t extended = FALSE; /* Whether the file was extended */
/* Check if the block asked for is too large for 'normal' aggregator block */
if(size >= aggr->alloc_size) {
@@ -298,32 +220,27 @@ HDfprintf(stderr, "%s: aggr = {%a, %Hu, %Hu}\n", FUNC, aggr->addr, aggr->tot_siz
if(H5F_addr_gt((aggr->addr + aggr->size + ext_size), f->shared->tmp_addr))
HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space")
- if((aggr->addr > 0) && (was_extended = H5FD_try_extend(f->shared->lf, alloc_type, f, dxpl_id, aggr->addr + aggr->size, ext_size)) < 0)
+ if((aggr->addr > 0) && (extended = H5F_try_extend(f, dxpl_id, alloc_type, (aggr->addr + aggr->size), ext_size)) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't extending space")
- else if(was_extended) {
+ else if (extended) {
/* aggr->size is unchanged */
ret_value = aggr->addr + aggr_frag_size;
aggr->addr += ext_size;
aggr->tot_size += ext_size;
- } /* end else-if */
- else {
- /* Check for overlapping into file's temporary allocation space */
- if(H5F_addr_gt((eoa + size), f->shared->tmp_addr))
- HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space")
-
+ } else {
/* Release "other" aggregator, if it exists, is at the end of the allocated space,
* has allocated more than one block and the unallocated space is greater than its
* allocation block size.
*/
if((other_aggr->size > 0) && (H5F_addr_eq((other_aggr->addr + other_aggr->size), eoa)) &&
(other_aggr->tot_size > other_aggr->size) && ((other_aggr->tot_size - other_aggr->size) >= other_aggr->alloc_size)) {
- if(H5MF_aggr_free(f, dxpl_id, other_alloc_type, other_aggr) < 0)
+ if(H5MF__aggr_free(f, dxpl_id, other_alloc_type, other_aggr) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free aggregation block")
} /* end if */
/* Allocate space from the VFD (i.e. at the end of the file) */
- if(HADDR_UNDEF == (ret_value = H5FD_alloc(f->shared->lf, dxpl_id, alloc_type, f, size, &eoa_frag_addr, &eoa_frag_size)))
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate aggregation block")
+ if(HADDR_UNDEF == (ret_value = H5F_alloc(f, dxpl_id, alloc_type, size, &eoa_frag_addr, &eoa_frag_size)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space")
} /* end else */
} /* end if */
else {
@@ -341,9 +258,9 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC);
if(H5F_addr_gt((aggr->addr + aggr->size + ext_size), f->shared->tmp_addr))
HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space")
- if((aggr->addr > 0) && (was_extended = H5FD_try_extend(f->shared->lf, alloc_type, f, dxpl_id, aggr->addr + aggr->size, ext_size)) < 0)
+ if((aggr->addr > 0) && (extended = H5F_try_extend(f, dxpl_id, alloc_type, (aggr->addr + aggr->size), ext_size)) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't extending space")
- else if(was_extended) {
+ else if(extended) {
aggr->addr += aggr_frag_size;
aggr->size += (ext_size - aggr_frag_size);
aggr->tot_size += ext_size;
@@ -351,23 +268,19 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC);
else {
haddr_t new_space; /* Address of new space allocated */
- /* Check for overlapping into file's temporary allocation space */
- if(H5F_addr_gt((eoa + aggr->alloc_size), f->shared->tmp_addr))
- HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space")
-
/* Release "other" aggregator, if it exists, is at the end of the allocated space,
* has allocated more than one block and the unallocated space is greater than its
* allocation block size.
*/
if((other_aggr->size > 0) && (H5F_addr_eq((other_aggr->addr + other_aggr->size), eoa)) &&
(other_aggr->tot_size > other_aggr->size) && ((other_aggr->tot_size - other_aggr->size) >= other_aggr->alloc_size)) {
- if(H5MF_aggr_free(f, dxpl_id, other_alloc_type, other_aggr) < 0)
+ if(H5MF__aggr_free(f, dxpl_id, other_alloc_type, other_aggr) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free aggregation block")
} /* end if */
/* Allocate space from the VFD (i.e. at the end of the file) */
- if(HADDR_UNDEF == (new_space = H5FD_alloc(f->shared->lf, dxpl_id, alloc_type, f, aggr->alloc_size, &eoa_frag_addr, &eoa_frag_size)))
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate aggregation block")
+ if(HADDR_UNDEF == (new_space = H5F_alloc(f, dxpl_id, alloc_type, aggr->alloc_size, &eoa_frag_addr, &eoa_frag_size)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space")
/* Return the unused portion of the block to a free list */
if(aggr->size > 0)
@@ -408,7 +321,7 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC);
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free eoa fragment")
/* Freeing any possible fragment due to alignment in the block after extension */
- if(was_extended && aggr_frag_size)
+ if(extended && aggr_frag_size)
if(H5MF_xfree(f, alloc_type, dxpl_id, aggr_frag_addr, aggr_frag_size) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free aggregation fragment")
} /* end if */
@@ -425,12 +338,8 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC);
} /* end else */
} /* end if */
else {
- /* Check for overlapping into file's temporary allocation space */
- if(H5F_addr_gt((eoa + size), f->shared->tmp_addr))
- HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space")
-
/* Allocate data from the file */
- if(HADDR_UNDEF == (ret_value = H5FD_alloc(f->shared->lf, dxpl_id, type, f, size, &eoa_frag_addr, &eoa_frag_size)))
+ if(HADDR_UNDEF == (ret_value = H5F_alloc(f, dxpl_id, type, size, &eoa_frag_addr, &eoa_frag_size)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space")
/* Check if fragment was generated */
@@ -444,15 +353,15 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC);
HDassert(H5F_addr_le((ret_value + size), f->shared->tmp_addr));
/* Post-condition sanity check */
- if(f->shared->alignment && size >= f->shared->threshold)
- HDassert(!((ret_value + H5FD_get_base_addr(f->shared->lf)) % f->shared->alignment));
+ if(H5F_ALIGNMENT(f) && size >= H5F_THRESHOLD(f))
+ HDassert(!((ret_value + H5FD_get_base_addr(f->shared->lf)) % H5F_ALIGNMENT(f)));
done:
#ifdef H5MF_AGGR_DEBUG
HDfprintf(stderr, "%s: ret_value = %a\n", FUNC, ret_value);
#endif /* H5MF_AGGR_DEBUG */
FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5MF_aggr_alloc() */
+} /* end H5MF__aggr_alloc() */
/*-------------------------------------------------------------------------
@@ -524,7 +433,7 @@ H5MF_aggr_try_extend(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr,
else {
hsize_t extra = (extra_requested < aggr->alloc_size) ? aggr->alloc_size : extra_requested;
- if((ret_value = H5FD_try_extend(f->shared->lf, type, f, dxpl_id, (aggr->addr + aggr->size), extra)) < 0)
+ if((ret_value = H5F_try_extend(f, dxpl_id, type, (aggr->addr + aggr->size), extra)) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file")
else if(ret_value == TRUE) {
/* Shift the aggregator block by the extra requested */
@@ -905,7 +814,7 @@ done:
/*-------------------------------------------------------------------------
- * Function: H5MF_aggr_free
+ * Function: H5MF__aggr_free
*
* Purpose: Free the aggregator's space in the file.
*
@@ -919,11 +828,11 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
-H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr)
+H5MF__aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr)
{
herr_t ret_value = SUCCEED; /* Return value */
- FUNC_ENTER_NOAPI(FAIL)
+ FUNC_ENTER_STATIC
/* Sanity check */
HDassert(f);
@@ -936,7 +845,7 @@ H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr)
HDassert(f->shared->feature_flags & aggr->feature_flag);
/* Free the remaining space at EOA in the aggregator */
- if(H5FD_free(f->shared->lf, dxpl_id, type, f, aggr->addr, aggr->size) < 0)
+ if(H5F_free(f, dxpl_id, type, aggr->addr, aggr->size) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free aggregation block")
/* Reset the aggregator */
@@ -946,7 +855,7 @@ H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr)
done:
FUNC_LEAVE_NOAPI(ret_value)
-} /* H5MF_aggr_free() */
+} /* H5MF__aggr_free() */
/*-------------------------------------------------------------------------
@@ -978,13 +887,13 @@ H5MF_aggrs_try_shrink_eoa(H5F_t *f, hid_t dxpl_id)
if((ma_status = H5MF_aggr_can_shrink_eoa(f, H5FD_MEM_DEFAULT, &(f->shared->meta_aggr))) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query metadata aggregator stats")
if(ma_status > 0)
- if(H5MF_aggr_free(f, dxpl_id, H5FD_MEM_DEFAULT, &(f->shared->meta_aggr)) < 0)
+ if(H5MF__aggr_free(f, dxpl_id, H5FD_MEM_DEFAULT, &(f->shared->meta_aggr)) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa")
if((sda_status = H5MF_aggr_can_shrink_eoa(f, H5FD_MEM_DRAW, &(f->shared->sdata_aggr))) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query small data aggregator stats")
if(sda_status > 0)
- if(H5MF_aggr_free(f, dxpl_id, H5FD_MEM_DRAW, &(f->shared->sdata_aggr)) < 0)
+ if(H5MF__aggr_free(f, dxpl_id, H5FD_MEM_DRAW, &(f->shared->sdata_aggr)) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa")
ret_value = (ma_status || sda_status);
diff --git a/src/H5MFdbg.c b/src/H5MFdbg.c
index 57bac9e..817ea00 100644
--- a/src/H5MFdbg.c
+++ b/src/H5MFdbg.c
@@ -116,7 +116,9 @@ H5MF_sects_debug_cb(H5FS_section_info_t *_sect, void *_udata)
/* Print generic section information */
HDfprintf(udata->stream, "%*s%-*s %s\n", udata->indent, "", udata->fwidth,
"Section type:",
- (sect->sect_info.type == H5MF_FSPACE_SECT_SIMPLE ? "simple" : "unknown"));
+ (sect->sect_info.type == H5MF_FSPACE_SECT_SIMPLE ? "simple" :
+ (sect->sect_info.type == H5MF_FSPACE_SECT_SMALL ? "small" :
+ (sect->sect_info.type == H5MF_FSPACE_SECT_LARGE ? "large" : "unknown"))));
HDfprintf(udata->stream, "%*s%-*s %a\n", udata->indent, "", udata->fwidth,
"Section address:",
sect->sect_info.addr);
@@ -171,7 +173,7 @@ H5MF_sects_debug(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr, FILE *stream, int ind
for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
if(H5F_addr_eq(f->shared->fs_addr[type], fs_addr)) {
if(!f->shared->fs_man[type])
- if(H5MF__alloc_open(f, dxpl_id, type) < 0)
+ if(H5MF_open_fstype(f, dxpl_id, type) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space")
if(f->shared->fs_man[type]) {
@@ -217,11 +219,6 @@ herr_t
H5MF_sects_dump(H5F_t *f, hid_t dxpl_id, FILE *stream)
{
haddr_t eoa; /* End of allocated space in the file */
- haddr_t ma_addr = HADDR_UNDEF; /* Base "metadata aggregator" address */
- hsize_t ma_size = 0; /* Size of "metadata aggregator" */
- haddr_t sda_addr = HADDR_UNDEF; /* Base "small data aggregator" address */
- hsize_t sda_size = 0; /* Size of "small data aggregator" */
- H5FD_mem_t type; /* Memory type for iteration */
int indent = 0; /* Amount to indent */
int fwidth = 50; /* Field width */
herr_t ret_value = SUCCEED; /* Return value */
@@ -244,56 +241,90 @@ HDfprintf(stderr, "%s: Dumping file free space sections\n", FUNC);
HDfprintf(stderr, "%s: for type = H5FD_MEM_DEFAULT, eoa = %a\n", FUNC, eoa);
#endif /* H5MF_ALLOC_DEBUG */
- /* Retrieve metadata aggregator info, if available */
- H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size);
-#ifdef H5MF_ALLOC_DEBUG
-HDfprintf(stderr, "%s: ma_addr = %a, ma_size = %Hu, end of ma = %a\n", FUNC, ma_addr, ma_size, (haddr_t)((ma_addr + ma_size) - 1));
-#endif /* H5MF_ALLOC_DEBUG */
+ if(H5F_PAGED_AGGR(f)) { /* File space paging */
+ H5F_mem_page_t ptype; /* Memory type for iteration -- page fs */
- /* Retrieve 'small data' aggregator info, if available */
- H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sda_addr, &sda_size);
-#ifdef H5MF_ALLOC_DEBUG
-HDfprintf(stderr, "%s: sda_addr = %a, sda_size = %Hu, end of sda = %a\n", FUNC, sda_addr, sda_size, (haddr_t)((sda_addr + sda_size) - 1));
-#endif /* H5MF_ALLOC_DEBUG */
-
- /* Iterate over all the free space types that have managers and dump each free list's space */
- for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) {
- /* Print header for type */
- HDfprintf(stream, "%*sFile Free Space Info for type = %u:\n", indent, "", (unsigned)type);
-
- /* Check for this type being mapped to another type */
- if(H5FD_MEM_DEFAULT == f->shared->fs_type_map[type] ||
- type == f->shared->fs_type_map[type]) {
- /* Retrieve the 'eoa' for this file memory type */
- if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, type)))
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed")
- HDfprintf(stream, "%*s%-*s %a\n", indent + 3, "", MAX(0, fwidth - 3), "eoa:", eoa);
+ for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) {
+ /* Print header for type */
+ HDfprintf(stream, "%*sFile Free Space Info for type = %u:\n", indent, "", (unsigned)ptype);
/* Print header for sections */
HDfprintf(stream, "%*sSections:\n", indent + 3, "");
/* If there is a free space manager for this type, iterate over them */
- if(f->shared->fs_man[type]) {
+ if(f->shared->fs_man[ptype]) {
H5MF_debug_iter_ud_t udata; /* User data for callbacks */
/* Prepare user data for section iteration callback */
- udata.fspace = f->shared->fs_man[type];
+ udata.fspace = f->shared->fs_man[ptype];
udata.stream = stream;
udata.indent = indent + 6;
udata.fwidth = MAX(0, fwidth - 6);
/* Iterate over all the free space sections */
- if(H5FS_sect_iterate(f, dxpl_id, f->shared->fs_man[type], H5MF_sects_debug_cb, &udata) < 0)
+ if(H5FS_sect_iterate(f, dxpl_id, f->shared->fs_man[ptype], H5MF_sects_debug_cb, &udata) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_BADITER, FAIL, "can't iterate over heap's free space")
} /* end if */
else
/* No sections of this type */
HDfprintf(stream, "%*s<none>\n", indent + 6, "");
- } /* end if */
- else
- HDfprintf(stream, "%*sMapped to type = %u\n", indent, "", (unsigned)f->shared->fs_type_map[type]);
+ } /* end for */
+ } /* end if */
+ else { /* not file space paging */
+ H5FD_mem_t atype; /* Memory type for iteration -- aggr fs */
+ haddr_t ma_addr = HADDR_UNDEF; /* Base "metadata aggregator" address */
+ hsize_t ma_size = 0; /* Size of "metadata aggregator" */
+ haddr_t sda_addr = HADDR_UNDEF; /* Base "small data aggregator" address */
+ hsize_t sda_size = 0; /* Size of "small data aggregator" */
+
+ /* Retrieve metadata aggregator info, if available */
+ H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size);
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: ma_addr = %a, ma_size = %Hu, end of ma = %a\n", FUNC, ma_addr, ma_size, (haddr_t)((ma_addr + ma_size) - 1));
+#endif /* H5MF_ALLOC_DEBUG */
+
+ /* Retrieve 'small data' aggregator info, if available */
+ H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sda_addr, &sda_size);
+#ifdef H5MF_ALLOC_DEBUG
+HDfprintf(stderr, "%s: sda_addr = %a, sda_size = %Hu, end of sda = %a\n", FUNC, sda_addr, sda_size, (haddr_t)((sda_addr + sda_size) - 1));
+#endif /* H5MF_ALLOC_DEBUG */
- } /* end for */
+ /* Iterate over all the free space types that have managers and dump each free list's space */
+ for(atype = H5FD_MEM_DEFAULT; atype < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, atype)) {
+ /* Print header for type */
+ HDfprintf(stream, "%*sFile Free Space Info for type = %u:\n", indent, "", (unsigned)atype);
+
+ /* Check for this type being mapped to another type */
+ if(H5FD_MEM_DEFAULT == f->shared->fs_type_map[atype] || atype == f->shared->fs_type_map[atype]) {
+ /* Retrieve the 'eoa' for this file memory type */
+ if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, atype)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed")
+ HDfprintf(stream, "%*s%-*s %a\n", indent + 3, "", MAX(0, fwidth - 3), "eoa:", eoa);
+
+ /* Print header for sections */
+ HDfprintf(stream, "%*sSections:\n", indent + 3, "");
+
+ /* If there is a free space manager for this type, iterate over them */
+ if(f->shared->fs.aggr.fs_man[atype]) {
+ H5MF_debug_iter_ud_t udata; /* User data for callbacks */
+
+ /* Prepare user data for section iteration callback */
+ udata.fspace = f->shared->fs_man[atype];
+ udata.stream = stream;
+ udata.indent = indent + 6;
+ udata.fwidth = MAX(0, fwidth - 6);
+
+ /* Iterate over all the free space sections */
+ if(H5FS_sect_iterate(f, dxpl_id, f->shared->fs_man[atype], H5MF_sects_debug_cb, &udata) < 0)
+ HGOTO_ERROR(H5E_HEAP, H5E_BADITER, FAIL, "can't iterate over heap's free space")
+ } /* end if */
+ else /* No sections of this type */
+ HDfprintf(stream, "%*s<none>\n", indent + 6, "");
+ } /* end if */
+ else
+ HDfprintf(stream, "%*sMapped to type = %u\n", indent, "", (unsigned)f->shared->fs_type_map[atype]);
+ } /* end for */
+ } /* end else */
done:
HDfprintf(stderr, "%s: Done dumping file free space sections\n", FUNC);
diff --git a/src/H5MFpkg.h b/src/H5MFpkg.h
index 43fc1cd..c4a0819 100644
--- a/src/H5MFpkg.h
+++ b/src/H5MFpkg.h
@@ -51,9 +51,38 @@
/* Define this to dump free space tracker contents after they've been modified */
/* #define H5MF_ALLOC_DEBUG_DUMP */
-/* Free space section types for file */
+/* Free-space section types for file */
/* (values stored in free space data structures in file) */
-#define H5MF_FSPACE_SECT_SIMPLE 0 /* Section is a range of actual bytes in file */
+#define H5MF_FSPACE_SECT_SIMPLE 0 /* For non-paged aggregation: section is a range of actual bytes in file */
+#define H5MF_FSPACE_SECT_SMALL 1 /* For paged aggregation: "small" meta/raw data section which is < fsp_size) */
+#define H5MF_FSPACE_SECT_LARGE 2 /* For paged aggregation: "large" Section which is >= fsp_size) */
+
+/* For non-paged aggregation: map allocation request type to tracked free-space type */
+/* F -- pointer to H5F_t; T -- H5FD_mem_t */
+#define H5MF_ALLOC_TO_FS_AGGR_TYPE(F, T) \
+ ((H5FD_MEM_DEFAULT == (F)->shared->fs_type_map[T]) ? (T) : (F)->shared->fs_type_map[T])
+
+/* Get section class type based on size */
+#define H5MF_SECT_CLASS_TYPE(F, S) \
+ ((H5F_PAGED_AGGR(F)) ? \
+ ((S >= (F)->shared->fs_page_size) ? H5MF_FSPACE_SECT_LARGE : H5MF_FSPACE_SECT_SMALL) : H5MF_FSPACE_SECT_SIMPLE)
+
+/* Get section class cls */
+#define H5MF_SECT_CLS_TYPE(F, S) \
+ ((H5F_PAGED_AGGR(F)) ? \
+ ((S >= (F)->shared->fs_page_size) ? \
+ H5MF_FSPACE_SECT_CLS_LARGE : H5MF_FSPACE_SECT_CLS_SMALL) : H5MF_FSPACE_SECT_CLS_SIMPLE)
+
+/* Calculate the mis-aligned fragment */
+#define H5MF_EOA_MISALIGN(F, E, A, FR) \
+{ \
+ hsize_t m; \
+ \
+ if(H5F_addr_gt((E), 0) && ((m) = ((E) + H5F_BASE_ADDR(F)) % (A))) \
+ (FR) = (A) - m; \
+ else \
+ (FR) = 0; \
+}
/****************************/
@@ -129,6 +158,15 @@ typedef struct H5MF_sect_ud_t {
H5F_blk_aggr_t *aggr; /* Aggregator block to operate on */
} H5MF_sect_ud_t;
+/* Information about the current free-space manager to use */
+typedef struct H5MF_fs_t {
+ H5F_fs_state_t *fs_state;
+ haddr_t *fs_addr;
+ H5FS_t **fs_man;
+ hsize_t align_thres; /* Threshold for alignment */
+ hsize_t alignment; /* Alignment */
+} H5MF_fs_t;
+
/*****************************/
/* Package Private Variables */
@@ -136,6 +174,8 @@ typedef struct H5MF_sect_ud_t {
/* H5MF single section inherits serializable properties from H5FS_section_class_t */
H5_DLLVAR H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1];
+H5_DLLVAR H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SMALL[1];
+H5_DLLVAR H5FS_section_class_t H5MF_FSPACE_SECT_CLS_LARGE[1];
/******************************/
@@ -143,22 +183,23 @@ H5_DLLVAR H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1];
/******************************/
/* Allocator routines */
-H5_DLL herr_t H5MF__alloc_start(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type);
-H5_DLL herr_t H5MF__alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type);
+H5_DLL herr_t H5MF_open_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type);
+H5_DLL herr_t H5MF_start_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type);
+
+H5_DLL htri_t H5MF_find_sect(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size, H5FS_t *fspace, haddr_t *addr);
+H5_DLL herr_t H5MF_add_sect(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, H5FS_t *fspace, H5MF_free_section_t *node);
+
H5_DLL herr_t H5MF_sects_dump(H5F_t *f, hid_t dxpl_id, FILE *stream);
-/* 'simple' section routines */
-H5_DLL H5MF_free_section_t *H5MF_sect_simple_new(haddr_t sect_off,
+H5_DLL void H5MF_alloc_to_fs_type(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size, H5F_mem_page_t *fs_type);
+
+/* 'simple/small/large' section routines */
+H5_DLL H5MF_free_section_t *H5MF_sect_new(unsigned ctype, haddr_t sect_off,
hsize_t sect_size);
-H5_DLL htri_t H5MF_sect_simple_can_shrink(const H5FS_section_info_t *_sect,
- void *udata);
-H5_DLL herr_t H5MF_sect_simple_shrink(H5FS_section_info_t **_sect,
- void *udata);
-H5_DLL herr_t H5MF_sect_simple_free(H5FS_section_info_t *sect);
+H5_DLL herr_t H5MF_sect_free(H5FS_section_info_t *sect);
+
/* Block aggregator routines */
-H5_DLL haddr_t H5MF_aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr,
- H5F_blk_aggr_t *other_aggr, H5FD_mem_t type, hsize_t size);
H5_DLL htri_t H5MF_aggr_try_extend(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr,
H5FD_mem_t type, haddr_t abs_blk_end, hsize_t extra_requested);
H5_DLL htri_t H5MF_aggr_can_absorb(const H5F_t *f, const H5F_blk_aggr_t *aggr,
diff --git a/src/H5MFprivate.h b/src/H5MFprivate.h
index e258677..70322c3 100644
--- a/src/H5MFprivate.h
+++ b/src/H5MFprivate.h
@@ -58,8 +58,6 @@ H5_DLL herr_t H5MF_try_close(H5F_t *f, hid_t dxpl_id);
/* File space allocation routines */
H5_DLL haddr_t H5MF_alloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, hsize_t size);
H5_DLL haddr_t H5MF_aggr_vfd_alloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, hsize_t size);
-H5_DLL haddr_t H5MF_vfd_alloc(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type,
- hsize_t size, hbool_t keep_fragment);
H5_DLL herr_t H5MF_xfree(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr,
hsize_t size);
H5_DLL herr_t H5MF_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type,
@@ -80,6 +78,14 @@ H5_DLL htri_t H5MF_aggrs_try_shrink_eoa(H5F_t *f, hid_t dxpl_id);
H5_DLL herr_t H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled);
H5_DLL herr_t H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled);
+/* This function has to be declared in H5MFprivate.h as it is needed
+ * in our test code to allow us to manually start a self referential
+ * free space manager prior to the first file space allocations /
+ * deallocation without causing assertion failures on the first
+ * file space allocation / deallocation.
+ */
+H5_DLL herr_t H5MF_tidy_self_referential_fsm_hack(H5F_t *f, hid_t dxpl_id);
+
/* Debugging routines */
#ifdef H5MF_DEBUGGING
H5_DLL herr_t H5MF_sects_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr,
diff --git a/src/H5MFsection.c b/src/H5MFsection.c
index e5a0cf0..617cb59 100644
--- a/src/H5MFsection.c
+++ b/src/H5MFsection.c
@@ -57,18 +57,47 @@
/* Local Prototypes */
/********************/
-/* 'simple' section callbacks */
-static H5FS_section_info_t *H5MF_sect_simple_deserialize(const H5FS_section_class_t *cls,
+/* 'simple/small/large' section callbacks */
+static H5FS_section_info_t *H5MF_sect_deserialize(const H5FS_section_class_t *cls,
hid_t dxpl_id, const uint8_t *buf, haddr_t sect_addr, hsize_t sect_size,
unsigned *des_flags);
+static herr_t H5MF_sect_valid(const H5FS_section_class_t *cls,
+ const H5FS_section_info_t *sect, hid_t dxpl_id);
+static H5FS_section_info_t *H5MF_sect_split(H5FS_section_info_t *sect,
+ hsize_t frag_size);
+
+
+/* 'simple' section callbacks */
static htri_t H5MF_sect_simple_can_merge(const H5FS_section_info_t *sect1,
const H5FS_section_info_t *sect2, void *udata);
-static herr_t H5MF_sect_simple_merge(H5FS_section_info_t *sect1,
+static herr_t H5MF_sect_simple_merge(H5FS_section_info_t **sect1,
H5FS_section_info_t *sect2, void *udata);
-static herr_t H5MF_sect_simple_valid(const H5FS_section_class_t *cls,
- const H5FS_section_info_t *sect, hid_t dxpl_id);
-static H5FS_section_info_t *H5MF_sect_simple_split(H5FS_section_info_t *sect,
- hsize_t frag_size);
+static htri_t H5MF_sect_simple_can_shrink(const H5FS_section_info_t *_sect,
+ void *udata);
+static herr_t H5MF_sect_simple_shrink(H5FS_section_info_t **_sect,
+ void *udata);
+
+
+/* 'small' section callbacks */
+static herr_t H5MF_sect_small_add(H5FS_section_info_t **_sect, unsigned *flags, void *_udata);
+static htri_t H5MF_sect_small_can_merge(const H5FS_section_info_t *sect1,
+ const H5FS_section_info_t *sect2, void *udata);
+static herr_t H5MF_sect_small_merge(H5FS_section_info_t **sect1,
+ H5FS_section_info_t *sect2, void *udata);
+static htri_t H5MF_sect_small_can_shrink(const H5FS_section_info_t *_sect,
+ void *udata);
+static herr_t H5MF_sect_small_shrink(H5FS_section_info_t **_sect,
+ void *udata);
+
+/* 'large' section callbacks */
+static htri_t H5MF_sect_large_can_merge(const H5FS_section_info_t *sect1,
+ const H5FS_section_info_t *sect2, void *udata);
+static herr_t H5MF_sect_large_merge(H5FS_section_info_t **sect1,
+ H5FS_section_info_t *sect2, void *udata);
+static htri_t H5MF_sect_large_can_shrink(const H5FS_section_info_t *_sect,
+ void *udata);
+static herr_t H5MF_sect_large_shrink(H5FS_section_info_t **_sect,
+ void *udata);
/*********************/
/* Package Variables */
@@ -89,17 +118,68 @@ H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1] = {{
/* Object methods */
NULL, /* Add section */
NULL, /* Serialize section */
- H5MF_sect_simple_deserialize, /* Deserialize section */
+ H5MF_sect_deserialize, /* Deserialize section */
H5MF_sect_simple_can_merge, /* Can sections merge? */
H5MF_sect_simple_merge, /* Merge sections */
H5MF_sect_simple_can_shrink, /* Can section shrink container?*/
H5MF_sect_simple_shrink, /* Shrink container w/section */
- H5MF_sect_simple_free, /* Free section */
- H5MF_sect_simple_valid, /* Check validity of section */
- H5MF_sect_simple_split, /* Split section node for alignment */
+ H5MF_sect_free, /* Free section */
+ H5MF_sect_valid, /* Check validity of section */
+ H5MF_sect_split, /* Split section node for alignment */
NULL, /* Dump debugging for section */
}};
+/* Class info for "small" free space sections */
+H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SMALL[1] = {{
+ /* Class variables */
+ H5MF_FSPACE_SECT_SMALL, /* Section type */
+ 0, /* Extra serialized size */
+ H5FS_CLS_MERGE_SYM | H5FS_CLS_ADJUST_OK, /* Class flags */
+ NULL, /* Class private info */
+
+ /* Class methods */
+ NULL, /* Initialize section class */
+ NULL, /* Terminate section class */
+
+ /* Object methods */
+ H5MF_sect_small_add, /* Add section */
+ NULL, /* Serialize section */
+ H5MF_sect_deserialize, /* Deserialize section */
+ H5MF_sect_small_can_merge, /* Can sections merge? */
+ H5MF_sect_small_merge, /* Merge sections */
+ H5MF_sect_small_can_shrink, /* Can section shrink container?*/
+ H5MF_sect_small_shrink, /* Shrink container w/section */
+ H5MF_sect_free, /* Free section */
+ H5MF_sect_valid, /* Check validity of section */
+ H5MF_sect_split, /* Split section node for alignment */
+ NULL, /* Dump debugging for section */
+}};
+
+/* Class info for "large" free space sections */
+H5FS_section_class_t H5MF_FSPACE_SECT_CLS_LARGE[1] = {{
+ /* Class variables */
+ H5MF_FSPACE_SECT_LARGE, /* Section type */
+ 0, /* Extra serialized size */
+ H5FS_CLS_MERGE_SYM | H5FS_CLS_ADJUST_OK, /* Class flags */
+ NULL, /* Class private info */
+
+ /* Class methods */
+ NULL, /* Initialize section class */
+ NULL, /* Terminate section class */
+
+ /* Object methods */
+ NULL, /* Add section */
+ NULL, /* Serialize section */
+ H5MF_sect_deserialize, /* Deserialize section */
+ H5MF_sect_large_can_merge, /* Can sections merge? */
+ H5MF_sect_large_merge, /* Merge sections */
+ H5MF_sect_large_can_shrink, /* Can section shrink container?*/
+ H5MF_sect_large_shrink, /* Shrink container w/section */
+ H5MF_sect_free, /* Free section */
+ H5MF_sect_valid, /* Check validity of section */
+ H5MF_sect_split, /* Split section node for alignment */
+ NULL, /* Dump debugging for section */
+}};
/*****************************/
/* Library Private Variables */
@@ -113,12 +193,15 @@ H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1] = {{
/* Declare a free list to manage the H5MF_free_section_t struct */
H5FL_DEFINE(H5MF_free_section_t);
+/*
+ * "simple/small/large" section callbacks
+ */
/*-------------------------------------------------------------------------
- * Function: H5MF_sect_simple_new
+ * Function: H5MF_sect_new
*
- * Purpose: Create a new 'simple' section and return it to the caller
+ * Purpose: Create a new section of "ctype" and return it to the caller
*
* Return: Pointer to new section on success/NULL on failure
*
@@ -129,9 +212,9 @@ H5FL_DEFINE(H5MF_free_section_t);
*-------------------------------------------------------------------------
*/
H5MF_free_section_t *
-H5MF_sect_simple_new(haddr_t sect_off, hsize_t sect_size)
+H5MF_sect_new(unsigned ctype, haddr_t sect_off, hsize_t sect_size)
{
- H5MF_free_section_t *sect = NULL; /* 'Simple' free space section to add */
+ H5MF_free_section_t *sect; /* 'Simple' free space section to add */
H5MF_free_section_t *ret_value = NULL; /* Return value */
FUNC_ENTER_NOAPI_NOINIT
@@ -148,7 +231,7 @@ H5MF_sect_simple_new(haddr_t sect_off, hsize_t sect_size)
sect->sect_info.size = sect_size;
/* Set the section's class & state */
- sect->sect_info.type = H5MF_FSPACE_SECT_SIMPLE;
+ sect->sect_info.type = ctype;
sect->sect_info.state = H5FS_SECT_LIVE;
/* Set return value */
@@ -156,13 +239,43 @@ H5MF_sect_simple_new(haddr_t sect_off, hsize_t sect_size)
done:
FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5MF_sect_simple_new() */
+} /* end H5MF_sect_new() */
/*-------------------------------------------------------------------------
- * Function: H5MF_sect_simple_deserialize
+ * Function: H5MF_sect_free
*
- * Purpose: Deserialize a buffer into a "live" single section
+ * Purpose: Free a 'simple/small/large' section node
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ * Programmer: Quincey Koziol
+ * Tuesday, January 8, 2008
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5MF_sect_free(H5FS_section_info_t *_sect)
+{
+ H5MF_free_section_t *sect = (H5MF_free_section_t *)_sect; /* File free section */
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ /* Check arguments. */
+ HDassert(sect);
+
+ /* Release the section */
+ sect = H5FL_FREE(H5MF_free_section_t, sect);
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5MF_sect_free() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF_sect_deserialize
+ *
+ * Purpose: Deserialize a buffer into a "live" section
*
* Return: Success: non-negative
* Failure: negative
@@ -173,7 +286,7 @@ done:
*-------------------------------------------------------------------------
*/
static H5FS_section_info_t *
-H5MF_sect_simple_deserialize(const H5FS_section_class_t H5_ATTR_UNUSED *cls,
+H5MF_sect_deserialize(const H5FS_section_class_t *cls,
hid_t H5_ATTR_UNUSED dxpl_id, const uint8_t H5_ATTR_UNUSED *buf, haddr_t sect_addr,
hsize_t sect_size, unsigned H5_ATTR_UNUSED *des_flags)
{
@@ -183,11 +296,12 @@ H5MF_sect_simple_deserialize(const H5FS_section_class_t H5_ATTR_UNUSED *cls,
FUNC_ENTER_NOAPI_NOINIT
/* Check arguments. */
+ HDassert(cls);
HDassert(H5F_addr_defined(sect_addr));
HDassert(sect_size);
/* Create free space section for block */
- if(NULL == (sect = H5MF_sect_simple_new(sect_addr, sect_size)))
+ if(NULL == (sect = H5MF_sect_new(cls->type, sect_addr, sect_size)))
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't initialize free space section")
/* Set return value */
@@ -195,10 +309,80 @@ H5MF_sect_simple_deserialize(const H5FS_section_class_t H5_ATTR_UNUSED *cls,
done:
FUNC_LEAVE_NOAPI(ret_value)
-} /* H5MF_sect_simple_deserialize() */
+} /* H5MF_sect_deserialize() */
/*-------------------------------------------------------------------------
+ * Function: H5MF_sect_valid
+ *
+ * Purpose: Check the validity of a section
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ * Programmer: Quincey Koziol
+ * Tuesday, January 8, 2008
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5MF_sect_valid(const H5FS_section_class_t H5_ATTR_UNUSED *cls,
+ const H5FS_section_info_t
+#ifdef NDEBUG
+ H5_ATTR_UNUSED
+#endif /* NDEBUG */
+ *_sect, hid_t H5_ATTR_UNUSED dxpl_id)
+{
+#ifndef NDEBUG
+ const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */
+#endif /* NDEBUG */
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ /* Check arguments. */
+ HDassert(sect);
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5MF_sect_valid() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF_sect_split
+ *
+ * Purpose: Split SECT into 2 sections: fragment for alignment & the aligned section
+ * SECT's addr and size are updated to point to the aligned section
+ *
+ * Return: Success: the fragment for aligning sect
+ * Failure: null
+ *
+ * Programmer: Vailin Choi, July 29, 2008
+ *
+ *-------------------------------------------------------------------------
+ */
+static H5FS_section_info_t *
+H5MF_sect_split(H5FS_section_info_t *sect, hsize_t frag_size)
+{
+ H5MF_free_section_t *ret_value; /* Return value */
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ /* Allocate space for new section */
+ if(NULL == (ret_value = H5MF_sect_new(sect->type, sect->addr, frag_size)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't initialize free space section")
+
+ /* Set new section's info */
+ sect->addr += frag_size;
+ sect->size -= frag_size;
+
+done:
+ FUNC_LEAVE_NOAPI((H5FS_section_info_t *)ret_value)
+} /* end H5MF_sect_split() */
+
+/*
+ * "simple" section callbacks
+ */
+
+/*-------------------------------------------------------------------------
* Function: H5MF_sect_simple_can_merge
*
* Purpose: Can two sections of this type merge?
@@ -252,10 +436,10 @@ H5MF_sect_simple_can_merge(const H5FS_section_info_t *_sect1,
*-------------------------------------------------------------------------
*/
static herr_t
-H5MF_sect_simple_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2,
+H5MF_sect_simple_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2,
void H5_ATTR_UNUSED *_udata)
{
- H5MF_free_section_t *sect1 = (H5MF_free_section_t *)_sect1; /* File free section */
+ H5MF_free_section_t **sect1 = (H5MF_free_section_t **)_sect1; /* File free section */
H5MF_free_section_t *sect2 = (H5MF_free_section_t *)_sect2; /* File free section */
herr_t ret_value = SUCCEED; /* Return value */
@@ -263,16 +447,16 @@ H5MF_sect_simple_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2,
/* Check arguments. */
HDassert(sect1);
- HDassert(sect1->sect_info.type == H5MF_FSPACE_SECT_SIMPLE);
+ HDassert((*sect1)->sect_info.type == H5MF_FSPACE_SECT_SIMPLE);
HDassert(sect2);
HDassert(sect2->sect_info.type == H5MF_FSPACE_SECT_SIMPLE);
- HDassert(H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr));
+ HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr));
/* Add second section's size to first section */
- sect1->sect_info.size += sect2->sect_info.size;
+ (*sect1)->sect_info.size += sect2->sect_info.size;
/* Get rid of second section */
- if(H5MF_sect_simple_free((H5FS_section_info_t *)sect2) < 0)
+ if(H5MF_sect_free((H5FS_section_info_t *)sect2) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node")
done:
@@ -293,7 +477,7 @@ done:
*
*-------------------------------------------------------------------------
*/
-htri_t
+static htri_t
H5MF_sect_simple_can_shrink(const H5FS_section_info_t *_sect, void *_udata)
{
const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */
@@ -392,7 +576,7 @@ done:
*
*-------------------------------------------------------------------------
*/
-herr_t
+static herr_t
H5MF_sect_simple_shrink(H5FS_section_info_t **_sect, void *_udata)
{
H5MF_free_section_t **sect = (H5MF_free_section_t **)_sect; /* File free section */
@@ -411,8 +595,8 @@ H5MF_sect_simple_shrink(H5FS_section_info_t **_sect, void *_udata)
/* Sanity check */
HDassert(H5F_INTENT(udata->f) & H5F_ACC_RDWR);
- /* Release section's space at EOA with file driver */
- if(H5FD_free(udata->f->shared->lf, udata->dxpl_id, udata->alloc_type, udata->f, (*sect)->sect_info.addr, (*sect)->sect_info.size) < 0)
+ /* Release section's space at EOA */
+ if(H5F_free(udata->f, udata->dxpl_id, udata->alloc_type, (*sect)->sect_info.addr, (*sect)->sect_info.size) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "driver free request failed")
} /* end if */
else {
@@ -427,7 +611,7 @@ H5MF_sect_simple_shrink(H5FS_section_info_t **_sect, void *_udata)
/* Check for freeing section */
if(udata->shrink != H5MF_SHRINK_SECT_ABSORB_AGGR) {
/* Free section */
- if(H5MF_sect_simple_free((H5FS_section_info_t *)*sect) < 0)
+ if(H5MF_sect_free((H5FS_section_info_t *)*sect) < 0)
HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node")
/* Mark section as freed, for free space manager */
@@ -438,100 +622,474 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5MF_sect_simple_shrink() */
+/*
+ * "small" section callbacks
+ */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF_sect_small_add
+ *
+ * Purpose: Perform actions on a small "meta" action before adding it to the free space manager:
+ * 1) Drop the section if it is at page end and its size <= page end threshold
+ * 2) Adjust section size to include page end threshold if
+ * (section size + threshold) is at page end
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ * Programmer: Vailin Choi; Dec 2012
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5MF_sect_small_add(H5FS_section_info_t **_sect, unsigned *flags, void *_udata)
+{
+ H5MF_free_section_t **sect = (H5MF_free_section_t **)_sect; /* Fractal heap free section */
+ H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */
+ haddr_t sect_end;
+ hsize_t rem, prem;
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: Entering, section {%a, %Hu}\n", FUNC, (*sect)->sect_info.addr, (*sect)->sect_info.size);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+
+ /* Do not adjust the section raw data or global heap data */
+ if(udata->alloc_type == H5FD_MEM_DRAW || udata->alloc_type == H5FD_MEM_GHEAP)
+ HGOTO_DONE(ret_value);
+
+ sect_end = (*sect)->sect_info.addr + (*sect)->sect_info.size;
+ rem = sect_end % udata->f->shared->fs_page_size;
+ prem = udata->f->shared->fs_page_size - rem;
+
+ /* Drop the section if it is at page end and its size is <= pgend threshold */
+ if(!rem && (*sect)->sect_info.size <= H5F_PGEND_META_THRES(udata->f) && (*flags & H5FS_ADD_RETURNED_SPACE)) {
+ if(H5MF_sect_free((H5FS_section_info_t *)(*sect)) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node")
+ *sect = NULL;
+ *flags &= (unsigned)~H5FS_ADD_RETURNED_SPACE;
+ *flags |= H5FS_PAGE_END_NO_ADD;
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: section is dropped\n", FUNC);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+ } /* end if */
+ /* Adjust the section if it is not at page end but its size + pgend threshold is at page end */
+ else
+ if(prem <= H5F_PGEND_META_THRES(udata->f)) {
+ (*sect)->sect_info.size += prem;
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: section is adjusted {%a, %Hu}\n", FUNC, (*sect)->sect_info.addr, (*sect)->sect_info.size);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+ } /* end if */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5MF_sect_small_add() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF_sect_small_can_shrink
+ *
+ * Purpose: Can this section shrink the container?
+ *
+ * Note: A small section is allowed to shrink only at closing.
+ *
+ * Return: Success: non-negative (TRUE/FALSE)
+ * Failure: negative
+ *
+ * Programmer: Vailin Choi; Dec 2012
+ *
+ *-------------------------------------------------------------------------
+ */
+static htri_t
+H5MF_sect_small_can_shrink(const H5FS_section_info_t *_sect, void *_udata)
+{
+ const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */
+ H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */
+ haddr_t eoa; /* End of address space in the file */
+ haddr_t end; /* End of section to extend */
+ htri_t ret_value = FALSE; /* Return value */
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ /* Check arguments. */
+ HDassert(sect);
+ HDassert(udata);
+ HDassert(udata->f);
+
+ /* Retrieve the end of the file's address space */
+ if(HADDR_UNDEF == (eoa = H5FD_get_eoa(udata->f->shared->lf, udata->alloc_type)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed")
+
+ /* Compute address of end of section to check */
+ end = sect->sect_info.addr + sect->sect_info.size;
+
+ /* Check if the section is exactly at the end of the allocated space in the file */
+ if(H5F_addr_eq(end, eoa) && sect->sect_info.size == udata->f->shared->fs_page_size) {
+ udata->shrink = H5MF_SHRINK_EOA;
+
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: section {%a, %Hu}, shrinks file, eoa = %a\n", FUNC, sect->sect_info.addr, sect->sect_info.size, eoa);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+
+ /* Indicate shrinking can occur */
+ HGOTO_DONE(TRUE)
+ } /* end if */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5MF_sect_small_can_shrink() */
+
/*-------------------------------------------------------------------------
- * Function: H5MF_sect_simple_free
+ * Function: H5MF_sect_small_shrink
*
- * Purpose: Free a 'single' section node
+ * Purpose: Shrink container with section
*
* Return: Success: non-negative
- * Failure: negative
+ * Failure: negative
+ *
+ * Programmer: Vailin Choi; Dec 2012
*
- * Programmer: Quincey Koziol
- * Tuesday, January 8, 2008
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5MF_sect_small_shrink(H5FS_section_info_t **_sect, void *_udata)
+{
+ H5MF_free_section_t **sect = (H5MF_free_section_t **)_sect; /* File free section */
+ H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ /* Check arguments. */
+ HDassert(sect);
+ HDassert((*sect)->sect_info.type == H5MF_FSPACE_SECT_SMALL);
+ HDassert(udata);
+ HDassert(udata->f);
+ HDassert(udata->shrink == H5MF_SHRINK_EOA);
+ HDassert(H5F_INTENT(udata->f) & H5F_ACC_RDWR);
+
+ /* Release section's space at EOA */
+ if(H5F_free(udata->f, udata->dxpl_id, udata->alloc_type, (*sect)->sect_info.addr, (*sect)->sect_info.size) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "driver free request failed")
+
+ /* Free section */
+ if(H5MF_sect_free((H5FS_section_info_t *)*sect) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node")
+
+ /* Mark section as freed, for free space manager */
+ *sect = NULL;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5MF_sect_small_shrink() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF_sect_small_can_merge
+ *
+ * Purpose: Can two sections of this type merge?
+ *
+ * Note: Second section must be "after" first section
+ * The "merged" section cannot cross page boundary.
+ *
+ * Return: Success: non-negative (TRUE/FALSE)
+ * Failure: negative
+ *
+ * Programmer: Vailin Choi; Dec 2012
*
*-------------------------------------------------------------------------
*/
-herr_t
-H5MF_sect_simple_free(H5FS_section_info_t *_sect)
+static htri_t
+H5MF_sect_small_can_merge(const H5FS_section_info_t *_sect1,
+ const H5FS_section_info_t *_sect2, void *_udata)
{
- H5MF_free_section_t *sect = (H5MF_free_section_t *)_sect; /* File free section */
+ const H5MF_free_section_t *sect1 = (const H5MF_free_section_t *)_sect1; /* File free section */
+ const H5MF_free_section_t *sect2 = (const H5MF_free_section_t *)_sect2; /* File free section */
+ H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */
+ htri_t ret_value = FALSE; /* Return value */
FUNC_ENTER_NOAPI_NOINIT_NOERR
/* Check arguments. */
- HDassert(sect);
+ HDassert(sect1);
+ HDassert(sect2);
+ HDassert(sect1->sect_info.type == sect2->sect_info.type); /* Checks "MERGE_SYM" flag */
+ HDassert(H5F_addr_lt(sect1->sect_info.addr, sect2->sect_info.addr));
- /* Release the section */
- sect = H5FL_FREE(H5MF_free_section_t, sect);
+ /* Check if second section adjoins first section */
+ ret_value = H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr);
+ if(ret_value > 0)
+ /* If they are on different pages, couldn't merge */
+ if((sect1->sect_info.addr / udata->f->shared->fs_page_size) != (((sect2->sect_info.addr + sect2->sect_info.size - 1) / udata->f->shared->fs_page_size)))
+ ret_value = FALSE;
- FUNC_LEAVE_NOAPI(SUCCEED)
-} /* H5MF_sect_simple_free() */
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: Leaving: ret_value = %t\n", FUNC, ret_value);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5MF_sect_small_can_merge() */
/*-------------------------------------------------------------------------
- * Function: H5MF_sect_simple_valid
+ * Function: H5MF_sect_small_merge
*
- * Purpose: Check the validity of a section
+ * Purpose: Merge two sections of this type
+ *
+ * Note: Second section always merges into first node.
+ * If the size of the "merged" section is equal to file space page size,
+ * free the section.
*
* Return: Success: non-negative
* Failure: negative
*
- * Programmer: Quincey Koziol
- * Tuesday, January 8, 2008
+ * Programmer: Vailin Choi; Dec 2012
*
*-------------------------------------------------------------------------
*/
static herr_t
-H5MF_sect_simple_valid(const H5FS_section_class_t H5_ATTR_UNUSED *cls,
- const H5FS_section_info_t
-#ifdef NDEBUG
- H5_ATTR_UNUSED
-#endif /* NDEBUG */
- *_sect, hid_t H5_ATTR_UNUSED dxpl_id)
+H5MF_sect_small_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2,
+ void *_udata)
{
-#ifndef NDEBUG
- const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */
-#endif /* NDEBUG */
+ H5MF_free_section_t **sect1 = (H5MF_free_section_t **)_sect1; /* File free section */
+ H5MF_free_section_t *sect2 = (H5MF_free_section_t *)_sect2; /* File free section */
+ H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ /* Check arguments. */
+ HDassert(sect1);
+ HDassert((*sect1)->sect_info.type == H5MF_FSPACE_SECT_SMALL);
+ HDassert(sect2);
+ HDassert(sect2->sect_info.type == H5MF_FSPACE_SECT_SMALL);
+ HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr));
+
+ /* Add second section's size to first section */
+ (*sect1)->sect_info.size += sect2->sect_info.size;
+
+ if((*sect1)->sect_info.size == udata->f->shared->fs_page_size) {
+ if(H5MF_xfree(udata->f, udata->alloc_type, udata->dxpl_id, (*sect1)->sect_info.addr, (*sect1)->sect_info.size) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free merged section")
+
+ /* Need to free possible metadata page in the PB cache */
+ /* This is in response to the data corruption bug from fheap.c with page buffering + page strategy */
+ /* Note: Large metadata page bypasses the PB cache */
+ /* Note: Update of raw data page (large or small sized) is handled by the PB cache */
+ if(udata->f->shared->page_buf != NULL && udata->alloc_type != H5FD_MEM_DRAW)
+ if(H5PB_remove_entry(udata->f, (*sect1)->sect_info.addr) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free merged section")
+
+ if(H5MF_sect_free((H5FS_section_info_t *)(*sect1)) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node")
+ *sect1 = NULL;
+ } /* end if */
+
+ /* Get rid of second section */
+ if(H5MF_sect_free((H5FS_section_info_t *)sect2) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5MF_sect_small_merge() */
+
+/*
+ * "Large" section callbacks
+ */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF_sect_large_can_merge (same as H5MF_sect_simple_can_merge)
+ *
+ * Purpose: Can two sections of this type merge?
+ *
+ * Note: Second section must be "after" first section
+ *
+ * Return: Success: non-negative (TRUE/FALSE)
+ * Failure: negative
+ *
+ * Programmer: Vailin Choi; Dec 2012
+ *
+ *-------------------------------------------------------------------------
+ */
+static htri_t
+H5MF_sect_large_can_merge(const H5FS_section_info_t *_sect1,
+ const H5FS_section_info_t *_sect2, void H5_ATTR_UNUSED *_udata)
+{
+ const H5MF_free_section_t *sect1 = (const H5MF_free_section_t *)_sect1; /* File free section */
+ const H5MF_free_section_t *sect2 = (const H5MF_free_section_t *)_sect2; /* File free section */
+ htri_t ret_value = FALSE; /* Return value */
FUNC_ENTER_NOAPI_NOINIT_NOERR
/* Check arguments. */
+ HDassert(sect1);
+ HDassert(sect2);
+ HDassert(sect1->sect_info.type == sect2->sect_info.type); /* Checks "MERGE_SYM" flag */
+ HDassert(H5F_addr_lt(sect1->sect_info.addr, sect2->sect_info.addr));
+
+ ret_value = H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr);
+
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: Leaving: ret_value = %t\n", FUNC, ret_value);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5MF_sect_large_can_merge() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF_sect_large_merge (same as H5MF_sect_simple_merge)
+ *
+ * Purpose: Merge two sections of this type
+ *
+ * Note: Second section always merges into first node
+ *
+ * Return: Success: non-negative
+ * Failure: negative
+ *
+ * Programmer: Vailin Choi; Dec 2012
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5MF_sect_large_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2,
+ void H5_ATTR_UNUSED *_udata)
+{
+ H5MF_free_section_t **sect1 = (H5MF_free_section_t **)_sect1; /* File free section */
+ H5MF_free_section_t *sect2 = (H5MF_free_section_t *)_sect2; /* File free section */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ /* Check arguments. */
+ HDassert(sect1);
+ HDassert((*sect1)->sect_info.type == H5MF_FSPACE_SECT_LARGE);
+ HDassert(sect2);
+ HDassert(sect2->sect_info.type == H5MF_FSPACE_SECT_LARGE);
+ HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr));
+
+ /* Add second section's size to first section */
+ (*sect1)->sect_info.size += sect2->sect_info.size;
+
+ /* Get rid of second section */
+ if(H5MF_sect_free((H5FS_section_info_t *)sect2) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5MF_sect_large_merge() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5MF_sect_large_can_shrink
+ *
+ * Purpose: Can this section shrink the container?
+ *
+ * Return: Success: non-negative (TRUE/FALSE)
+ * Failure: negative
+ *
+ * Programmer: Vailin Choi; Dec 2012
+ *
+ *-------------------------------------------------------------------------
+ */
+static htri_t
+H5MF_sect_large_can_shrink(const H5FS_section_info_t *_sect, void *_udata)
+{
+ const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */
+ H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */
+ haddr_t eoa; /* End of address space in the file */
+ haddr_t end; /* End of section to extend */
+ htri_t ret_value = FALSE; /* Return value */
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ /* Check arguments. */
HDassert(sect);
+ HDassert(sect->sect_info.type == H5MF_FSPACE_SECT_LARGE);
+ HDassert(udata);
+ HDassert(udata->f);
- FUNC_LEAVE_NOAPI(SUCCEED)
-} /* H5MF_sect_simple_valid() */
+ /* Retrieve the end of the file's address space */
+ if(HADDR_UNDEF == (eoa = H5FD_get_eoa(udata->f->shared->lf, udata->alloc_type)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed")
+
+ /* Compute address of end of section to check */
+ end = sect->sect_info.addr + sect->sect_info.size;
+
+ /* Check if the section is exactly at the end of the allocated space in the file */
+ if(H5F_addr_eq(end, eoa) && sect->sect_info.size >= udata->f->shared->fs_page_size) {
+ /* Set the shrinking type */
+ udata->shrink = H5MF_SHRINK_EOA;
+#ifdef H5MF_ALLOC_DEBUG_MORE
+HDfprintf(stderr, "%s: section {%a, %Hu}, shrinks file, eoa = %a\n", FUNC, sect->sect_info.addr, sect->sect_info.size, eoa);
+#endif /* H5MF_ALLOC_DEBUG_MORE */
+ /* Indicate shrinking can occur */
+ HGOTO_DONE(TRUE)
+ } /* end if */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5MF_sect_large_can_shrink() */
+
/*-------------------------------------------------------------------------
- * Function: H5MF_sect_simple_split
+ * Function: H5MF_sect_large_shrink
*
- * Purpose: Split SECT into 2 sections: fragment for alignment & the aligned section
- * SECT's addr and size are updated to point to the aligned section
+ * Purpose: Shrink a large-sized section
*
- * Return: Success: the fragment for aligning sect
- * Failure: null
+ * Return: Success: non-negative
+ * Failure: negative
*
- * Programmer: Vailin Choi, July 29, 2008
+ * Programmer: Vailin Choi; Dec 2012
*
*-------------------------------------------------------------------------
*/
-static H5FS_section_info_t *
-H5MF_sect_simple_split(H5FS_section_info_t *sect, hsize_t frag_size)
+static herr_t
+H5MF_sect_large_shrink(H5FS_section_info_t **_sect, void *_udata)
{
- H5MF_free_section_t *ret_value = NULL; /* Return value */
+ H5MF_free_section_t **sect = (H5MF_free_section_t **)_sect; /* File free section */
+ H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */
+ hsize_t frag_size = 0; /* Fragment size */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT
- /* Allocate space for new section */
- if(NULL == (ret_value = H5MF_sect_simple_new(sect->addr, frag_size)))
- HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't initialize free space section")
+ /* Check arguments. */
+ HDassert(sect);
+ HDassert((*sect)->sect_info.type == H5MF_FSPACE_SECT_LARGE);
+ HDassert(udata);
+ HDassert(udata->f);
+ HDassert(udata->shrink == H5MF_SHRINK_EOA);
+ HDassert(H5F_INTENT(udata->f) & H5F_ACC_RDWR);
+ HDassert(H5F_PAGED_AGGR(udata->f));
- /* Set new section's info */
- sect->addr += frag_size;
- sect->size -= frag_size;
+ /* Calculate possible mis-aligned fragment */
+ H5MF_EOA_MISALIGN(udata->f, (*sect)->sect_info.addr, udata->f->shared->fs_page_size, frag_size);
+
+ /* Free full pages from EOA */
+ /* Retain partial page in the free-space manager so as to keep EOA at page boundary */
+ if(H5F_free(udata->f, udata->dxpl_id, udata->alloc_type, (*sect)->sect_info.addr+frag_size, (*sect)->sect_info.size-frag_size) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "driver free request failed")
+
+ if(frag_size) /* Adjust section size for the partial page */
+ (*sect)->sect_info.size = frag_size;
+ else {
+ /* Free section */
+ if(H5MF_sect_free((H5FS_section_info_t *)*sect) < 0)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node")
+
+ /* Mark section as freed, for free space manager */
+ *sect = NULL;
+ } /* end else */
done:
- FUNC_LEAVE_NOAPI((H5FS_section_info_t *)ret_value)
-} /* end H5MF_sect_simple_split() */
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5MF_sect_large_shrink() */
diff --git a/src/H5Oalloc.c b/src/H5Oalloc.c
index 4f98cfa..40c9c38 100644
--- a/src/H5Oalloc.c
+++ b/src/H5Oalloc.c
@@ -2023,6 +2023,7 @@ H5O_merge_null(H5F_t *f, hid_t dxpl_id, H5O_t *oh)
/* Second message has been merged, delete it */
if(merged_msg) {
H5O_chunk_proxy_t *curr_chk_proxy; /* Chunk that message is in */
+ htri_t result;
/* Release any information/memory for second message */
H5O_msg_free_mesg(curr_msg2);
@@ -2050,6 +2051,13 @@ H5O_merge_null(H5F_t *f, hid_t dxpl_id, H5O_t *oh)
/* (Don't bother reducing size of message array for now -QAK) */
oh->nmesgs--;
+ /* The merge null message might span the entire chunk: scan for empty chunk to remove */
+ if((result = H5O_remove_empty_chunks(f, dxpl_id, oh)) < 0)
+ HGOTO_ERROR(H5E_OHDR, H5E_CANTPACK, FAIL, "can't remove empty chunk")
+ else if(result > 0)
+ /* Get out of loop */
+ break;
+
/* If the merged message is too large, shrink the chunk */
if(curr_msg->raw_size >= H5O_MESG_MAX_SIZE)
if(H5O_alloc_shrink_chunk(f, dxpl_id, oh, curr_msg->chunkno) < 0)
diff --git a/src/H5Ocache_image.c b/src/H5Ocache_image.c
index 65f6aa2..bea99a2 100644
--- a/src/H5Ocache_image.c
+++ b/src/H5Ocache_image.c
@@ -33,6 +33,7 @@
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
+#include "H5Fprivate.h" /* Files */
#include "H5FLprivate.h" /* Free Lists */
#include "H5Opkg.h" /* Object headers */
#include "H5MFprivate.h" /* File space management */
@@ -290,9 +291,50 @@ H5O__mdci_delete(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, void *_mesg)
HDassert(mesg);
/* Free file space for cache image */
- if(H5F_addr_defined(mesg->addr))
- if(H5MF_xfree(f, H5FD_MEM_SUPER, dxpl_id, mesg->addr, mesg->size) < 0)
- HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to free file space for cache image block")
+ if(H5F_addr_defined(mesg->addr)) {
+ /* The space for the cache image block was allocated directly
+ * from the VFD layer at the end of file. As this was the
+ * last file space allocation before shutdown, the cache image
+ * should still be the last item in the file.
+ *
+ * If the hack to work around the self referential free space
+ * manager issue is in use, file space for the non-empty self
+ * referential free space managers was also allocated from VFD
+ * layer at the end of file. Since these allocations directly
+ * preceeded the cache image allocation they should be directly
+ * adjacent to the cache image block at the end of file.
+ *
+ * In this case, just call H5MF_tidy_self_referential_fsm_hack().
+ *
+ * That routine will float the self referential free space
+ * managers, and reduce the eoa to its value just prior to
+ * allocation of space for same. Since the cache image appears
+ * just after the self referential free space managers, this
+ * will release the file space for the cache image as well.
+ *
+ * Note that in this case, there must not have been any file
+ * space allocations / deallocations prior to the free of the
+ * cache image. Verify this to the extent possible.
+ *
+ * If the hack to work around the persistant self referential
+ * free space manager issue is NOT in use, just call H5MF_xfree()
+ * to release the cache iamge. In principle, we should be able
+ * to just reduce the EOA to the base address of the cache
+ * image block, as there shouldn't be any file space allocation
+ * before the first metadata cache access. However, given
+ * time constraints, I don't want to go there now.
+ */
+ if(H5F_FIRST_ALLOC_DEALLOC(f)) {
+ HDassert(HADDR_UNDEF !=H5F_EOA_PRE_FSM_FSALLOC(f));
+ HDassert(H5F_addr_ge(mesg->addr, H5F_EOA_PRE_FSM_FSALLOC(f)));
+ if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0)
+ HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed")
+ } /* end if */
+ else {
+ if(H5MF_xfree(f, H5FD_MEM_SUPER, dxpl_id, mesg->addr, mesg->size) < 0)
+ HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to free file space for cache image block")
+ } /* end else */
+ } /* end if */
done:
FUNC_LEAVE_NOAPI(ret_value)
diff --git a/src/H5Ofsinfo.c b/src/H5Ofsinfo.c
index 938d319..37ec627 100644
--- a/src/H5Ofsinfo.c
+++ b/src/H5Ofsinfo.c
@@ -23,12 +23,13 @@
*
*-------------------------------------------------------------------------
*/
-
+#define H5F_FRIEND /*suppress error about including H5Fpkg */
#include "H5Omodule.h" /* This source code file is part of the H5O module */
#include "H5private.h" /* Generic Functions */
#include "H5Eprivate.h" /* Error handling */
+#include "H5Fpkg.h" /* File access */
#include "H5FLprivate.h" /* Free lists */
#include "H5Opkg.h" /* Object headers */
@@ -66,7 +67,8 @@ const H5O_msg_class_t H5O_MSG_FSINFO[1] = {{
}};
/* Current version of free-space manager info information */
-#define H5O_FSINFO_VERSION 0
+#define H5O_FSINFO_VERSION_0 0
+#define H5O_FSINFO_VERSION_1 1
/* Declare a free list to manage the H5O_fsinfo_t struct */
H5FL_DEFINE_STATIC(H5O_fsinfo_t);
@@ -85,12 +87,13 @@ H5FL_DEFINE_STATIC(H5O_fsinfo_t);
*-------------------------------------------------------------------------
*/
static void *
-H5O_fsinfo_decode(H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5_ATTR_UNUSED *open_oh,
+H5O_fsinfo_decode(H5F_t *f, hid_t dxpl_id, H5O_t H5_ATTR_UNUSED *open_oh,
unsigned H5_ATTR_UNUSED mesg_flags, unsigned H5_ATTR_UNUSED *ioflags, const uint8_t *p)
{
- H5O_fsinfo_t *fsinfo = NULL; /* free-space manager info */
- H5FD_mem_t type; /* Memory type for iteration */
- void *ret_value = NULL; /* Return value */
+ H5O_fsinfo_t *fsinfo = NULL; /* File space info message */
+ H5F_mem_page_t ptype; /* Memory type for iteration */
+ unsigned vers; /* message version */
+ void *ret_value = NULL; /* Return value */
FUNC_ENTER_NOAPI_NOINIT
@@ -98,26 +101,83 @@ H5O_fsinfo_decode(H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5_ATTR_UNUSED *
HDassert(f);
HDassert(p);
- /* Version of message */
- if(*p++ != H5O_FSINFO_VERSION)
- HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "bad version number for message")
-
/* Allocate space for message */
if(NULL == (fsinfo = H5FL_CALLOC(H5O_fsinfo_t)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
- fsinfo->strategy = (H5F_file_space_type_t)*p++; /* file space strategy */
- H5F_DECODE_LENGTH(f, p, fsinfo->threshold); /* free space section size threshold */
+ for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype))
+ fsinfo->fs_addr[ptype - 1] = HADDR_UNDEF;
- /* Addresses of free space managers: only exist for H5F_FILE_SPACE_ALL_PERSIST */
- if(fsinfo->strategy == H5F_FILE_SPACE_ALL_PERSIST) {
- for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
- H5F_addr_decode(f, &p, &(fsinfo->fs_addr[type-1]));
- } /* end if */
- else {
- for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
- fsinfo->fs_addr[type-1] = HADDR_UNDEF;
- } /* end else */
+ /* Version of message */
+ vers = *p++;
+
+ if(vers == H5O_FSINFO_VERSION_0) {
+ H5F_file_space_type_t strategy; /* Strategy */
+ hsize_t threshold; /* Threshold */
+ H5FD_mem_t type; /* Memory type for iteration */
+
+ fsinfo->persist = H5F_FREE_SPACE_PERSIST_DEF;
+ fsinfo->threshold = H5F_FREE_SPACE_THRESHOLD_DEF;
+ fsinfo->page_size = H5F_FILE_SPACE_PAGE_SIZE_DEF;
+ fsinfo->pgend_meta_thres = H5F_FILE_SPACE_PGEND_META_THRES;
+ fsinfo->eoa_pre_fsm_fsalloc = HADDR_UNDEF;
+
+ strategy = (H5F_file_space_type_t)*p++; /* File space strategy */
+ H5F_DECODE_LENGTH(f, p, threshold); /* Free-space section threshold */
+
+ /* Map version 0 (deprecated) to version 1 message */
+ switch(strategy) {
+
+ case H5F_FILE_SPACE_ALL_PERSIST:
+ fsinfo->strategy = H5F_FSPACE_STRATEGY_FSM_AGGR;
+ fsinfo->persist = TRUE;
+ fsinfo->threshold = threshold;
+ if(HADDR_UNDEF == (fsinfo->eoa_pre_fsm_fsalloc = H5F_get_eoa(f, H5FD_MEM_DEFAULT)) )
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to get file size")
+ for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
+ H5F_addr_decode(f, &p, &(fsinfo->fs_addr[type-1]));
+ break;
+
+ case H5F_FILE_SPACE_ALL:
+ fsinfo->strategy = H5F_FSPACE_STRATEGY_FSM_AGGR;
+ fsinfo->threshold = threshold;
+ break;
+
+ case H5F_FILE_SPACE_AGGR_VFD:
+ fsinfo->strategy = H5F_FSPACE_STRATEGY_AGGR;
+ break;
+
+ case H5F_FILE_SPACE_VFD:
+ fsinfo->strategy = H5F_FSPACE_STRATEGY_NONE;
+ break;
+
+ case H5F_FILE_SPACE_NTYPES:
+ case H5F_FILE_SPACE_DEFAULT:
+ default:
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file space strategy")
+ } /* end switch */
+
+ fsinfo->mapped = TRUE;
+
+ } else {
+ HDassert(vers == H5O_FSINFO_VERSION_1);
+
+ fsinfo->strategy = (H5F_fspace_strategy_t)*p++; /* File space strategy */
+ fsinfo->persist = *p++; /* Free-space persist or not */
+ H5F_DECODE_LENGTH(f, p, fsinfo->threshold); /* Free-space section threshold */
+
+ H5F_DECODE_LENGTH(f, p, fsinfo->page_size); /* File space page size */
+ UINT16DECODE(p, fsinfo->pgend_meta_thres); /* Page end metdata threshold */
+ H5F_addr_decode(f, &p, &(fsinfo->eoa_pre_fsm_fsalloc)); /* EOA before free-space header and section info */
+
+ /* Decode addresses of free space managers, if persisting */
+ if(fsinfo->persist) {
+ for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype))
+ H5F_addr_decode(f, &p, &(fsinfo->fs_addr[ptype - 1]));
+ } /* end if */
+
+ fsinfo->mapped = FALSE;
+ }
/* Set return value */
ret_value = fsinfo;
@@ -145,7 +205,7 @@ static herr_t
H5O_fsinfo_encode(H5F_t *f, hbool_t H5_ATTR_UNUSED disable_shared, uint8_t *p, const void *_mesg)
{
const H5O_fsinfo_t *fsinfo = (const H5O_fsinfo_t *)_mesg;
- H5FD_mem_t type; /* Memory type for iteration */
+ H5F_mem_page_t ptype; /* Memory type for iteration */
FUNC_ENTER_NOAPI_NOINIT_NOERR
@@ -154,14 +214,20 @@ H5O_fsinfo_encode(H5F_t *f, hbool_t H5_ATTR_UNUSED disable_shared, uint8_t *p, c
HDassert(p);
HDassert(fsinfo);
- *p++ = H5O_FSINFO_VERSION; /* message version */
- *p++ = fsinfo->strategy; /* file space strategy */
- H5F_ENCODE_LENGTH(f, p, fsinfo->threshold); /* free-space section size threshold */
+ *p++ = H5O_FSINFO_VERSION_1; /* message version */
+ *p++ = fsinfo->strategy; /* File space strategy */
+ *p++ = (unsigned char)fsinfo->persist; /* Free-space persist or not */
+ H5F_ENCODE_LENGTH(f, p, fsinfo->threshold); /* Free-space section size threshold */
+
+ H5F_ENCODE_LENGTH(f, p, fsinfo->page_size); /* File space page size */
+ UINT16ENCODE(p, fsinfo->pgend_meta_thres); /* Page end metadata threshold */
+ H5F_addr_encode(f, &p, fsinfo->eoa_pre_fsm_fsalloc); /* EOA before free-space header and section info */
- /* Addresses of free space managers: only exist for H5F_FILE_SPACE_ALL_PERSIST */
- if(fsinfo->strategy == H5F_FILE_SPACE_ALL_PERSIST) {
- for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
- H5F_addr_encode(f, &p, fsinfo->fs_addr[type-1]);
+ /* Store addresses of free-space managers, if persisting */
+ if(fsinfo->persist) {
+ /* Addresses of free-space managers */
+ for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype))
+ H5F_addr_encode(f, &p, fsinfo->fs_addr[ptype - 1]);
} /* end if */
FUNC_LEAVE_NOAPI(SUCCEED)
@@ -224,19 +290,19 @@ static size_t
H5O_fsinfo_size(const H5F_t *f, hbool_t H5_ATTR_UNUSED disable_shared, const void *_mesg)
{
const H5O_fsinfo_t *fsinfo = (const H5O_fsinfo_t *)_mesg;
- size_t fs_addr_size = 0;
size_t ret_value = 0; /* Return value */
FUNC_ENTER_NOAPI_NOINIT_NOERR
-
- /* Addresses of free-space managers exist only for H5F_FILE_SPACE_ALL_PERSIST type */
- if(H5F_FILE_SPACE_ALL_PERSIST == fsinfo->strategy)
- fs_addr_size = (H5FD_MEM_NTYPES - 1) * (size_t)H5F_SIZEOF_ADDR(f);
-
- ret_value = 2 /* Version & strategy */
- + (size_t)H5F_SIZEOF_SIZE(f) /* Threshold */
- + fs_addr_size; /* Addresses of free-space managers */
+ ret_value = 3 /* Version, strategy & persist */
+ + (size_t)H5F_SIZEOF_SIZE(f) /* Free-space section threshold */
+ + (size_t)H5F_SIZEOF_SIZE(f) /* File space page size */
+ + 2 /* Page end meta threshold */
+ + (size_t)H5F_SIZEOF_ADDR(f);
+
+ /* Free-space manager addresses */
+ if(fsinfo->persist)
+ ret_value += (H5F_MEM_PAGE_NTYPES - 1) * (size_t)H5F_SIZEOF_ADDR(f);
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5O_fsinfo_size() */
@@ -282,7 +348,7 @@ H5O_fsinfo_debug(H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id, const vo
int indent, int fwidth)
{
const H5O_fsinfo_t *fsinfo = (const H5O_fsinfo_t *) _mesg;
- H5FD_mem_t type; /* Memory type for iteration */
+ H5F_mem_page_t ptype; /* Free-space types for iteration */
FUNC_ENTER_NOAPI_NOINIT_NOERR
@@ -293,16 +359,48 @@ H5O_fsinfo_debug(H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id, const vo
HDassert(indent >= 0);
HDassert(fwidth >= 0);
- HDfprintf(stream, "%*s%-*s %u\n", indent, "", fwidth,
- "File space strategy:", fsinfo->strategy);
+ HDfprintf(stream, "%*s%-*s ", indent, "", fwidth, "File space strategy:");
+ switch(fsinfo->strategy) {
+ case H5F_FSPACE_STRATEGY_FSM_AGGR:
+ HDfprintf(stream, "%s\n", "H5F_FSPACE_STRATEGY_FSM_AGGR");
+ break;
+
+ case H5F_FSPACE_STRATEGY_PAGE:
+ HDfprintf(stream, "%s\n", "H5F_FSPACE_STRATEGY_PAGE");
+ break;
+
+ case H5F_FSPACE_STRATEGY_AGGR:
+ HDfprintf(stream, "%s\n", "H5F_FSPACE_STRATEGY_AGGR");
+ break;
+
+ case H5F_FSPACE_STRATEGY_NONE:
+ HDfprintf(stream, "%s\n", "H5F_FSPACE_STRATEGY_NONE");
+ break;
+
+ case H5F_FSPACE_STRATEGY_NTYPES:
+ default:
+ HDfprintf(stream, "%s\n", "unknown");
+ } /* end switch */
+
+ HDfprintf(stream, "%*s%-*s %t\n", indent, "", fwidth,
+ "Free-space persist:", fsinfo->persist);
HDfprintf(stream, "%*s%-*s %Hu\n", indent, "", fwidth,
- "Free space section threshold:", fsinfo->threshold);
+ "Free-space section threshold:", fsinfo->threshold);
+
+ HDfprintf(stream, "%*s%-*s %Hu\n", indent, "", fwidth,
+ "File space page size:", fsinfo->page_size);
+
+ HDfprintf(stream, "%*s%-*s %u\n", indent, "", fwidth,
+ "Page end metadata threshold:", fsinfo->pgend_meta_thres);
+
+ HDfprintf(stream, "%*s%-*s %a\n", indent, "", fwidth,
+ "eoa_pre_fsm_fsalloc:", fsinfo->eoa_pre_fsm_fsalloc);
- if(fsinfo->strategy == H5F_FILE_SPACE_ALL_PERSIST) {
- for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type))
- HDfprintf(stream, "%*s%-*s %a\n", indent, "", fwidth,
- "Free space manager address:", fsinfo->fs_addr[type-1]);
+ if(fsinfo->persist) {
+ for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype))
+ HDfprintf(stream, "%*s%-*s %a\n", indent, "", fwidth,
+ "Free space manager address:", fsinfo->fs_addr[ptype-1]);
} /* end if */
FUNC_LEAVE_NOAPI(SUCCEED)
diff --git a/src/H5Omessage.c b/src/H5Omessage.c
index 376c888..7e6463a 100644
--- a/src/H5Omessage.c
+++ b/src/H5Omessage.c
@@ -2253,3 +2253,55 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5O_flush_msgs() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5O_msg_get_flags
+ *
+ * Purpose: Queries a message's message flags in the object header
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Vailin; Jan 2013
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5O_msg_get_flags(const H5O_loc_t *loc, unsigned type_id, hid_t dxpl_id, uint8_t *flags)
+{
+ H5O_t *oh = NULL; /* Object header to use */
+ const H5O_msg_class_t *type; /* Actual H5O class type for the ID */
+ H5O_mesg_t *idx_msg; /* Pointer to message to modify */
+ unsigned idx; /* Index of message to modify */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* check args */
+ HDassert(loc);
+ HDassert(loc->file);
+ HDassert(H5F_addr_defined(loc->addr));
+ HDassert(type_id < NELMTS(H5O_msg_class_g));
+ type = H5O_msg_class_g[type_id]; /* map the type ID to the actual type object */
+ HDassert(type);
+
+ /* Get the object header */
+ if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE)))
+ HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to protect object header")
+
+ /* Locate message of correct type */
+ for(idx = 0, idx_msg = &oh->mesg[0]; idx < oh->nmesgs; idx++, idx_msg++)
+ if(type == idx_msg->type)
+ break;
+
+ if(idx == oh->nmesgs)
+ HGOTO_ERROR(H5E_OHDR, H5E_NOTFOUND, FAIL, "message type not found")
+
+ /* Set return value */
+ *flags = idx_msg->flags;
+
+done:
+ if(oh && H5O_unprotect(loc, dxpl_id, oh, H5AC__NO_FLAGS_SET) < 0)
+ HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to release object header")
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5O_msg_get_flags() */
diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h
index 71c512c..f0fbe72 100644
--- a/src/H5Oprivate.h
+++ b/src/H5Oprivate.h
@@ -791,9 +791,16 @@ typedef unsigned H5O_unknown_t; /* Original message type ID */
* (Data structure in memory)
*/
typedef struct H5O_fsinfo_t {
- H5F_file_space_type_t strategy; /* File space strategy */
- hsize_t threshold; /* Free space section threshold */
- haddr_t fs_addr[H5FD_MEM_NTYPES-1]; /* Addresses of free space managers */
+ H5F_fspace_strategy_t strategy; /* File space strategy */
+ hbool_t persist; /* Persisting free-space or not */
+ hsize_t threshold; /* Free-space section threshold */
+ hsize_t page_size; /* For paged aggregation: file space page size */
+ size_t pgend_meta_thres; /* For paged aggregation: page end metadata threshold */
+ haddr_t eoa_pre_fsm_fsalloc; /* For paged aggregation: the eoa before free-space headers & sinfo */
+ haddr_t fs_addr[H5F_MEM_PAGE_NTYPES - 1]; /* 13 addresses of free-space managers */
+ /* For non-paged aggregation: only 6 addresses are used */
+ hbool_t mapped; /* Not stored */
+ /* Indicate the message is mapped from version 0 to version 1 */
} H5O_fsinfo_t;
/*
@@ -920,6 +927,7 @@ H5_DLL void* H5O_msg_decode(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh,
unsigned type_id, const unsigned char *buf);
H5_DLL herr_t H5O_msg_delete(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh,
unsigned type_id, void *mesg);
+H5_DLL herr_t H5O_msg_get_flags(const H5O_loc_t *loc, unsigned type_id, hid_t dxpl_id, uint8_t *flags);
/* Object metadata flush/refresh routines */
H5_DLL herr_t H5O_flush_common(H5O_loc_t *oloc, hid_t obj_id, hid_t dxpl_id);
diff --git a/src/H5PB.c b/src/H5PB.c
new file mode 100644
index 0000000..c67ae59
--- /dev/null
+++ b/src/H5PB.c
@@ -0,0 +1,1533 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the files COPYING and Copyright.html. COPYING can be found at the root *
+ * of the source code distribution tree; Copyright.html can be found at the *
+ * root level of an installed copy of the electronic HDF5 document set and *
+ * is linked from the top-level documents page. It can also be found at *
+ * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have *
+ * access to either file, you may request a copy from help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*-------------------------------------------------------------------------
+ *
+ * Created: H5PB.c
+ *
+ * Purpose: Page Buffer routines.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+/****************/
+/* Module Setup */
+/****************/
+
+#define H5F_FRIEND /*suppress error about including H5Fpkg */
+#include "H5PBmodule.h" /* This source code file is part of the H5PB module */
+
+
+/***********/
+/* Headers */
+/***********/
+#include "H5private.h" /* Generic Functions */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5Fpkg.h" /* Files */
+#include "H5FDprivate.h" /* File drivers */
+#include "H5Iprivate.h" /* IDs */
+#include "H5PBpkg.h" /* File access */
+#include "H5SLprivate.h" /* Skip List */
+
+
+/****************/
+/* Local Macros */
+/****************/
+#define H5PB__PREPEND(page_ptr, head_ptr, tail_ptr, len) { \
+ if((head_ptr) == NULL) { \
+ (head_ptr) = (page_ptr); \
+ (tail_ptr) = (page_ptr); \
+ } /* end if */ \
+ else { \
+ (head_ptr)->prev = (page_ptr); \
+ (page_ptr)->next = (head_ptr); \
+ (head_ptr) = (page_ptr); \
+ } /* end else */ \
+ (len)++; \
+} /* H5PB__PREPEND() */
+
+#define H5PB__REMOVE(page_ptr, head_ptr, tail_ptr, len) { \
+ if((head_ptr) == (page_ptr)) { \
+ (head_ptr) = (page_ptr)->next; \
+ if((head_ptr) != NULL) \
+ (head_ptr)->prev = NULL; \
+ } /* end if */ \
+ else \
+ (page_ptr)->prev->next = (page_ptr)->next; \
+ if((tail_ptr) == (page_ptr)) { \
+ (tail_ptr) = (page_ptr)->prev; \
+ if((tail_ptr) != NULL) \
+ (tail_ptr)->next = NULL; \
+ } /* end if */ \
+ else \
+ (page_ptr)->next->prev = (page_ptr)->prev; \
+ page_ptr->next = NULL; \
+ page_ptr->prev = NULL; \
+ (len)--; \
+}
+
+#define H5PB__INSERT_LRU(page_buf, page_ptr) { \
+ HDassert(page_buf); \
+ HDassert(page_ptr); \
+ /* insert the entry at the head of the list. */ \
+ H5PB__PREPEND((page_ptr), (page_buf)->LRU_head_ptr, \
+ (page_buf)->LRU_tail_ptr, (page_buf)->LRU_list_len) \
+}
+
+#define H5PB__REMOVE_LRU(page_buf, page_ptr) { \
+ HDassert(page_buf); \
+ HDassert(page_ptr); \
+ /* remove the entry from the list. */ \
+ H5PB__REMOVE((page_ptr), (page_buf)->LRU_head_ptr, \
+ (page_buf)->LRU_tail_ptr, (page_buf)->LRU_list_len) \
+}
+
+#define H5PB__MOVE_TO_TOP_LRU(page_buf, page_ptr) { \
+ HDassert(page_buf); \
+ HDassert(page_ptr); \
+ /* Remove entry and insert at the head of the list. */ \
+ H5PB__REMOVE((page_ptr), (page_buf)->LRU_head_ptr, \
+ (page_buf)->LRU_tail_ptr, (page_buf)->LRU_list_len) \
+ H5PB__PREPEND((page_ptr), (page_buf)->LRU_head_ptr, \
+ (page_buf)->LRU_tail_ptr, (page_buf)->LRU_list_len) \
+}
+
+
+/******************/
+/* Local Typedefs */
+/******************/
+
+/* Iteration context for destroying page buffer */
+typedef struct {
+ H5PB_t *page_buf;
+ hbool_t actual_slist;
+} H5PB_ud1_t;
+
+
+/********************/
+/* Package Typedefs */
+/********************/
+
+
+/********************/
+/* Local Prototypes */
+/********************/
+static herr_t H5PB__insert_entry(H5PB_t *page_buf, H5PB_entry_t *page_entry);
+static htri_t H5PB__make_space(const H5F_io_info2_t *fio_info, H5PB_t *page_buf, H5FD_mem_t inserted_type);
+static herr_t H5PB__write_entry(const H5F_io_info2_t *fio_info, H5PB_entry_t *page_entry);
+
+
+/*********************/
+/* Package Variables */
+/*********************/
+
+/* Package initialization variable */
+hbool_t H5_PKG_INIT_VAR = FALSE;
+
+
+/*****************************/
+/* Library Private Variables */
+/*****************************/
+
+
+/*******************/
+/* Local Variables */
+/*******************/
+/* Declare a free list to manage the H5PB_t struct */
+H5FL_DEFINE_STATIC(H5PB_t);
+
+/* Declare a free list to manage the H5PB_entry_t struct */
+H5FL_DEFINE_STATIC(H5PB_entry_t);
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PB_reset_stats
+ *
+ * Purpose: This function was created without documentation.
+ * What follows is my best understanding of Mohamad's intent.
+ *
+ * Reset statistics collected for the page buffer layer.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Mohamad Chaarawi
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PB_reset_stats(H5PB_t *page_buf)
+{
+ FUNC_ENTER_NOAPI_NOERR
+
+ /* Sanity checks */
+ HDassert(page_buf);
+
+ page_buf->accesses[0] = 0;
+ page_buf->accesses[1] = 0;
+ page_buf->hits[0] = 0;
+ page_buf->hits[1] = 0;
+ page_buf->misses[0] = 0;
+ page_buf->misses[1] = 0;
+ page_buf->evictions[0] = 0;
+ page_buf->evictions[1] = 0;
+ page_buf->bypasses[0] = 0;
+ page_buf->bypasses[1] = 0;
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5PB_reset_stats() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PB_get_stats
+ *
+ * Purpose: This function was created without documentation.
+ * What follows is my best understanding of Mohamad's intent.
+ *
+ * Retrieve statistics collected about page accesses for the page buffer layer.
+ * --accesses: the number of metadata and raw data accesses to the page buffer layer
+ * --hits: the number of metadata and raw data hits in the page buffer layer
+ * --misses: the number of metadata and raw data misses in the page buffer layer
+ * --evictions: the number of metadata and raw data evictions from the page buffer layer
+ * --bypasses: the number of metadata and raw data accesses that bypass the page buffer layer
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Mohamad Chaarawi
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PB_get_stats(const H5PB_t *page_buf, unsigned accesses[2], unsigned hits[2],
+ unsigned misses[2], unsigned evictions[2], unsigned bypasses[2])
+{
+ FUNC_ENTER_NOAPI_NOERR
+
+ /* Sanity checks */
+ HDassert(page_buf);
+
+ accesses[0] = page_buf->accesses[0];
+ accesses[1] = page_buf->accesses[1];
+ hits[0] = page_buf->hits[0];
+ hits[1] = page_buf->hits[1];
+ misses[0] = page_buf->misses[0];
+ misses[1] = page_buf->misses[1];
+ evictions[0] = page_buf->evictions[0];
+ evictions[1] = page_buf->evictions[1];
+ bypasses[0] = page_buf->bypasses[0];
+ bypasses[1] = page_buf->bypasses[1];
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5PB_get_stats */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PB_print_stats()
+ *
+ * Purpose: This function was created without documentation.
+ * What follows is my best understanding of Mohamad's intent.
+ *
+ * Print out statistics collected for the page buffer layer.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Mohamad Chaarawi
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PB_print_stats(const H5PB_t *page_buf)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ HDassert(page_buf);
+
+ printf("PAGE BUFFER STATISTICS:\n");
+
+ printf("******* METADATA\n");
+ printf("\t Total Accesses: %u\n", page_buf->accesses[0]);
+ printf("\t Hits: %u\n", page_buf->hits[0]);
+ printf("\t Misses: %u\n", page_buf->misses[0]);
+ printf("\t Evictions: %u\n", page_buf->evictions[0]);
+ printf("\t Bypasses: %u\n", page_buf->bypasses[0]);
+ printf("\t Hit Rate = %f%%\n", ((double)page_buf->hits[0]/(page_buf->accesses[0] - page_buf->bypasses[0]))*100);
+ printf("*****************\n\n");
+
+ printf("******* RAWDATA\n");
+ printf("\t Total Accesses: %u\n", page_buf->accesses[1]);
+ printf("\t Hits: %u\n", page_buf->hits[1]);
+ printf("\t Misses: %u\n", page_buf->misses[1]);
+ printf("\t Evictions: %u\n", page_buf->evictions[1]);
+ printf("\t Bypasses: %u\n", page_buf->bypasses[1]);
+ printf("\t Hit Rate = %f%%\n", ((double)page_buf->hits[1]/(page_buf->accesses[1]-page_buf->bypasses[0]))*100);
+ printf("*****************\n\n");
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5PB_print_stats */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PB_create
+ *
+ * Purpose: Create and setup the PB on the file.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Mohamad Chaarawi
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PB_create(H5F_t *f, size_t size, unsigned page_buf_min_meta_perc, unsigned page_buf_min_raw_perc)
+{
+ H5PB_t *page_buf = NULL;
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* Sanity checks */
+ HDassert(f);
+ HDassert(f->shared);
+
+ /* Check args */
+ if(f->shared->fs_strategy != H5F_FSPACE_STRATEGY_PAGE)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "Enabling Page Buffering requires PAGE file space strategy")
+ /* round down the size if it is larger than the page size */
+ else if(size > f->shared->fs_page_size) {
+ hsize_t temp_size;
+
+ temp_size = (size / f->shared->fs_page_size) * f->shared->fs_page_size;
+ H5_CHECKED_ASSIGN(size, size_t, temp_size, hsize_t);
+ } /* end if */
+ else if(0 != size % f->shared->fs_page_size)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTINIT, FAIL, "Page Buffer size must be >= to the page size")
+
+ /* Allocate the new page buffering structure */
+ if(NULL == (page_buf = H5FL_CALLOC(H5PB_t)))
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "memory allocation failed")
+
+ page_buf->max_size = size;
+ H5_CHECKED_ASSIGN(page_buf->page_size, size_t, f->shared->fs_page_size, hsize_t);
+ page_buf->min_meta_perc = page_buf_min_meta_perc;
+ page_buf->min_raw_perc = page_buf_min_raw_perc;
+
+ /* Calculate the minimum page count for metadata and raw data
+ * based on the fractions provided
+ */
+ page_buf->min_meta_count = (unsigned)((size * page_buf_min_meta_perc) / (f->shared->fs_page_size * 100));
+ page_buf->min_raw_count = (unsigned)((size * page_buf_min_raw_perc) / (f->shared->fs_page_size * 100));
+
+ if(NULL == (page_buf->slist_ptr = H5SL_create(H5SL_TYPE_HADDR, NULL)))
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTCREATE, FAIL, "can't create skip list")
+ if(NULL == (page_buf->mf_slist_ptr = H5SL_create(H5SL_TYPE_HADDR, NULL)))
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTCREATE, FAIL, "can't create skip list")
+
+ if(NULL == (page_buf->page_fac = H5FL_fac_init(page_buf->page_size)))
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTINIT, FAIL, "can't create page factory")
+
+ f->shared->page_buf = page_buf;
+
+done:
+ if(ret_value < 0) {
+ if(page_buf != NULL) {
+ if(page_buf->slist_ptr != NULL)
+ H5SL_close(page_buf->slist_ptr);
+ if(page_buf->mf_slist_ptr != NULL)
+ H5SL_close(page_buf->mf_slist_ptr);
+ if(page_buf->page_fac != NULL)
+ H5FL_fac_term(page_buf->page_fac);
+ page_buf = H5FL_FREE(H5PB_t, page_buf);
+ } /* end if */
+ } /* end if */
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5PB_create */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PB__flush_cb
+ *
+ * Purpose: Callback to flush PB skiplist entries.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Mohamad Chaarawi
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5PB__flush_cb(void *item, void H5_ATTR_UNUSED *key, void *_op_data)
+{
+ H5PB_entry_t *page_entry = (H5PB_entry_t *)item; /* Pointer to page entry node */
+ const H5F_io_info2_t *fio_info = (const H5F_io_info2_t *)_op_data;
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_STATIC
+
+ /* Sanity checks */
+ HDassert(page_entry);
+ HDassert(fio_info);
+
+ /* Flush the page if it's dirty */
+ if(page_entry->is_dirty)
+ if(H5PB__write_entry(fio_info, page_entry) < 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "file write failed")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5PB__flush_cb() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PB_flush
+ *
+ * Purpose: Flush/Free all the PB entries to the file.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Mohamad Chaarawi
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PB_flush(const H5F_io_info2_t *fio_info)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* Sanity check */
+ HDassert(fio_info);
+ HDassert(fio_info->f);
+ HDassert(fio_info->meta_dxpl);
+ HDassert(fio_info->raw_dxpl);
+
+ /* Flush all the entries in the PB skiplist, if we have write access on the file */
+ if(fio_info->f->shared->page_buf && (H5F_ACC_RDWR & H5F_INTENT(fio_info->f))) {
+ H5PB_t *page_buf = fio_info->f->shared->page_buf;
+
+ /* Iterate over all entries in page buffer skip list */
+ if(H5SL_iterate(page_buf->slist_ptr, H5PB__flush_cb, (void *)fio_info))
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_BADITER, FAIL, "can't flush page buffer skip list")
+ } /* end if */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5PB_flush */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PB__dest_cb
+ *
+ * Purpose: Callback to free PB skiplist entries.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Mohamad Chaarawi
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5PB__dest_cb(void *item, void H5_ATTR_UNUSED *key, void *_op_data)
+{
+ H5PB_entry_t *page_entry = (H5PB_entry_t *)item; /* Pointer to page entry node */
+ H5PB_ud1_t *op_data = (H5PB_ud1_t *)_op_data;
+
+ FUNC_ENTER_STATIC_NOERR
+
+ /* Sanity checking */
+ HDassert(page_entry);
+ HDassert(op_data);
+ HDassert(op_data->page_buf);
+
+ /* Remove entry from LRU list */
+ if(op_data->actual_slist) {
+ H5PB__REMOVE_LRU(op_data->page_buf, page_entry)
+ page_entry->page_buf_ptr = H5FL_FAC_FREE(op_data->page_buf->page_fac, page_entry->page_buf_ptr);
+ } /* end if */
+
+ /* Free page entry */
+ page_entry = H5FL_FREE(H5PB_entry_t, page_entry);
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5PB__dest_cb() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PB_dest
+ *
+ * Purpose: destroy the PB on the file.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Mohamad Chaarawi
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PB_dest(H5F_t *f)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* Sanity checks */
+ HDassert(f);
+
+ /* Destroy page buffer info, if there is any */
+ if(f->shared->page_buf) {
+ H5PB_t *page_buf = f->shared->page_buf;
+ H5PB_ud1_t op_data; /* Iteration context */
+
+ /* Set up context info */
+ op_data.page_buf = page_buf;
+
+ /* Destroy the skip list containing all the entries in the PB */
+ op_data.actual_slist = TRUE;
+ if(H5SL_destroy(page_buf->slist_ptr, H5PB__dest_cb, &op_data))
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTCLOSEOBJ, FAIL, "can't destroy page buffer skip list")
+
+ /* Destroy the skip list containing the new entries */
+ op_data.actual_slist = FALSE;
+ if(H5SL_destroy(page_buf->mf_slist_ptr, H5PB__dest_cb, &op_data))
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTCLOSEOBJ, FAIL, "can't destroy page buffer skip list")
+
+ /* Destroy the page factory */
+ if(H5FL_fac_term(page_buf->page_fac) < 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTRELEASE, FAIL, "can't destroy page buffer page factory")
+
+#ifdef QAK
+H5PB_print_stats(page_buf);
+#endif /* QAK */
+
+ f->shared->page_buf = H5FL_FREE(H5PB_t, page_buf);
+ } /* end if */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5PB_dest */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PB_add_new_page
+ *
+ * Purpose: Add a new page to the new page skip list. This is called
+ * from the MF layer when a new page is allocated to
+ * indicate to the page buffer layer that a read of the page
+ * from the file is not necessary since it's an empty page.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Mohamad Chaarawi
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PB_add_new_page(H5F_t *f, H5FD_mem_t type, haddr_t page_addr)
+{
+ H5PB_t *page_buf = f->shared->page_buf;
+ H5PB_entry_t *page_entry = NULL; /* pointer to the corresponding page entry */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* Sanity checks */
+ HDassert(page_buf);
+
+ /* If there is an existing page, this means that at some point the
+ * file free space manager freed and re-allocated a page at the same
+ * address. No need to do anything here then...
+ */
+ /* MSC - to be safe, might want to dig in the MF layer and remove
+ * the page when it is freed from this list if it still exists and
+ * remove this check
+ */
+ if(NULL == H5SL_search(page_buf->mf_slist_ptr, &(page_addr))) {
+ /* Create the new PB entry */
+ if(NULL == (page_entry = H5FL_CALLOC(H5PB_entry_t)))
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "memory allocation failed")
+
+ /* Initialize page fields */
+ page_entry->addr = page_addr;
+ page_entry->type = (H5F_mem_page_t)type;
+ page_entry->is_dirty = FALSE;
+
+ /* Insert entry in skip list */
+ if(H5SL_insert(page_buf->mf_slist_ptr, page_entry, &(page_entry->addr)) < 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_BADVALUE, FAIL, "Can't insert entry in skip list")
+ } /* end if */
+
+done:
+ if(ret_value < 0)
+ if(page_entry)
+ page_entry = H5FL_FREE(H5PB_entry_t, page_entry);
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5PB_add_new_page */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PB_update_entry
+ *
+ * Purpose: In PHDF5, entries that are written by other processes and just
+ * marked clean by this process have to have their corresponding
+ * pages updated if they exist in the page buffer.
+ * This routine checks and update the pages.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Mohamad Chaarawi
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PB_update_entry(H5PB_t *page_buf, haddr_t addr, size_t size, const void *buf)
+{
+ H5PB_entry_t *page_entry; /* Pointer to the corresponding page entry */
+ haddr_t page_addr;
+
+ FUNC_ENTER_NOAPI_NOERR
+
+ /* Sanity checks */
+ HDassert(page_buf);
+ HDassert(size <= page_buf->page_size);
+ HDassert(buf);
+
+ /* calculate the aligned address of the first page */
+ page_addr = (addr / page_buf->page_size) * page_buf->page_size;
+
+ /* search for the page and update if found */
+ page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&page_addr));
+ if(page_entry) {
+ haddr_t offset;
+
+ HDassert(addr + size <= page_addr + page_buf->page_size);
+ offset = addr - page_addr;
+ HDmemcpy((uint8_t *)page_entry->page_buf_ptr + offset, buf, size);
+
+ /* move to top of LRU list */
+ H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry)
+ } /* end if */
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5PB_update_entry */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PB_remove_entry
+ *
+ * Purpose: Remove possible metadata entry with ADDR from the PB cache.
+ * This is in response to the data corruption bug from fheap.c
+ * with page buffering + page strategy.
+ * Note: Large metadata page bypasses the PB cache.
+ * Note: Update of raw data page (large or small sized) is handled by the PB cache.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Vailin Choi; Feb 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PB_remove_entry(const H5F_t *f, haddr_t addr)
+{
+ H5PB_t *page_buf = f->shared->page_buf;
+ H5PB_entry_t *page_entry = NULL; /* pointer to the page entry being searched */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* Sanity checks */
+ HDassert(page_buf);
+
+ /* Search for address in the skip list */
+ page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&addr));
+
+ /* If found, remove the entry from the PB cache */
+ if(page_entry) {
+ HDassert(page_entry->type != H5F_MEM_PAGE_DRAW);
+ if(NULL == H5SL_remove(page_buf->slist_ptr, &(page_entry->addr)))
+ HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "Page Entry is not in skip list")
+
+ /* Remove from LRU list */
+ H5PB__REMOVE_LRU(page_buf, page_entry)
+ HDassert(H5SL_count(page_buf->slist_ptr) == page_buf->LRU_list_len);
+
+ page_buf->meta_count--;
+
+ page_entry->page_buf_ptr = H5FL_FAC_FREE(page_buf->page_fac, page_entry->page_buf_ptr);
+ page_entry = H5FL_FREE(H5PB_entry_t, page_entry);
+ } /* end if */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5PB_remove_entry */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PB_read
+ *
+ * Purpose: Reads in the data from the page containing it if it exists
+ * in the PB cache; otherwise reads in the page through the VFD.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Mohamad Chaarawi
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PB_read(const H5F_io_info2_t *fio_info, H5FD_mem_t type, haddr_t addr,
+ size_t size, void *buf/*out*/)
+{
+ H5PB_t *page_buf; /* Page buffering info for this file */
+ H5PB_entry_t *page_entry; /* Pointer to the corresponding page entry */
+ H5FD_io_info_t fdio_info; /* File driver I/O info */
+ haddr_t first_page_addr, last_page_addr; /* Addresses of the first and last pages covered by I/O */
+ haddr_t offset;
+ haddr_t search_addr; /* Address of current page */
+ hsize_t num_touched_pages; /* Number of pages accessed */
+ size_t access_size;
+ hbool_t bypass_pb = FALSE; /* Whether to bypass page buffering */
+ hsize_t i; /* Local index variable */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* Sanity checks */
+ HDassert(fio_info);
+
+ /* Get pointer to page buffer info for this file */
+ page_buf = fio_info->f->shared->page_buf;
+
+#ifdef H5_HAVE_PARALLEL
+ if(H5F_HAS_FEATURE(fio_info->f, H5FD_FEAT_HAS_MPI)) {
+#if 1
+ bypass_pb = TRUE;
+#else
+ /* MSC - why this stopped working ? */
+ int mpi_size;
+
+ if((mpi_size = H5F_mpi_get_size(fio_info->f)) < 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "can't retrieve MPI communicator size")
+ if(1 != mpi_size)
+ bypass_pb = TRUE;
+#endif
+ } /* end if */
+#endif
+
+ /* If page buffering is disabled, or the I/O size is larger than that of a
+ * single page, or if this is a parallel raw data access, bypass page
+ * buffering.
+ */
+ if(NULL == page_buf || size >= page_buf->page_size ||
+ (bypass_pb && H5FD_MEM_DRAW == type)) {
+ if(H5F__accum_read(fio_info, type, addr, size, buf) < 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_READERROR, FAIL, "read through metadata accumulator failed")
+
+ /* Update statistics */
+ if(page_buf) {
+ if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP)
+ page_buf->bypasses[1] ++;
+ else
+ page_buf->bypasses[0] ++;
+ } /* end if */
+
+ /* If page buffering is disabled, or if this is a large metadata access,
+ * or if this is parallel raw data access, we are done here
+ */
+ if(NULL == page_buf || (size >= page_buf->page_size && H5FD_MEM_DRAW != type) ||
+ (bypass_pb && H5FD_MEM_DRAW == type))
+ HGOTO_DONE(SUCCEED)
+ } /* end if */
+
+ /* Update statistics */
+ if(page_buf) {
+ if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP)
+ page_buf->accesses[1]++;
+ else
+ page_buf->accesses[0]++;
+ } /* end if */
+
+ /* Calculate the aligned address of the first page */
+ first_page_addr = (addr / page_buf->page_size) * page_buf->page_size;
+
+ /* For Raw data calculate the aligned address of the last page and
+ * the number of pages accessed if more than 1 page is accessed
+ */
+ if(H5FD_MEM_DRAW == type) {
+ last_page_addr = ((addr + size - 1) / page_buf->page_size) * page_buf->page_size;
+
+ /* How many pages does this write span */
+ num_touched_pages = (last_page_addr / page_buf->page_size + 1) -
+ (first_page_addr / page_buf->page_size);
+ if(first_page_addr == last_page_addr) {
+ HDassert(1 == num_touched_pages);
+ last_page_addr = HADDR_UNDEF;
+ } /* end if */
+ } /* end if */
+ /* Otherwise set last page addr to HADDR_UNDEF */
+ else {
+ num_touched_pages = 1;
+ last_page_addr = HADDR_UNDEF;
+ } /* end else */
+
+ /* Translate to file driver I/O info object */
+ fdio_info.file = fio_info->f->shared->lf;
+ fdio_info.meta_dxpl = fio_info->meta_dxpl;
+ fdio_info.raw_dxpl = fio_info->raw_dxpl;
+
+ /* Copy raw data from dirty pages into the read buffer if the read
+ request spans pages in the page buffer*/
+ if(H5FD_MEM_DRAW == type && size >= page_buf->page_size) {
+ H5SL_node_t *node;
+
+ /* For each touched page in the page buffer, check if it
+ * exists in the page Buffer and is dirty. If it does, we
+ * update the buffer with what's in the page so we get the up
+ * to date data into the buffer after the big read from the file.
+ */
+ node = H5SL_find(page_buf->slist_ptr, (void *)(&first_page_addr));
+ for(i = 0; i < num_touched_pages; i++) {
+ search_addr = i*page_buf->page_size + first_page_addr;
+
+ /* if we still haven't located a starting page, search again */
+ if(!node && i!=0)
+ node = H5SL_find(page_buf->slist_ptr, (void *)(&search_addr));
+
+ /* if the current page is in the Page Buffer, do the updates */
+ if(node) {
+ page_entry = (H5PB_entry_t *)H5SL_item(node);
+
+ HDassert(page_entry);
+
+ /* If the current page address falls out of the access
+ block, then there are no more pages to go over */
+ if(page_entry->addr >= addr + size)
+ break;
+
+ HDassert(page_entry->addr == search_addr);
+
+ if(page_entry->is_dirty) {
+ /* special handling for the first page if it is not a full page access */
+ if(i == 0 && first_page_addr != addr) {
+ offset = addr - first_page_addr;
+ HDassert(page_buf->page_size > offset);
+
+ HDmemcpy(buf, (uint8_t *)page_entry->page_buf_ptr + offset,
+ page_buf->page_size - (size_t)offset);
+
+ /* move to top of LRU list */
+ H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry)
+ } /* end if */
+ /* special handling for the last page if it is not a full page access */
+ else if(num_touched_pages > 1 && i == num_touched_pages-1 && search_addr < addr+size) {
+ offset = (num_touched_pages-2)*page_buf->page_size +
+ (page_buf->page_size - (addr - first_page_addr));
+
+ HDmemcpy((uint8_t *)buf + offset, page_entry->page_buf_ptr,
+ (size_t)((addr + size) - last_page_addr));
+
+ /* move to top of LRU list */
+ H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry)
+ } /* end else-if */
+ /* copy the entire fully accessed pages */
+ else {
+ offset = i*page_buf->page_size;
+
+ HDmemcpy((uint8_t *)buf+(i*page_buf->page_size) , page_entry->page_buf_ptr,
+ page_buf->page_size);
+ } /* end else */
+ } /* end if */
+ node = H5SL_next(node);
+ } /* end if */
+ } /* end for */
+ } /* end if */
+ else {
+ /* A raw data access could span 1 or 2 PB entries at this point so
+ we need to handle that */
+ HDassert(1 == num_touched_pages || 2 == num_touched_pages);
+ for(i = 0 ; i < num_touched_pages; i++) {
+ haddr_t buf_offset;
+
+ /* Calculate the aligned address of the page to search for it in the skip list */
+ search_addr = (0==i ? first_page_addr : last_page_addr);
+
+ /* Calculate the access size if the access spans more than 1 page */
+ if(1 == num_touched_pages)
+ access_size = size;
+ else
+ access_size = (0 == i ? (size_t)((first_page_addr + page_buf->page_size) - addr) : (size - access_size));
+
+ /* Lookup the page in the skip list */
+ page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&search_addr));
+
+ /* if found */
+ if(page_entry) {
+ offset = (0 == i ? addr - page_entry->addr : 0);
+ buf_offset = (0 == i ? 0 : size - access_size);
+
+ /* copy the requested data from the page into the input buffer */
+ HDmemcpy((uint8_t *)buf + buf_offset, (uint8_t *)page_entry->page_buf_ptr + offset, access_size);
+
+ /* Update LRU */
+ H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry)
+
+ /* Update statistics */
+ if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP)
+ page_buf->hits[1]++;
+ else
+ page_buf->hits[0]++;
+ } /* end if */
+ /* if not found */
+ else {
+ void *new_page_buf = NULL;
+ size_t page_size = page_buf->page_size;
+ haddr_t eoa;
+
+ /* make space for new entry */
+ if((H5SL_count(page_buf->slist_ptr) * page_buf->page_size) >= page_buf->max_size) {
+ htri_t can_make_space;
+
+ /* check if we can make space in page buffer */
+ if((can_make_space = H5PB__make_space(fio_info, page_buf, type)) < 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "make space in Page buffer Failed")
+
+ /* if make_space returns 0, then we can't use the page
+ buffer for this I/O and we need to bypass */
+ if(0 == can_make_space) {
+ /* make space can't return FALSE on second touched page since the first is of the same type */
+ HDassert(0 == i);
+
+ /* read entire block from VFD and return */
+ if(H5FD_read(&fdio_info, type, addr, size, buf) < 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_READERROR, FAIL, "driver read request failed")
+
+ /* Break out of loop */
+ break;
+ } /* end if */
+ } /* end if */
+
+ /* Read page from VFD */
+ if(NULL == (new_page_buf = H5FL_FAC_MALLOC(page_buf->page_fac)))
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTALLOC, FAIL, "memory allocation failed for page buffer entry")
+
+ /* Read page through the VFD layer, but make sure we don't read past the EOA. */
+
+ /* Retrieve the 'eoa' for the file */
+ if(HADDR_UNDEF == (eoa = H5F_get_eoa(fio_info->f, type)))
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eoa request failed")
+
+ /* If the entire page falls outside the EOA, then fail */
+ if(search_addr > eoa)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_BADVALUE, FAIL, "reading an entire page that is outside the file EOA")
+
+ /* Adjust the read size to not go beyond the EOA */
+ if(search_addr + page_size > eoa)
+ page_size = (size_t)(eoa - search_addr);
+
+ /* Read page from VFD */
+ if(H5FD_read(&fdio_info, type, search_addr, page_size, new_page_buf) < 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_READERROR, FAIL, "driver read request failed")
+
+ /* Copy the requested data from the page into the input buffer */
+ offset = (0 == i ? addr - search_addr : 0);
+ buf_offset = (0 == i ? 0 : size - access_size);
+ HDmemcpy((uint8_t *)buf + buf_offset, (uint8_t *)new_page_buf + offset, access_size);
+
+ /* Create the new PB entry */
+ if(NULL == (page_entry = H5FL_CALLOC(H5PB_entry_t)))
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "memory allocation failed")
+
+ page_entry->page_buf_ptr = new_page_buf;
+ page_entry->addr = search_addr;
+ page_entry->type = (H5F_mem_page_t)type;
+ page_entry->is_dirty = FALSE;
+
+ /* Insert page into PB */
+ if(H5PB__insert_entry(page_buf, page_entry) < 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTSET, FAIL, "error inserting new page in page buffer")
+
+ /* Update statistics */
+ if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP)
+ page_buf->misses[1]++;
+ else
+ page_buf->misses[0]++;
+ } /* end else */
+ } /* end for */
+ } /* end else */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PB_read() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PB_write
+ *
+ * Purpose: Write data into the Page Buffer. If the page exists in the
+ * cache, update it; otherwise read it from disk, update it, and
+ * insert into cache.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Mohamad Chaarawi
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5PB_write(const H5F_io_info2_t *fio_info, H5FD_mem_t type, haddr_t addr,
+ size_t size, const void *buf)
+{
+ H5PB_t *page_buf; /* Page buffering info for this file */
+ H5PB_entry_t *page_entry; /* Pointer to the corresponding page entry */
+ H5FD_io_info_t fdio_info; /* File driver I/O info */
+ haddr_t first_page_addr, last_page_addr; /* Addresses of the first and last pages covered by I/O */
+ haddr_t offset;
+ haddr_t search_addr; /* Address of current page */
+ hsize_t num_touched_pages; /* Number of pages accessed */
+ size_t access_size;
+ hbool_t bypass_pb = FALSE; /* Whether to bypass page buffering */
+ hsize_t i; /* Local index variable */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+ /* Sanity checks */
+ HDassert(fio_info);
+ HDassert(fio_info->f);
+
+ /* Get pointer to page buffer info for this file */
+ page_buf = fio_info->f->shared->page_buf;
+
+#ifdef H5_HAVE_PARALLEL
+ if(H5F_HAS_FEATURE(fio_info->f, H5FD_FEAT_HAS_MPI)) {
+#if 1
+ bypass_pb = TRUE;
+#else
+ /* MSC - why this stopped working ? */
+ int mpi_size;
+
+ if((mpi_size = H5F_mpi_get_size(fio_info->f)) < 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "can't retrieve MPI communicator size")
+ if(1 != mpi_size)
+ bypass_pb = TRUE;
+#endif
+ } /* end if */
+#endif
+
+ /* If page buffering is disabled, or the I/O size is larger than that of a
+ * single page, or if this is a parallel raw data access, bypass page
+ * buffering.
+ */
+ if(NULL == page_buf || size >= page_buf->page_size || bypass_pb) {
+ if(H5F__accum_write(fio_info, type, addr, size, buf) < 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "write through metadata accumulator failed")
+
+ /* Update statistics */
+ if(page_buf) {
+ if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP)
+ page_buf->bypasses[1]++;
+ else
+ page_buf->bypasses[0]++;
+ } /* end if */
+
+ /* If page buffering is disabled, or if this is a large metadata access,
+ * or if this is a parallel raw data access, we are done here
+ */
+ if(NULL == page_buf || (size >= page_buf->page_size && H5FD_MEM_DRAW != type) ||
+ (bypass_pb && H5FD_MEM_DRAW == type))
+ HGOTO_DONE(SUCCEED)
+
+#ifdef H5_HAVE_PARALLEL
+ if(bypass_pb) {
+ if(H5PB_update_entry(page_buf, addr, size, buf) > 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTUPDATE, FAIL, "failed to update PB with metadata cache")
+ HGOTO_DONE(SUCCEED)
+ } /* end if */
+#endif
+ } /* end if */
+
+ /* Update statistics */
+ if(page_buf) {
+ if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP)
+ page_buf->accesses[1]++;
+ else
+ page_buf->accesses[0]++;
+ } /* end if */
+
+ /* Calculate the aligned address of the first page */
+ first_page_addr = (addr / page_buf->page_size) * page_buf->page_size;
+
+ /* For raw data calculate the aligned address of the last page and
+ * the number of pages accessed if more than 1 page is accessed
+ */
+ if(H5FD_MEM_DRAW == type) {
+ last_page_addr = (addr + size - 1) / page_buf->page_size * page_buf->page_size;
+
+ /* how many pages does this write span */
+ num_touched_pages = (last_page_addr/page_buf->page_size + 1) -
+ (first_page_addr / page_buf->page_size);
+ if(first_page_addr == last_page_addr) {
+ HDassert(1 == num_touched_pages);
+ last_page_addr = HADDR_UNDEF;
+ } /* end if */
+ } /* end if */
+ /* Otherwise set last page addr to HADDR_UNDEF */
+ else {
+ num_touched_pages = 1;
+ last_page_addr = HADDR_UNDEF;
+ } /* end else */
+
+ /* Translate to file driver I/O info object */
+ fdio_info.file = fio_info->f->shared->lf;
+ fdio_info.meta_dxpl = fio_info->meta_dxpl;
+ fdio_info.raw_dxpl = fio_info->raw_dxpl;
+
+ /* Check if existing pages for raw data need to be updated since raw data access is not atomic */
+ if(H5FD_MEM_DRAW == type && size >= page_buf->page_size) {
+ /* For each touched page, check if it exists in the page buffer, and
+ * update it with the data in the buffer to keep it up to date
+ */
+ for(i = 0; i < num_touched_pages; i++) {
+ search_addr = i * page_buf->page_size + first_page_addr;
+
+ /* Special handling for the first page if it is not a full page update */
+ if(i == 0 && first_page_addr != addr) {
+ /* Lookup the page in the skip list */
+ page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&search_addr));
+ if(page_entry) {
+ offset = addr - first_page_addr;
+ HDassert(page_buf->page_size > offset);
+
+ /* Update page's data */
+ HDmemcpy((uint8_t *)page_entry->page_buf_ptr + offset, buf, page_buf->page_size - (size_t)offset);
+
+ /* Mark page dirty and push to top of LRU */
+ page_entry->is_dirty = TRUE;
+ H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry)
+ } /* end if */
+ } /* end if */
+ /* Special handling for the last page if it is not a full page update */
+ else if(num_touched_pages > 1 && i == (num_touched_pages - 1) &&
+ (search_addr + page_buf->page_size) != (addr + size)) {
+ HDassert(search_addr+page_buf->page_size > addr+size);
+
+ /* Lookup the page in the skip list */
+ page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&search_addr));
+ if(page_entry) {
+ offset = (num_touched_pages - 2) * page_buf->page_size +
+ (page_buf->page_size - (addr - first_page_addr));
+
+ /* Update page's data */
+ HDmemcpy(page_entry->page_buf_ptr, (const uint8_t *)buf + offset,
+ (size_t)((addr + size) - last_page_addr));
+
+ /* Mark page dirty and push to top of LRU */
+ page_entry->is_dirty = TRUE;
+ H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry)
+ } /* end if */
+ } /* end else-if */
+ /* Discard all fully written pages from the page buffer */
+ else {
+ page_entry = (H5PB_entry_t *)H5SL_remove(page_buf->slist_ptr, (void *)(&search_addr));
+ if(page_entry) {
+ /* Remove from LRU list */
+ H5PB__REMOVE_LRU(page_buf, page_entry)
+
+ /* Decrement page count of appropriate type */
+ if(H5F_MEM_PAGE_DRAW == page_entry->type || H5F_MEM_PAGE_GHEAP == page_entry->type)
+ page_buf->raw_count--;
+ else
+ page_buf->meta_count--;
+
+ /* Free page info */
+ page_entry->page_buf_ptr = H5FL_FAC_FREE(page_buf->page_fac, page_entry->page_buf_ptr);
+ page_entry = H5FL_FREE(H5PB_entry_t, page_entry);
+ } /* end if */
+ } /* end else */
+ } /* end for */
+ } /* end if */
+ else {
+ /* An access could span 1 or 2 PBs at this point so we need to handle that */
+ HDassert(1 == num_touched_pages || 2 == num_touched_pages);
+ for(i = 0; i < num_touched_pages; i++) {
+ haddr_t buf_offset;
+
+ /* Calculate the aligned address of the page to search for it in the skip list */
+ search_addr = (0 == i ? first_page_addr : last_page_addr);
+
+ /* Calculate the access size if the access spans more than 1 page */
+ if(1 == num_touched_pages)
+ access_size = size;
+ else
+ access_size = (0 == i ? (size_t)(first_page_addr + page_buf->page_size - addr) : (size - access_size));
+
+ /* Lookup the page in the skip list */
+ page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&search_addr));
+
+ /* If found */
+ if(page_entry) {
+ offset = (0 == i ? addr - page_entry->addr : 0);
+ buf_offset = (0 == i ? 0 : size - access_size);
+
+ /* Copy the requested data from the input buffer into the page */
+ HDmemcpy((uint8_t *)page_entry->page_buf_ptr + offset, (const uint8_t *)buf + buf_offset, access_size);
+
+ /* Mark page dirty and push to top of LRU */
+ page_entry->is_dirty = TRUE;
+ H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry)
+
+ /* Update statistics */
+ if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP)
+ page_buf->hits[1]++;
+ else
+ page_buf->hits[0]++;
+ } /* end if */
+ /* If not found */
+ else {
+ void *new_page_buf;
+ size_t page_size = page_buf->page_size;
+
+ /* Make space for new entry */
+ if((H5SL_count(page_buf->slist_ptr) * page_buf->page_size) >= page_buf->max_size) {
+ htri_t can_make_space;
+
+ /* Check if we can make space in page buffer */
+ if((can_make_space = H5PB__make_space(fio_info, page_buf, type)) < 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "make space in Page buffer Failed")
+
+ /* If make_space returns 0, then we can't use the page
+ * buffer for this I/O and we need to bypass
+ */
+ if(0 == can_make_space) {
+ HDassert(0 == i);
+
+ /* Write to VFD and return */
+ if(H5FD_write(&fdio_info, type, addr, size, buf) < 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "driver write request failed")
+
+ /* Break out of loop */
+ break;
+ } /* end if */
+ } /* end if */
+
+ /* Don't bother searching if there is no write access */
+ if(H5F_ACC_RDWR & H5F_INTENT(fio_info->f))
+ /* Lookup & remove the page from the new skip list page if
+ * it exists to see if this is a new page from the MF layer
+ */
+ page_entry = (H5PB_entry_t *)H5SL_remove(page_buf->mf_slist_ptr, (void *)(&search_addr));
+
+ /* Calculate offset into the buffer of the page and the user buffer */
+ offset = (0 == i ? addr - search_addr : 0);
+ buf_offset = (0 == i ? 0 : size - access_size);
+
+ /* If found, then just update the buffer pointer to the newly allocate buffer */
+ if(page_entry) {
+ /* Allocate space for the page buffer */
+ if(NULL == (new_page_buf = H5FL_FAC_MALLOC(page_buf->page_fac)))
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTALLOC, FAIL, "memory allocation failed for page buffer entry")
+ HDmemset(new_page_buf, 0, (size_t)offset);
+ HDmemset((uint8_t *)new_page_buf + offset + access_size, 0, page_size - ((size_t)offset + access_size));
+
+ page_entry->page_buf_ptr = new_page_buf;
+
+ /* Update statistics */
+ if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP)
+ page_buf->hits[1]++;
+ else
+ page_buf->hits[0]++;
+ } /* end if */
+ /* Otherwise read page through the VFD layer, but make sure we don't read past the EOA. */
+ else {
+ haddr_t eoa, eof = HADDR_UNDEF;
+
+ /* Allocate space for the page buffer */
+ if(NULL == (new_page_buf = H5FL_FAC_CALLOC(page_buf->page_fac)))
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTALLOC, FAIL, "memory allocation failed for page buffer entry")
+
+ /* Create the new loaded PB entry */
+ if(NULL == (page_entry = H5FL_CALLOC(H5PB_entry_t)))
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTALLOC, FAIL, "memory allocation failed")
+
+ page_entry->page_buf_ptr = new_page_buf;
+ page_entry->addr = search_addr;
+ page_entry->type = (H5F_mem_page_t)type;
+
+ /* Retrieve the 'eoa' for the file */
+ if(HADDR_UNDEF == (eoa = H5F_get_eoa(fio_info->f, type)))
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eoa request failed")
+
+ /* If the entire page falls outside the EOA, then fail */
+ if(search_addr > eoa)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_BADVALUE, FAIL, "writing to a page that is outside the file EOA")
+
+ /* Retrieve the 'eof' for the file - The MPI-VFD EOF
+ * returned will most likely be HADDR_UNDEF, so skip
+ * that check.
+ */
+ if(!H5F_HAS_FEATURE(fio_info->f, H5FD_FEAT_HAS_MPI))
+ if(HADDR_UNDEF == (eof = H5FD_get_eof(fio_info->f->shared->lf, H5FD_MEM_DEFAULT)))
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eof request failed")
+
+ /* Adjust the read size to not go beyond the EOA */
+ if(search_addr + page_size > eoa)
+ page_size = (size_t)(eoa - search_addr);
+
+ if(search_addr < eof) {
+ if(H5FD_read(&fdio_info, type, search_addr, page_size, new_page_buf) < 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_READERROR, FAIL, "driver read request failed")
+
+ /* Update statistics */
+ if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP)
+ page_buf->misses[1]++;
+ else
+ page_buf->misses[0]++;
+ } /* end if */
+ } /* end else */
+
+ /* Copy the requested data from the page into the input buffer */
+ HDmemcpy((uint8_t *)new_page_buf + offset, (const uint8_t *)buf+buf_offset, access_size);
+
+ /* Page is dirty now */
+ page_entry->is_dirty = TRUE;
+
+ /* Insert page into PB, evicting other pages as necessary */
+ if(H5PB__insert_entry(page_buf, page_entry) < 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTSET, FAIL, "error inserting new page in page buffer")
+ } /* end else */
+ } /* end for */
+ } /* end else */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PB_write() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PB__insert_entry()
+ *
+ * Purpose: ???
+ *
+ * This function was created without documentation.
+ * What follows is my best understanding of Mohamad's intent.
+ *
+ * Insert the supplied page into the page buffer, both the
+ * skip list and the LRU.
+ *
+ * As best I can tell, this function imposes no limit on the
+ * number of entries in the page buffer beyond an assertion
+ * failure it the page count exceeds the limit.
+ *
+ * JRM -- 12/22/16
+ *
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Mohamad Chaarawi
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5PB__insert_entry(H5PB_t *page_buf, H5PB_entry_t *page_entry)
+{
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_STATIC
+
+ /* Insert entry in skip list */
+ if(H5SL_insert(page_buf->slist_ptr, page_entry, &(page_entry->addr)) < 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTINSERT, FAIL, "can't insert entry in skip list")
+ HDassert(H5SL_count(page_buf->slist_ptr) * page_buf->page_size <= page_buf->max_size);
+
+ /* Increment appropriate page count */
+ if(H5F_MEM_PAGE_DRAW == page_entry->type || H5F_MEM_PAGE_GHEAP == page_entry->type)
+ page_buf->raw_count++;
+ else
+ page_buf->meta_count++;
+
+ /* Insert entry in LRU */
+ H5PB__INSERT_LRU(page_buf, page_entry)
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PB__insert_entry() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PB__make_space()
+ *
+ * Purpose: ???
+ *
+ * This function was created without documentation.
+ * What follows is my best understanding of Mohamad's intent.
+ *
+ * If necessary and if possible, evict a page from the page
+ * buffer to make space for the supplied page. Depending on
+ * the page buffer configuration and contents, and the page
+ * supplied this may or may not be possible.
+ *
+ * JRM -- 12/22/16
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Mohamad Chaarawi
+ *
+ *-------------------------------------------------------------------------
+ */
+static htri_t
+H5PB__make_space(const H5F_io_info2_t *fio_info, H5PB_t *page_buf,
+ H5FD_mem_t inserted_type)
+{
+ H5PB_entry_t *page_entry; /* Pointer to page eviction candidate */
+ htri_t ret_value = TRUE; /* Return value */
+
+ FUNC_ENTER_STATIC
+
+ /* Sanity check */
+ HDassert(fio_info);
+ HDassert(page_buf);
+
+ /* Get oldest entry */
+ page_entry = page_buf->LRU_tail_ptr;
+
+ if(H5FD_MEM_DRAW == inserted_type) {
+ /* If threshould is 100% metadata and page buffer is full of
+ metadata, then we can't make space for raw data */
+ if(0 == page_buf->raw_count && page_buf->min_meta_count == page_buf->meta_count) {
+ HDassert(page_buf->meta_count * page_buf->page_size == page_buf->max_size);
+ HGOTO_DONE(FALSE)
+ } /* end if */
+
+ /* check the metadata threshold before evicting metadata items */
+ while(1) {
+ if(page_entry->prev && H5F_MEM_PAGE_META == page_entry->type &&
+ page_buf->min_meta_count >= page_buf->meta_count)
+ page_entry = page_entry->prev;
+ else
+ break;
+ } /* end while */
+ } /* end if */
+ else {
+ /* If threshould is 100% raw data and page buffer is full of
+ raw data, then we can't make space for meta data */
+ if(0 == page_buf->meta_count && page_buf->min_raw_count == page_buf->raw_count) {
+ HDassert(page_buf->raw_count * page_buf->page_size == page_buf->max_size);
+ HGOTO_DONE(FALSE)
+ } /* end if */
+
+ /* check the raw data threshold before evicting raw data items */
+ while(1) {
+ if(page_entry->prev && (H5F_MEM_PAGE_DRAW == page_entry->type || H5F_MEM_PAGE_GHEAP == page_entry->type) &&
+ page_buf->min_raw_count >= page_buf->raw_count)
+ page_entry = page_entry->prev;
+ else
+ break;
+ } /* end while */
+ } /* end else */
+
+ /* Remove from page index */
+ if(NULL == H5SL_remove(page_buf->slist_ptr, &(page_entry->addr)))
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_BADVALUE, FAIL, "Tail Page Entry is not in skip list")
+
+ /* Remove entry from LRU list */
+ H5PB__REMOVE_LRU(page_buf, page_entry)
+ HDassert(H5SL_count(page_buf->slist_ptr) == page_buf->LRU_list_len);
+
+ /* Decrement appropriate page type counter */
+ if(H5F_MEM_PAGE_DRAW == page_entry->type || H5F_MEM_PAGE_GHEAP == page_entry->type)
+ page_buf->raw_count--;
+ else
+ page_buf->meta_count--;
+
+ /* Flush page if dirty */
+ if(page_entry->is_dirty)
+ if(H5PB__write_entry(fio_info, page_entry) < 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "file write failed")
+
+ /* Update statistics */
+ if(page_entry->type == H5F_MEM_PAGE_DRAW || H5F_MEM_PAGE_GHEAP == page_entry->type)
+ page_buf->evictions[1]++;
+ else
+ page_buf->evictions[0]++;
+
+ /* Release page */
+ page_entry->page_buf_ptr = H5FL_FAC_FREE(page_buf->page_fac, page_entry->page_buf_ptr);
+ page_entry = H5FL_FREE(H5PB_entry_t, page_entry);
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PB__make_space() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5PB__write_entry()
+ *
+ * Purpose: ???
+ *
+ * This function was created without documentation.
+ * What follows is my best understanding of Mohamad's intent.
+ *
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Mohamad Chaarawi
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5PB__write_entry(const H5F_io_info2_t *fio_info, H5PB_entry_t *page_entry)
+{
+ haddr_t eoa; /* Current EOA for the file */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_STATIC
+
+ /* Sanity check */
+ HDassert(fio_info);
+ HDassert(fio_info->f);
+ HDassert(page_entry);
+
+ /* Retrieve the 'eoa' for the file */
+ if(HADDR_UNDEF == (eoa = H5F_get_eoa(fio_info->f, page_entry->type)))
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eoa request failed")
+
+ /* If the starting address of the page is larger than
+ * the EOA, then the entire page is discarded without writing.
+ */
+ if(page_entry->addr <= eoa) {
+ H5FD_io_info_t fdio_info; /* File driver I/O info */
+ size_t page_size = fio_info->f->shared->page_buf->page_size;
+
+ /* Adjust the page length if it exceeds the EOA */
+ if((page_entry->addr + page_size) > eoa)
+ page_size = (size_t)(eoa - page_entry->addr);
+
+ /* Translate to file driver I/O info object */
+ fdio_info.file = fio_info->f->shared->lf;
+ fdio_info.meta_dxpl = fio_info->meta_dxpl;
+ fdio_info.raw_dxpl = fio_info->raw_dxpl;
+
+ if(H5FD_write(&fdio_info, page_entry->type, page_entry->addr, page_size, page_entry->page_buf_ptr) < 0)
+ HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "file write failed")
+ } /* end if */
+
+ page_entry->is_dirty = FALSE;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5PB__write_entry() */
+
diff --git a/src/H5PBmodule.h b/src/H5PBmodule.h
new file mode 100644
index 0000000..35da3f6
--- /dev/null
+++ b/src/H5PBmodule.h
@@ -0,0 +1,34 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the files COPYING and Copyright.html. COPYING can be found at the root *
+ * of the source code distribution tree; Copyright.html can be found at the *
+ * root level of an installed copy of the electronic HDF5 document set and *
+ * is linked from the top-level documents page. It can also be found at *
+ * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have *
+ * access to either file, you may request a copy from help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Programmer: Quincey Koziol <koziol@hdfgroup.org>
+ * Saturday, September 12, 2015
+ *
+ * Purpose: This file contains declarations which define macros for the
+ * H5PB package. Including this header means that the source file
+ * is part of the H5PB package.
+ */
+#ifndef _H5PBmodule_H
+#define _H5PBmodule_H
+
+/* Define the proper control macros for the generic FUNC_ENTER/LEAVE and error
+ * reporting macros.
+ */
+#define H5PB_MODULE
+#define H5_MY_PKG H5PB
+#define H5_MY_PKG_ERR H5E_RESOURCE
+#define H5_MY_PKG_INIT NO
+
+#endif /* _H5PBmodule_H */
diff --git a/src/H5PBpkg.h b/src/H5PBpkg.h
new file mode 100644
index 0000000..976f18d
--- /dev/null
+++ b/src/H5PBpkg.h
@@ -0,0 +1,60 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the files COPYING and Copyright.html. COPYING can be found at the root *
+ * of the source code distribution tree; Copyright.html can be found at the *
+ * root level of an installed copy of the electronic HDF5 document set and *
+ * is linked from the top-level documents page. It can also be found at *
+ * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have *
+ * access to either file, you may request a copy from help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+#if !(defined H5PB_FRIEND || defined H5PB_MODULE)
+#error "Do not include this file outside the H5PB package!"
+#endif
+
+#ifndef _H5PBpkg_H
+#define _H5PBpkg_H
+
+/* Get package's private header */
+#include "H5PBprivate.h"
+
+/* Other private headers needed by this file */
+
+/**************************/
+/* Package Private Macros */
+/**************************/
+
+
+/****************************/
+/* Package Private Typedefs */
+/****************************/
+
+typedef struct H5PB_entry_t {
+ void *page_buf_ptr; /* Pointer to the buffer containing the data */
+ haddr_t addr; /* Address of the page in the file */
+ H5F_mem_page_t type; /* Type of the page entry (H5F_MEM_PAGE_RAW/META) */
+ hbool_t is_dirty; /* Flag indicating whether the page has dirty data or not */
+
+ /* Fields supporting replacement policies */
+ struct H5PB_entry_t *next; /* next pointer in the LRU list */
+ struct H5PB_entry_t *prev; /* previous pointer in the LRU list */
+} H5PB_entry_t;
+
+
+/*****************************/
+/* Package Private Variables */
+/*****************************/
+
+
+/******************************/
+/* Package Private Prototypes */
+/******************************/
+
+
+#endif /* _H5PBpkg_H */
+
diff --git a/src/H5PBprivate.h b/src/H5PBprivate.h
new file mode 100644
index 0000000..3b5dcae
--- /dev/null
+++ b/src/H5PBprivate.h
@@ -0,0 +1,108 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the files COPYING and Copyright.html. COPYING can be found at the root *
+ * of the source code distribution tree; Copyright.html can be found at the *
+ * root level of an installed copy of the electronic HDF5 document set and *
+ * is linked from the top-level documents page. It can also be found at *
+ * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have *
+ * access to either file, you may request a copy from help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*-------------------------------------------------------------------------
+ *
+ * Created: H5PBprivate.h
+ * June 2014
+ * Mohamad Chaarawi
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef _H5PBprivate_H
+#define _H5PBprivate_H
+
+/* Include package's public header */
+#ifdef NOT_YET
+#include "H5PBpublic.h"
+#endif /* NOT_YET */
+
+/* Private headers needed by this header */
+#include "H5private.h" /* Generic Functions */
+#include "H5Fprivate.h" /* File access */
+#include "H5FLprivate.h" /* Free Lists */
+#include "H5SLprivate.h" /* Skip List */
+
+
+/**************************/
+/* Library Private Macros */
+/**************************/
+
+
+/****************************/
+/* Library Private Typedefs */
+/****************************/
+
+/* Forward declaration for a page buffer entry */
+struct H5PB_entry_t;
+
+/* Typedef for the main structure for the page buffer */
+typedef struct H5PB_t {
+ size_t max_size; /* The total page buffer size */
+ size_t page_size; /* Size of a single page */
+ unsigned min_meta_perc; /* Minimum ratio of metadata entries required before evicting meta entries */
+ unsigned min_raw_perc; /* Minimum ratio of raw data entries required before evicting raw entries */
+ unsigned meta_count; /* Number of entries for metadata */
+ unsigned raw_count; /* Number of entries for raw data */
+ unsigned min_meta_count; /* Minimum # of entries for metadata */
+ unsigned min_raw_count; /* Minimum # of entries for raw data */
+
+ H5SL_t *slist_ptr; /* Skip list with all the active page entries */
+ H5SL_t *mf_slist_ptr; /* Skip list containing newly allocated page entries inserted from the MF layer */
+
+ size_t LRU_list_len; /* Number of entries in the LRU (identical to slist_ptr count) */
+ struct H5PB_entry_t *LRU_head_ptr; /* Head pointer of the LRU */
+ struct H5PB_entry_t *LRU_tail_ptr; /* Tail pointer of the LRU */
+
+ H5FL_fac_head_t *page_fac; /* Factory for allocating pages */
+
+ /* Statistics */
+ unsigned accesses[2];
+ unsigned hits[2];
+ unsigned misses[2];
+ unsigned evictions[2];
+ unsigned bypasses[2];
+} H5PB_t;
+
+/*****************************/
+/* Library-private Variables */
+/*****************************/
+
+
+/***************************************/
+/* Library-private Function Prototypes */
+/***************************************/
+
+/* General routines */
+H5_DLL herr_t H5PB_create(H5F_t *file, size_t page_buffer_size, unsigned page_buf_min_meta_perc, unsigned page_buf_min_raw_perc);
+H5_DLL herr_t H5PB_flush(const H5F_io_info2_t *fio_info);
+H5_DLL herr_t H5PB_dest(H5F_t *file);
+H5_DLL herr_t H5PB_add_new_page(H5F_t *f, H5FD_mem_t type, haddr_t page_addr);
+H5_DLL herr_t H5PB_update_entry(H5PB_t *page_buf, haddr_t addr, size_t size, const void *buf);
+H5_DLL herr_t H5PB_remove_entry(const H5F_t *f, haddr_t addr);
+H5_DLL herr_t H5PB_read(const H5F_io_info2_t *fio_info, H5FD_mem_t type,
+ haddr_t addr, size_t size, void *buf/*out*/);
+H5_DLL herr_t H5PB_write(const H5F_io_info2_t *f, H5FD_mem_t type, haddr_t addr,
+ size_t size, const void *buf);
+
+/* Statistics routines */
+H5_DLL herr_t H5PB_reset_stats(H5PB_t *page_buf);
+H5_DLL herr_t H5PB_get_stats(const H5PB_t *page_buf, unsigned accesses[2],
+ unsigned hits[2], unsigned misses[2], unsigned evictions[2], unsigned bypasses[2]);
+H5_DLL herr_t H5PB_print_stats(const H5PB_t *page_buf);
+
+#endif /* !_H5PBprivate_H */
+
diff --git a/src/H5Pdeprec.c b/src/H5Pdeprec.c
index ecf2bea..d225dfa 100644
--- a/src/H5Pdeprec.c
+++ b/src/H5Pdeprec.c
@@ -488,5 +488,135 @@ done:
FUNC_LEAVE_API(ret_value)
} /* end H5Pget_version() */
+/*-------------------------------------------------------------------------
+ * Function: H5Pset_file_space
+ *
+ * Purpose: It is mapped to H5Pset_file_space_strategy().
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Vailin Choi; Jan 2017
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold)
+{
+
+ H5F_fspace_strategy_t new_strategy; /* File space strategy type */
+ hbool_t new_persist = H5F_FREE_SPACE_PERSIST_DEF; /* Persisting free-space or not */
+ hsize_t new_threshold = H5F_FREE_SPACE_THRESHOLD_DEF; /* Free-space section threshold */
+ H5F_file_space_type_t in_strategy = strategy; /* Input strategy */
+ hsize_t in_threshold = threshold; /* Input threshold */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE3("e", "iFfh", plist_id, strategy, threshold);
+
+ if((unsigned)in_strategy >= H5F_FILE_SPACE_NTYPES)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid strategy")
+ /*
+ * For 1.10.0 H5Pset_file_space:
+ * If strategy is zero, the property is not changed;
+ * the existing strategy is retained.
+ * If threshold is zero, the property is not changed;
+ * the existing threshold is retained.
+ */
+ if(!in_strategy)
+ H5Pget_file_space(plist_id, &in_strategy, NULL);
+ if(!in_threshold)
+ H5Pget_file_space(plist_id, NULL, &in_threshold);
+
+ switch(in_strategy) {
+ case H5F_FILE_SPACE_ALL_PERSIST:
+ new_strategy = H5F_FSPACE_STRATEGY_FSM_AGGR;
+ new_persist = TRUE;
+ new_threshold = in_threshold;
+ break;
+
+ case H5F_FILE_SPACE_ALL:
+ new_strategy = H5F_FSPACE_STRATEGY_FSM_AGGR;
+ new_threshold = in_threshold;
+ break;
+
+ case H5F_FILE_SPACE_AGGR_VFD:
+ new_strategy = H5F_FSPACE_STRATEGY_AGGR;
+ break;
+
+ case H5F_FILE_SPACE_VFD:
+ new_strategy = H5F_FSPACE_STRATEGY_NONE;
+ break;
+
+ case H5F_FILE_SPACE_NTYPES:
+ case H5F_FILE_SPACE_DEFAULT:
+ default:
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file space strategy")
+ }
+
+ if(H5Pset_file_space_strategy(plist_id, new_strategy, new_persist, new_threshold) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set file space strategy")
+
+done:
+ FUNC_LEAVE_API(ret_value)
+} /* H5Pset_file_space() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pget_file_space
+ *
+ * Purpose: It is mapped to H5Pget_file_space_strategy().
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Vailin Choi; Jan 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold)
+{
+ H5F_fspace_strategy_t new_strategy; /* File space strategy type */
+ hbool_t new_persist; /* Persisting free-space or not */
+ hsize_t new_threshold; /* Free-space section threshold */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE3("e", "i*Ff*h", plist_id, strategy, threshold);
+
+ /* Get current file space info */
+ if(H5Pget_file_space_strategy(plist_id, &new_strategy, &new_persist, &new_threshold) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file space strategy")
+
+ /* Get value(s) */
+ if(strategy) {
+ switch(new_strategy) {
+
+ case H5F_FSPACE_STRATEGY_FSM_AGGR:
+ if(new_persist)
+ *strategy = H5F_FILE_SPACE_ALL_PERSIST;
+ else
+ *strategy = H5F_FILE_SPACE_ALL;
+ break;
+
+ case H5F_FSPACE_STRATEGY_AGGR:
+ *strategy = H5F_FILE_SPACE_AGGR_VFD;
+ break;
+
+ case H5F_FSPACE_STRATEGY_NONE:
+ *strategy = H5F_FILE_SPACE_VFD;
+ break;
+
+ case H5F_FSPACE_STRATEGY_PAGE:
+ case H5F_FSPACE_STRATEGY_NTYPES:
+ default:
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file space strategy")
+ }
+ }
+
+ if(threshold)
+ *threshold = new_threshold;
+
+done:
+ FUNC_LEAVE_API(ret_value)
+} /* H5Pget_file_space() */
#endif /* H5_NO_DEPRECATED_SYMBOLS */
diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c
index 7ea0fe0..7865fdf 100644
--- a/src/H5Pfapl.c
+++ b/src/H5Pfapl.c
@@ -80,18 +80,18 @@
#define H5F_ACS_PREEMPT_READ_CHUNKS_DEC H5P__decode_double
/* Definition for threshold for alignment */
#define H5F_ACS_ALIGN_THRHD_SIZE sizeof(hsize_t)
-#define H5F_ACS_ALIGN_THRHD_DEF 1
+#define H5F_ACS_ALIGN_THRHD_DEF H5F_ALIGN_THRHD_DEF
#define H5F_ACS_ALIGN_THRHD_ENC H5P__encode_hsize_t
#define H5F_ACS_ALIGN_THRHD_DEC H5P__decode_hsize_t
/* Definition for alignment */
#define H5F_ACS_ALIGN_SIZE sizeof(hsize_t)
-#define H5F_ACS_ALIGN_DEF 1
+#define H5F_ACS_ALIGN_DEF H5F_ALIGN_DEF
#define H5F_ACS_ALIGN_ENC H5P__encode_hsize_t
#define H5F_ACS_ALIGN_DEC H5P__decode_hsize_t
/* Definition for minimum metadata allocation block size (when
aggregating metadata allocations. */
#define H5F_ACS_META_BLOCK_SIZE_SIZE sizeof(hsize_t)
-#define H5F_ACS_META_BLOCK_SIZE_DEF 2048
+#define H5F_ACS_META_BLOCK_SIZE_DEF H5F_META_BLOCK_SIZE_DEF
#define H5F_ACS_META_BLOCK_SIZE_ENC H5P__encode_hsize_t
#define H5F_ACS_META_BLOCK_SIZE_DEC H5P__decode_hsize_t
/* Definition for maximum sieve buffer size (when data sieving
@@ -103,7 +103,7 @@
/* Definition for minimum "small data" allocation block size (when
aggregating "small" raw data allocations. */
#define H5F_ACS_SDATA_BLOCK_SIZE_SIZE sizeof(hsize_t)
-#define H5F_ACS_SDATA_BLOCK_SIZE_DEF 2048
+#define H5F_ACS_SDATA_BLOCK_SIZE_DEF H5F_SDATA_BLOCK_SIZE_DEF
#define H5F_ACS_SDATA_BLOCK_SIZE_ENC H5P__encode_hsize_t
#define H5F_ACS_SDATA_BLOCK_SIZE_DEC H5P__decode_hsize_t
/* Definition for garbage-collect references */
@@ -231,6 +231,21 @@
#define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_ENC H5P__facc_cache_image_config_enc
#define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_DEC H5P__facc_cache_image_config_dec
#define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_CMP H5P__facc_cache_image_config_cmp
+/* Definition for total size of page buffer(bytes) */
+#define H5F_ACS_PAGE_BUFFER_SIZE_SIZE sizeof(size_t)
+#define H5F_ACS_PAGE_BUFFER_SIZE_DEF 0
+#define H5F_ACS_PAGE_BUFFER_SIZE_ENC H5P__encode_size_t
+#define H5F_ACS_PAGE_BUFFER_SIZE_DEC H5P__decode_size_t
+/* Definition for minimum metadata size of page buffer(bytes) */
+#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_SIZE sizeof(unsigned)
+#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEF 0
+#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_ENC H5P__encode_unsigned
+#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEC H5P__decode_unsigned
+/* Definition for minimum raw data size of page buffer(bytes) */
+#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_SIZE sizeof(unsigned)
+#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF 0
+#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_ENC H5P__encode_unsigned
+#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEC H5P__decode_unsigned
/******************/
@@ -359,6 +374,9 @@ static const H5P_coll_md_read_flag_t H5F_def_coll_md_read_flag_g = H5F_ACS_COLL_
static const hbool_t H5F_def_coll_md_write_flag_g = H5F_ACS_COLL_MD_WRITE_FLAG_DEF; /* Default setting for the collective metedata write flag */
#endif /* H5_HAVE_PARALLEL */
static const H5AC_cache_image_config_t H5F_def_mdc_initCacheImageCfg_g = H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_DEF; /* Default metadata cache image settings */
+static const size_t H5F_def_page_buf_size_g = H5F_ACS_PAGE_BUFFER_SIZE_DEF; /* Default page buffer size */
+static const unsigned H5F_def_page_buf_min_meta_perc_g = H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEF; /* Default page buffer minimum metadata size */
+static const unsigned H5F_def_page_buf_min_raw_perc_g = H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF; /* Default page buffer minumum raw data size */
/*-------------------------------------------------------------------------
@@ -574,6 +592,22 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass)
NULL, NULL, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_CMP, NULL) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
+ /* Register the size of the page buffer size */
+ if(H5P_register_real(pclass, H5F_ACS_PAGE_BUFFER_SIZE_NAME, H5F_ACS_PAGE_BUFFER_SIZE_SIZE, &H5F_def_page_buf_size_g,
+ NULL, NULL, NULL, H5F_ACS_PAGE_BUFFER_SIZE_ENC, H5F_ACS_PAGE_BUFFER_SIZE_DEC,
+ NULL, NULL, NULL, NULL) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
+ /* Register the size of the page buffer minimum metadata size */
+ if(H5P_register_real(pclass, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_SIZE, &H5F_def_page_buf_min_meta_perc_g,
+ NULL, NULL, NULL, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_ENC, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEC,
+ NULL, NULL, NULL, NULL) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
+ /* Register the size of the page buffer minimum raw data size */
+ if(H5P_register_real(pclass, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_SIZE, &H5F_def_page_buf_min_raw_perc_g,
+ NULL, NULL, NULL, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_ENC, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEC,
+ NULL, NULL, NULL, NULL) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
+
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5P__facc_reg_prop() */
@@ -4681,3 +4715,93 @@ done:
} /* end H5Pget_coll_metadata_write() */
#endif /* H5_HAVE_PARALLEL */
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pset_page_buffer_size
+ *
+ * Purpose: Set the maximum page buffering size. This has to be a
+ * multiple of the page allocation size which must be enabled;
+ * otherwise file create/open will fail.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Mohamad Chaarawi
+ * June 2015
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pset_page_buffer_size(hid_t plist_id, size_t buf_size, unsigned min_meta_perc, unsigned min_raw_perc)
+{
+ H5P_genplist_t *plist; /* Property list pointer */
+ herr_t ret_value = SUCCEED; /* return value */
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE4("e", "izIuIu", plist_id, buf_size, min_meta_perc, min_raw_perc);
+
+ /* Get the plist structure */
+ if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+ HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
+
+ if(min_meta_perc > 100)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Minimum metadata fractions must be between 0 and 100 inclusive")
+ if(min_raw_perc > 100)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Minimum rawdata fractions must be between 0 and 100 inclusive")
+
+ if(min_meta_perc + min_raw_perc > 100)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Sum of minimum metadata and raw data fractions can't be bigger than 100");
+
+ /* Set size */
+ if(H5P_set(plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &buf_size) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTSET,FAIL, "can't set page buffer size")
+ if(H5P_set(plist, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, &min_meta_perc) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTSET,FAIL, "can't set percentage of min metadata entries")
+ if(H5P_set(plist, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, &min_raw_perc) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTSET,FAIL, "can't set percentage of min rawdata entries")
+
+done:
+ FUNC_LEAVE_API(ret_value)
+} /* end H5Pset_page_buffer_size() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pget_page_buffer_size
+ *
+ * Purpose: Retrieves the maximum page buffer size.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Mohamad Chaarawi
+ * June 2015
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pget_page_buffer_size(hid_t plist_id, size_t *buf_size, unsigned *min_meta_perc, unsigned *min_raw_perc)
+{
+ H5P_genplist_t *plist; /* Property list pointer */
+ herr_t ret_value = SUCCEED; /* return value */
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE4("e", "i*z*Iu*Iu", plist_id, buf_size, min_meta_perc, min_raw_perc);
+
+ /* Get the plist structure */
+ if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS)))
+ HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
+
+ /* Get size */
+
+ if(buf_size)
+ if(H5P_get(plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, buf_size) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET,FAIL, "can't get page buffer size")
+ if(min_meta_perc)
+ if(H5P_get(plist, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, min_meta_perc) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET,FAIL, "can't get page buffer minimum metadata percent")
+ if(min_raw_perc)
+ if(H5P_get(plist, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, min_raw_perc) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET,FAIL, "can't get page buffer minimum raw data percent")
+
+done:
+ FUNC_LEAVE_API(ret_value)
+} /* end H5Pget_page_buffer_size() */
+
diff --git a/src/H5Pfcpl.c b/src/H5Pfcpl.c
index d451982..c778801 100644
--- a/src/H5Pfcpl.c
+++ b/src/H5Pfcpl.c
@@ -97,14 +97,23 @@
#define H5F_CRT_SHMSG_BTREE_MIN_ENC H5P__encode_unsigned
#define H5F_CRT_SHMSG_BTREE_MIN_DEC H5P__decode_unsigned
/* Definitions for file space handling strategy */
-#define H5F_CRT_FILE_SPACE_STRATEGY_SIZE sizeof(unsigned)
-#define H5F_CRT_FILE_SPACE_STRATEGY_DEF H5F_FILE_SPACE_STRATEGY_DEF
-#define H5F_CRT_FILE_SPACE_STRATEGY_ENC H5P__encode_unsigned
-#define H5F_CRT_FILE_SPACE_STRATEGY_DEC H5P__decode_unsigned
-#define H5F_CRT_FREE_SPACE_THRESHOLD_SIZE sizeof(hsize_t)
-#define H5F_CRT_FREE_SPACE_THRESHOLD_DEF H5F_FREE_SPACE_THRESHOLD_DEF
-#define H5F_CRT_FREE_SPACE_THRESHOLD_ENC H5P__encode_hsize_t
-#define H5F_CRT_FREE_SPACE_THRESHOLD_DEC H5P__decode_hsize_t
+#define H5F_CRT_FILE_SPACE_STRATEGY_SIZE sizeof(H5F_fspace_strategy_t)
+#define H5F_CRT_FILE_SPACE_STRATEGY_DEF H5F_FILE_SPACE_STRATEGY_DEF
+#define H5F_CRT_FILE_SPACE_STRATEGY_ENC H5P__fcrt_fspace_strategy_enc
+#define H5F_CRT_FILE_SPACE_STRATEGY_DEC H5P__fcrt_fspace_strategy_dec
+#define H5F_CRT_FREE_SPACE_PERSIST_SIZE sizeof(hbool_t)
+#define H5F_CRT_FREE_SPACE_PERSIST_DEF H5F_FREE_SPACE_PERSIST_DEF
+#define H5F_CRT_FREE_SPACE_PERSIST_ENC H5P__encode_hbool_t
+#define H5F_CRT_FREE_SPACE_PERSIST_DEC H5P__decode_hbool_t
+#define H5F_CRT_FREE_SPACE_THRESHOLD_SIZE sizeof(hsize_t)
+#define H5F_CRT_FREE_SPACE_THRESHOLD_DEF H5F_FREE_SPACE_THRESHOLD_DEF
+#define H5F_CRT_FREE_SPACE_THRESHOLD_ENC H5P__encode_hsize_t
+#define H5F_CRT_FREE_SPACE_THRESHOLD_DEC H5P__decode_hsize_t
+/* Definitions for file space page size in support of level-2 page caching */
+#define H5F_CRT_FILE_SPACE_PAGE_SIZE_SIZE sizeof(hsize_t)
+#define H5F_CRT_FILE_SPACE_PAGE_SIZE_DEF H5F_FILE_SPACE_PAGE_SIZE_DEF
+#define H5F_CRT_FILE_SPACE_PAGE_SIZE_ENC H5P__encode_hsize_t
+#define H5F_CRT_FILE_SPACE_PAGE_SIZE_DEC H5P__decode_hsize_t
/******************/
@@ -131,6 +140,8 @@ static herr_t H5P__fcrt_shmsg_index_types_enc(const void *value, void **_pp, siz
static herr_t H5P__fcrt_shmsg_index_types_dec(const void **_pp, void *value);
static herr_t H5P__fcrt_shmsg_index_minsize_enc(const void *value, void **_pp, size_t *size);
static herr_t H5P__fcrt_shmsg_index_minsize_dec(const void **_pp, void *value);
+static herr_t H5P__fcrt_fspace_strategy_enc(const void *value, void **_pp, size_t *size);
+static herr_t H5P__fcrt_fspace_strategy_dec(const void **_pp, void *_value);
/*********************/
@@ -178,8 +189,10 @@ static const unsigned H5F_def_sohm_index_flags_g[H5O_SHMESG_MAX_NINDEXES] = H
static const unsigned H5F_def_sohm_index_minsizes_g[H5O_SHMESG_MAX_NINDEXES] = H5F_CRT_SHMSG_INDEX_MINSIZE_DEF;
static const unsigned H5F_def_sohm_list_max_g = H5F_CRT_SHMSG_LIST_MAX_DEF;
static const unsigned H5F_def_sohm_btree_min_g = H5F_CRT_SHMSG_BTREE_MIN_DEF;
-static const unsigned H5F_def_file_space_strategy_g = H5F_CRT_FILE_SPACE_STRATEGY_DEF;
+static const H5F_fspace_strategy_t H5F_def_file_space_strategy_g = H5F_CRT_FILE_SPACE_STRATEGY_DEF;
+static const hbool_t H5F_def_free_space_persist_g = H5F_CRT_FREE_SPACE_PERSIST_DEF;
static const hsize_t H5F_def_free_space_threshold_g = H5F_CRT_FREE_SPACE_THRESHOLD_DEF;
+static const hsize_t H5F_def_file_space_page_size_g = H5F_CRT_FILE_SPACE_PAGE_SIZE_DEF;
@@ -267,12 +280,24 @@ H5P_fcrt_reg_prop(H5P_genclass_t *pclass)
NULL, NULL, NULL, NULL) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
+ /* Register the free-space persist flag */
+ if(H5P_register_real(pclass, H5F_CRT_FREE_SPACE_PERSIST_NAME, H5F_CRT_FREE_SPACE_PERSIST_SIZE, &H5F_def_free_space_persist_g,
+ NULL, NULL, NULL, H5F_CRT_FREE_SPACE_PERSIST_ENC, H5F_CRT_FREE_SPACE_PERSIST_DEC,
+ NULL, NULL, NULL, NULL) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
+
/* Register the free space section threshold */
if(H5P_register_real(pclass, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, H5F_CRT_FREE_SPACE_THRESHOLD_SIZE, &H5F_def_free_space_threshold_g,
NULL, NULL, NULL, H5F_CRT_FREE_SPACE_THRESHOLD_ENC, H5F_CRT_FREE_SPACE_THRESHOLD_DEC,
NULL, NULL, NULL, NULL) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
+ /* Register the file space page size */
+ if(H5P_register_real(pclass, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, H5F_CRT_FILE_SPACE_PAGE_SIZE_SIZE, &H5F_def_file_space_page_size_g,
+ NULL, NULL, NULL, H5F_CRT_FILE_SPACE_PAGE_SIZE_ENC, H5F_CRT_FILE_SPACE_PAGE_SIZE_DEC,
+ NULL, NULL, NULL, NULL) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class")
+
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5P_fcrt_reg_prop() */
@@ -1246,15 +1271,13 @@ H5Pget_shared_mesg_phase_change(hid_t plist_id, unsigned *max_list, unsigned *mi
if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_CREATE)))
HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID");
- /* Get value */
- if (max_list) {
+ /* Get value(s) */
+ if(max_list)
if(H5P_get(plist, H5F_CRT_SHMSG_LIST_MAX_NAME, max_list) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get list maximum");
- }
- if (min_btree) {
+ if(min_btree)
if(H5P_get(plist, H5F_CRT_SHMSG_BTREE_MIN_NAME, min_btree) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get SOHM information");
- }
done:
FUNC_LEAVE_API(ret_value)
@@ -1262,15 +1285,12 @@ done:
/*-------------------------------------------------------------------------
- * Function: H5Pset_file_space
+ * Function: H5Pset_file_space_strategy
*
- * Purpose: Sets the strategy that the library employs in managing file space.
- * If strategy is zero, the property is not changed; the existing
- * strategy is retained.
- * Sets the threshold value that the file's free space
- * manager(s) will use to track free space sections.
- * If threshold is zero, the property is not changed; the existing
- * threshold is retained.
+ * Purpose: Sets the "strategy" that the library employs in managing file space
+ * Sets the "persist" value as to persist free-space or not
+ * Sets the "threshold" value that the free space manager(s) will use to track free space sections.
+ * Ignore "persist" and "threshold" for strategies that do not use free-space managers
*
* Return: Non-negative on success/Negative on failure
*
@@ -1279,15 +1299,16 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
-H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold)
+H5Pset_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t strategy, hbool_t persist, hsize_t threshold)
{
H5P_genplist_t *plist; /* Property list pointer */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
- H5TRACE3("e", "iFfh", plist_id, strategy, threshold);
+ H5TRACE4("e", "iFfbh", plist_id, strategy, persist, threshold);
- if((unsigned)strategy >= H5F_FILE_SPACE_NTYPES)
+ /* Check arguments */
+ if(strategy >= H5F_FSPACE_STRATEGY_NTYPES)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid strategy")
/* Get the plist structure */
@@ -1295,24 +1316,28 @@ H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t thresh
HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
/* Set value(s), if non-zero */
- if(strategy)
- if(H5P_set(plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &strategy) < 0)
- HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set file space strategy")
- if(threshold)
- if(H5P_set(plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &threshold) < 0)
- HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set free-space threshold")
+ if(H5P_set(plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &strategy) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set file space strategy")
+
+ /* Ignore persist and threshold settings for strategies that do not use FSM */
+ if(strategy == H5F_FSPACE_STRATEGY_FSM_AGGR || strategy == H5F_FSPACE_STRATEGY_PAGE) {
+ if(H5P_set(plist, H5F_CRT_FREE_SPACE_PERSIST_NAME, &persist) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set free-space persisting status")
+
+ if(H5P_set(plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &threshold) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set free-space threshold")
+ } /* end if */
done:
FUNC_LEAVE_API(ret_value)
-} /* H5Pset_file_space() */
+} /* H5Pset_file_space_strategy() */
/*-------------------------------------------------------------------------
- * Function: H5Pget_file_space
+ * Function: H5Pget_file_space_strategy
*
- * Purpose: Retrieves the strategy that the library uses in managing file space.
- * Retrieves the threshold value that the file's free space
- * managers use to track free space sections.
+ * Purpose: Retrieves the strategy, persist, and threshold that the library
+ * uses in managing file space.
*
* Return: Non-negative on success/Negative on failure
*
@@ -1321,13 +1346,13 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
-H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold)
+H5Pget_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t *strategy, hbool_t *persist, hsize_t *threshold)
{
H5P_genplist_t *plist; /* Property list pointer */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(FAIL)
- H5TRACE3("e", "i*Ff*h", plist_id, strategy, threshold);
+ H5TRACE4("e", "i*Ff*b*h", plist_id, strategy, persist, threshold);
/* Get the plist structure */
if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_CREATE)))
@@ -1337,11 +1362,158 @@ H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *thre
if(strategy)
if(H5P_get(plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, strategy) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file space strategy")
+ if(persist)
+ if(H5P_get(plist, H5F_CRT_FREE_SPACE_PERSIST_NAME, persist) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get free-space persisting status")
if(threshold)
if(H5P_get(plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, threshold) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get free-space threshold")
done:
FUNC_LEAVE_API(ret_value)
-} /* H5Pget_file_space() */
+} /* H5Pget_file_space_strategy() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5P__fcrt_fspace_strategy_enc
+ *
+ * Purpose: Callback routine which is called whenever the free-space
+ * strategy property in the file creation property list
+ * is encoded.
+ *
+ * Return: Success: Non-negative
+ * Failure: Negative
+ *
+ * Programmer: Quincey Koziol
+ * Friday, December 27, 2013
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5P__fcrt_fspace_strategy_enc(const void *value, void **_pp, size_t *size)
+{
+ const H5F_fspace_strategy_t *strategy = (const H5F_fspace_strategy_t *)value; /* Create local alias for values */
+ uint8_t **pp = (uint8_t **)_pp;
+
+ FUNC_ENTER_STATIC_NOERR
+
+ /* Sanity check */
+ HDassert(strategy);
+ HDassert(size);
+
+ if(NULL != *pp)
+ /* Encode free-space strategy */
+ *(*pp)++ = (uint8_t)*strategy;
+
+ /* Size of free-space strategy */
+ (*size)++;
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5P__fcrt_fspace_strategy_enc() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5P__fcrt_fspace_strategy_dec
+ *
+ * Purpose: Callback routine which is called whenever the free-space
+ * strategy property in the file creation property list
+ * is decoded.
+ *
+ * Return: Success: Non-negative
+ * Failure: Negative
+ *
+ * Programmer: Quincey Koziol
+ * Friday, December 27, 2013
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5P__fcrt_fspace_strategy_dec(const void **_pp, void *_value)
+{
+ H5F_fspace_strategy_t *strategy = (H5F_fspace_strategy_t *)_value; /* Free-space strategy */
+ const uint8_t **pp = (const uint8_t **)_pp;
+
+ FUNC_ENTER_STATIC_NOERR
+
+ /* Sanity checks */
+ HDassert(pp);
+ HDassert(*pp);
+ HDassert(strategy);
+
+ /* Decode free-space strategy */
+ *strategy = (H5F_fspace_strategy_t)*(*pp)++;
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5P__fcrt_fspace_strategy_dec() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pset_file_space_page_size
+ *
+ * Purpose: Sets the file space page size for paged aggregation.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Vailin Choi; August 2012
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pset_file_space_page_size(hid_t plist_id, hsize_t fsp_size)
+{
+ H5P_genplist_t *plist; /* Property list pointer */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE2("e", "ih", plist_id, fsp_size);
+
+ /* Get the plist structure */
+ if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_CREATE)))
+ HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
+
+ if(fsp_size < H5F_FILE_SPACE_PAGE_SIZE_MIN)
+ HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "cannot set file space page size to less than 512")
+
+ /* Set the value*/
+ if(H5P_set(plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, &fsp_size) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set file space page size")
+
+done:
+ FUNC_LEAVE_API(ret_value)
+} /* H5Pset_file_space_page_size() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pget_file_space_page_size
+ *
+ * Purpose: Retrieves the file space page size for aggregating small metadata
+ * or raw data in the parameter "fsp_size".
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Vailin Choi; August 2012
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pget_file_space_page_size(hid_t plist_id, hsize_t *fsp_size)
+{
+ H5P_genplist_t *plist; /* Property list pointer */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE2("e", "i*h", plist_id, fsp_size);
+
+ /* Get the plist structure */
+ if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_CREATE)))
+ HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
+
+ /* Get value */
+ if(fsp_size)
+ if(H5P_get(plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, fsp_size) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file space page size")
+
+done:
+ FUNC_LEAVE_API(ret_value)
+} /* H5Pget_file_space_page_size() */
diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h
index 5aa8301..50b9eb0 100644
--- a/src/H5Ppublic.h
+++ b/src/H5Ppublic.h
@@ -300,8 +300,10 @@ H5_DLL herr_t H5Pset_shared_mesg_index(hid_t plist_id, unsigned index_num, unsig
H5_DLL herr_t H5Pget_shared_mesg_index(hid_t plist_id, unsigned index_num, unsigned *mesg_type_flags, unsigned *min_mesg_size);
H5_DLL herr_t H5Pset_shared_mesg_phase_change(hid_t plist_id, unsigned max_list, unsigned min_btree);
H5_DLL herr_t H5Pget_shared_mesg_phase_change(hid_t plist_id, unsigned *max_list, unsigned *min_btree);
-H5_DLL herr_t H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold);
-H5_DLL herr_t H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold);
+H5_DLL herr_t H5Pset_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t strategy, hbool_t persist, hsize_t threshold);
+H5_DLL herr_t H5Pget_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t *strategy, hbool_t *persist, hsize_t *threshold);
+H5_DLL herr_t H5Pset_file_space_page_size(hid_t plist_id, hsize_t fsp_size);
+H5_DLL herr_t H5Pget_file_space_page_size(hid_t plist_id, hsize_t *fsp_size);
/* File access property list (FAPL) routines */
H5_DLL herr_t H5Pset_alignment(hid_t fapl_id, hsize_t threshold,
@@ -367,6 +369,8 @@ H5_DLL herr_t H5Pget_coll_metadata_write(hid_t plist_id, hbool_t *is_collective)
#endif /* H5_HAVE_PARALLEL */
H5_DLL herr_t H5Pset_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t *config_ptr);
H5_DLL herr_t H5Pget_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t *config_ptr /*out*/);
+H5_DLL herr_t H5Pset_page_buffer_size(hid_t plist_id, size_t buf_size, unsigned min_meta_per, unsigned min_raw_per);
+H5_DLL herr_t H5Pget_page_buffer_size(hid_t plist_id, size_t *buf_size, unsigned *min_meta_per, unsigned *min_raw_per);
/* Dataset creation property list (DCPL) routines */
H5_DLL herr_t H5Pset_layout(hid_t plist_id, H5D_layout_t layout);
@@ -534,6 +538,8 @@ H5_DLL herr_t H5Pget_filter_by_id1(hid_t plist_id, H5Z_filter_t id,
H5_DLL herr_t H5Pget_version(hid_t plist_id, unsigned *boot/*out*/,
unsigned *freelist/*out*/, unsigned *stab/*out*/,
unsigned *shhdr/*out*/);
+H5_DLL herr_t H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold);
+H5_DLL herr_t H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold);
#endif /* H5_NO_DEPRECATED_SYMBOLS */
#ifdef __cplusplus
diff --git a/src/H5Z.c b/src/H5Z.c
index e7a2186..0be0bbe 100644
--- a/src/H5Z.c
+++ b/src/H5Z.c
@@ -626,7 +626,7 @@ H5Z__flush_file_cb(void *obj_ptr, hid_t H5_ATTR_UNUSED obj_id, void H5_ATTR_UNUS
/* Call the flush routine for mounted file hierarchies. Do a global flush
* if the file is opened for write */
if(H5F_ACC_RDWR & H5F_INTENT((H5F_t *)obj_ptr)) {
- if(H5F_flush_mounts((H5F_t *)obj_ptr, H5AC_ind_read_dxpl_id) < 0)
+ if(H5F_flush_mounts((H5F_t *)obj_ptr, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id) < 0)
HGOTO_ERROR(H5E_PLINE, H5E_CANTFLUSH, FAIL, "unable to flush file hierarchy")
} /* end if */
diff --git a/src/H5err.txt b/src/H5err.txt
index 48069a5..b2ff3bd 100644
--- a/src/H5err.txt
+++ b/src/H5err.txt
@@ -77,6 +77,7 @@ MAJOR, H5E_SOHM, Shared Object Header Messages
MAJOR, H5E_EARRAY, Extensible Array
MAJOR, H5E_FARRAY, Fixed Array
MAJOR, H5E_PLUGIN, Plugin for dynamically loaded library
+MAJOR, H5E_PAGEBUF, Page Buffering
MAJOR, H5E_NONE_MAJOR, No error
# Sections (for grouping minor errors)
diff --git a/src/H5private.h b/src/H5private.h
index 00de96c..61ecc5b 100644
--- a/src/H5private.h
+++ b/src/H5private.h
@@ -1386,7 +1386,13 @@ typedef off_t h5_stat_size_t;
#ifndef HDstrtol
#define HDstrtol(S,R,N) strtol(S,R,N)
#endif /* HDstrtol */
-H5_DLL int64_t HDstrtoll (const char *s, const char **rest, int base);
+#ifndef HDstrtoll
+ #ifdef H5_HAVE_STRTOLL
+ #define HDstrtoll(S,R,N) strtoll(S,R,N)
+ #else
+ H5_DLL int64_t HDstrtoll (const char *s, const char **rest, int base);
+ #endif /* H5_HAVE_STRTOLL */
+#endif /* HDstrtoll */
#ifndef HDstrtoul
#define HDstrtoul(S,R,N) strtoul(S,R,N)
#endif /* HDstrtoul */
diff --git a/src/H5system.c b/src/H5system.c
index ac323c0..1f92e19 100644
--- a/src/H5system.c
+++ b/src/H5system.c
@@ -32,10 +32,10 @@
/***********/
/* Headers */
/***********/
-#include "H5private.h" /* Generic Functions */
-#include "H5Eprivate.h" /* Error handling */
-#include "H5Fprivate.h" /* File access */
-#include "H5MMprivate.h" /* Memory management */
+#include "H5private.h" /* Generic Functions */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5Fprivate.h" /* File access */
+#include "H5MMprivate.h" /* Memory management */
/****************/
@@ -474,6 +474,7 @@ HDfprintf(FILE *stream, const char *fmt, ...)
*
*-------------------------------------------------------------------------
*/
+#ifndef HDstrtoll
int64_t
HDstrtoll(const char *s, const char **rest, int base)
{
@@ -549,7 +550,7 @@ HDstrtoll(const char *s, const char **rest, int base)
*rest = s;
return acc;
} /* end HDstrtoll() */
-
+#endif
/*-------------------------------------------------------------------------
* Function: HDrand/HDsrand
@@ -604,7 +605,7 @@ void HDsrand(unsigned int seed)
#ifdef H5_HAVE_FCNTL
int
Pflock(int fd, int operation) {
-
+
struct flock flk;
/* Set the lock type */
@@ -649,18 +650,18 @@ Nflock(int H5_ATTR_UNUSED fd, int H5_ATTR_UNUSED operation) {
/*-------------------------------------------------------------------------
- * Function: H5_make_time
+ * Function: H5_make_time
*
- * Purpose: Portability routine to abstract converting a 'tm' struct into
- * a time_t value.
+ * Purpose: Portability routine to abstract converting a 'tm' struct into
+ * a time_t value.
*
- * Note: This is a little problematic because mktime() operates on
- * local times. We convert to local time and then figure out the
- * adjustment based on the local time zone and daylight savings
- * setting.
+ * Note: This is a little problematic because mktime() operates on
+ * local times. We convert to local time and then figure out the
+ * adjustment based on the local time zone and daylight savings
+ * setting.
*
- * Return: Success: The value of timezone
- * Failure: -1
+ * Return: Success: The value of timezone
+ * Failure: -1
*
* Programmer: Quincey Koziol
* November 18, 2015
@@ -1138,7 +1139,7 @@ H5_combine_path(const char* path1, const char* path2, char **full_name /*out*/)
if(NULL == (*full_name = (char *)H5MM_strdup(path2)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed")
- } /* end if */
+ } /* end if */
else if(H5_CHECK_ABS_PATH(path2)) {
/* On windows path2 is a path absolute name */
diff --git a/src/H5trace.c b/src/H5trace.c
index 44b2ed5..48c357a 100644
--- a/src/H5trace.c
+++ b/src/H5trace.c
@@ -895,32 +895,28 @@ H5_trace(const double *returning, const char *func, const char *type, ...)
fprintf(out, "NULL");
} /* end if */
else {
- H5F_file_space_type_t fs_type = (H5F_file_space_type_t)va_arg(ap, int);
+ H5F_fspace_strategy_t fs_strategy = (H5F_fspace_strategy_t)va_arg(ap, int);
- switch(fs_type) {
- case H5F_FILE_SPACE_DEFAULT:
- fprintf(out, "H5F_FILE_SPACE_DEFAULT");
+ switch(fs_strategy) {
+ case H5F_FSPACE_STRATEGY_FSM_AGGR:
+ fprintf(out, "H5F_FSPACE_STRATEGY_FSM_AGGR");
break;
- case H5F_FILE_SPACE_ALL_PERSIST:
- fprintf(out, "H5F_FILE_SPACE_ALL_PERSIST");
+ case H5F_FSPACE_STRATEGY_PAGE:
+ fprintf(out, "H5F_FSPACE_STRATEGY_PAGE");
break;
- case H5F_FILE_SPACE_ALL:
- fprintf(out, "H5F_FILE_SPACE_ALL");
+ case H5F_FSPACE_STRATEGY_AGGR:
+ fprintf(out, "H5F_FSPACE_STRATEGY_AGGR");
break;
- case H5F_FILE_SPACE_AGGR_VFD:
- fprintf(out, "H5F_FILE_SPACE_AGGR_VFD");
+ case H5F_FSPACE_STRATEGY_NONE:
+ fprintf(out, "H5F_FSPACE_STRATEGY_NONE");
break;
- case H5F_FILE_SPACE_VFD:
- fprintf(out, "H5F_FILE_SPACE_VFD");
- break;
-
- case H5F_FILE_SPACE_NTYPES:
+ case H5F_FSPACE_STRATEGY_NTYPES:
default:
- fprintf(out, "%ld", (long)fs_type);
+ fprintf(out, "%ld", (long)fs_strategy);
break;
} /* end switch */
} /* end else */
@@ -1003,6 +999,15 @@ H5_trace(const double *returning, const char *func, const char *type, ...)
} /* end else */
break;
+ case 't':
+ if(ptr) {
+ if(vp)
+ fprintf(out, "0x%lx", (unsigned long)vp);
+ else
+ fprintf(out, "NULL");
+ } /* end if */
+ break;
+
case 'v':
if(ptr) {
if(vp)
diff --git a/src/H5win32defs.h b/src/H5win32defs.h
index b419f06..e005b51 100644
--- a/src/H5win32defs.h
+++ b/src/H5win32defs.h
@@ -23,6 +23,11 @@
*
*/
+/*
+ * _MSC_VER = 1900 VS2015
+ * _MSC_VER = 1800 VS2013
+ * _MSC_VER = 1700 VS2012
+ */
#ifdef H5_HAVE_WIN32_API
typedef struct _stati64 h5_stat_t;
@@ -54,13 +59,22 @@ typedef __int64 h5_stat_size_t;
#define HDsleep(S) Sleep(S*1000)
#define HDstat(S,B) _stati64(S,B)
#define HDstrcasecmp(A,B) _stricmp(A,B)
-#define HDstrtoull(S,R,N) _strtoui64(S,R,N)
#define HDstrdup(S) _strdup(S)
#define HDtzset() _tzset()
#define HDunlink(S) _unlink(S)
#define HDwrite(F,M,Z) _write(F,M,Z)
#ifdef H5_HAVE_VISUAL_STUDIO
+
+#if (_MSC_VER < 1800)
+ #ifndef H5_HAVE_STRTOLL
+ #define HDstrtoll(S,R,N) _strtoi64(S,R,N)
+ #endif /* H5_HAVE_STRTOLL */
+ #ifndef H5_HAVE_STRTOULL
+ #define HDstrtoull(S,R,N) _strtoui64(S,R,N)
+ #endif /* H5_HAVE_STRTOULL */
+#endif /* MSC_VER < 1800 */
+
/*
* The (void*) cast just avoids a compiler warning in H5_HAVE_VISUAL_STUDIO
*/
diff --git a/src/Makefile.am b/src/Makefile.am
index 69b54b4..59df4c2 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -60,7 +60,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \
H5F.c H5Fint.c H5Faccum.c H5Fcwfs.c \
H5Fdbg.c H5Fdeprec.c H5Fefc.c H5Ffake.c H5Fio.c \
H5Fmount.c H5Fquery.c \
- H5Fsfile.c H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \
+ H5Fsfile.c H5Fspace.c H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \
H5FA.c H5FAcache.c H5FAdbg.c H5FAdblock.c H5FAdblkpage.c H5FAhdr.c \
H5FAint.c H5FAstat.c H5FAtest.c \
H5FD.c H5FDcore.c \
@@ -98,6 +98,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \
H5Pfapl.c H5Pfcpl.c H5Pfmpl.c \
H5Pgcpl.c H5Pint.c \
H5Plapl.c H5Plcpl.c H5Pocpl.c H5Pocpypl.c H5Pstrcpl.c H5Ptest.c \
+ H5PB.c \
H5PL.c \
H5R.c H5Rdeprec.c \
H5UC.c \