diff options
| author | Jordan Henderson <jhenderson@hdfgroup.org> | 2017-03-30 17:58:20 (GMT) |
|---|---|---|
| committer | Jordan Henderson <jhenderson@hdfgroup.org> | 2017-03-30 17:58:20 (GMT) |
| commit | 6bc14164e6707ab201234e02d8eadf719a0caa05 (patch) | |
| tree | 55b8773c59982a455635c15377ed43be0a6b9980 /src | |
| parent | 588e829b740e600484c965e7d2f90913c45de89f (diff) | |
| parent | 3a01afc0b15fc63a0fe6800ae3f8f55fc6bfed0a (diff) | |
| download | hdf5-6bc14164e6707ab201234e02d8eadf719a0caa05.zip hdf5-6bc14164e6707ab201234e02d8eadf719a0caa05.tar.gz hdf5-6bc14164e6707ab201234e02d8eadf719a0caa05.tar.bz2 | |
Merge branch 'feature/parallel_filters' of ssh://bitbucket.hdfgroup.org:7999/~jhenderson/hdf5.git into feature/parallel_filters
Diffstat (limited to 'src')
62 files changed, 8793 insertions, 2472 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ef361b2..9321bbd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -195,6 +195,7 @@ set (H5F_SOURCES ${HDF5_SRC_DIR}/H5Fmpi.c ${HDF5_SRC_DIR}/H5Fquery.c ${HDF5_SRC_DIR}/H5Fsfile.c + ${HDF5_SRC_DIR}/H5Fspace.c ${HDF5_SRC_DIR}/H5Fsuper.c ${HDF5_SRC_DIR}/H5Fsuper_cache.c ${HDF5_SRC_DIR}/H5Ftest.c @@ -504,6 +505,15 @@ set (H5P_HDRS ) IDE_GENERATED_PROPERTIES ("H5P" "${H5P_HDRS}" "${H5P_SOURCES}" ) +set (H5PB_SOURCES + ${HDF5_SRC_DIR}/H5PB.c +) + +set (H5PB_HDRS + ${HDF5_SRC_DIR}/H5PBpkg.h +) +IDE_GENERATED_PROPERTIES ("H5PB" "${H5PB_HDRS}" "${H5PB_SOURCES}" ) + set (H5PL_SOURCES ${HDF5_SRC_DIR}/H5PL.c ) @@ -699,6 +709,7 @@ set (common_SRCS ${H5MP_SOURCES} ${H5O_SOURCES} ${H5P_SOURCES} + ${H5PB_SOURCES} ${H5PL_SOURCES} ${H5R_SOURCES} ${H5UC_SOURCES} @@ -739,6 +750,7 @@ set (H5_PUBLIC_HEADERS ${H5MP_HDRS} ${H5O_HDRS} ${H5P_HDRS} + ${H5PB_HDRS} ${H5PL_HDRS} ${H5R_HDRS} ${H5S_HDRS} @@ -777,6 +789,7 @@ set (H5_PRIVATE_HEADERS ${HDF5_SRC_DIR}/H5MPprivate.h ${HDF5_SRC_DIR}/H5Oprivate.h ${HDF5_SRC_DIR}/H5Pprivate.h + ${HDF5_SRC_DIR}/H5PBprivate.h ${HDF5_SRC_DIR}/H5PLprivate.h ${HDF5_SRC_DIR}/H5UCprivate.h ${HDF5_SRC_DIR}/H5Rprivate.h @@ -353,6 +353,44 @@ H5AC_term_package(void) /*------------------------------------------------------------------------- + * + * Function: H5AC_cache_image_pending() + * + * Purpose: Debugging function that tests to see if the load of a + * metadata cache image load is pending (i.e. will be executed + * on the next protect or insert) + * + * Returns TRUE if a cache image load is pending, and FALSE + * if not. Throws an assertion failure on error. + * + * Return: TRUE if a cache image load is pending, and FALSE otherwise. + * + * Programmer: John Mainzer, 1/10/17 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +hbool_t +H5AC_cache_image_pending(const H5F_t *f) +{ + H5C_t *cache_ptr; + hbool_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + cache_ptr = f->shared->cache; + + ret_value = H5C_cache_image_pending(cache_ptr); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_cache_image_pending() */ + + +/*------------------------------------------------------------------------- * Function: H5AC_create * * Purpose: Initialize the cache just after a file is opened. The @@ -501,7 +539,7 @@ H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr, H5AC_cache_image_co /* Turn on metadata cache logging, if being used */ if(H5F_USE_MDC_LOGGING(f)) { if(H5C_set_up_logging(f->shared->cache, H5F_MDC_LOG_LOCATION(f), H5F_START_MDC_LOG_ON_ACCESS(f)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "mdc logging setup failed") + HGOTO_ERROR(H5E_CACHE, H5E_CANTINIT, FAIL, "mdc logging setup failed") /* Write the log header regardless of current logging status */ if(H5AC__write_create_cache_log_msg(f->shared->cache) < 0) @@ -510,9 +548,9 @@ H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr, H5AC_cache_image_co /* Set the cache parameters */ if(H5AC_set_cache_auto_resize_config(f->shared->cache, config_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "auto resize configuration failed") + HGOTO_ERROR(H5E_CACHE, H5E_CANTSET, FAIL, "auto resize configuration failed") - /* don't need to get the current H5C image config here since the + /* Don't need to get the current H5C image config here since the * cache has just been created, and thus f->shared->cache->image_ctl * must still set to its initial value (H5C__DEFAULT_CACHE_IMAGE_CTL). * Note that this not true as soon as control returns to the application @@ -522,9 +560,8 @@ H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr, H5AC_cache_image_co int_ci_config.generate_image = image_config_ptr->generate_image; int_ci_config.save_resize_status = image_config_ptr->save_resize_status; int_ci_config.entry_ageout = image_config_ptr->entry_ageout; - if(H5C_set_cache_image_config(f, f->shared->cache, &int_ci_config) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "auto resize configuration failed") + HGOTO_ERROR(H5E_CACHE, H5E_CANTSET, FAIL, "auto resize configuration failed") done: #ifdef H5_HAVE_PARALLEL @@ -836,6 +873,46 @@ done: /*------------------------------------------------------------------------- + * Function: H5AC_force_cache_image_load() + * + * Purpose: On rare occasions, it is necessary to run + * H5MF_tidy_self_referential_fsm_hack() prior to the first + * metadata cache access. This is a problem as if there is a + * cache image at the end of the file, that routine will + * discard it. + * + * We solve this issue by calling this function, which will + * load the cache image and then call + * H5MF_tidy_self_referential_fsm_hack() to discard it. + * + * Return: SUCCEED on success, and FAIL on failure. + * + * Programmer: John Mainzer + * 1/11/17 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_force_cache_image_load(H5F_t *f, hid_t dxpl_id) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->cache); + + if(H5C_force_cache_image_load(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, FAIL, "Can't load cache image") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_force_cache_image_load() */ + + +/*------------------------------------------------------------------------- * Function: H5AC_get_entry_status * * Purpose: Given a file address, determine whether the metadata @@ -3120,6 +3197,48 @@ done: /*------------------------------------------------------------------------- + * Function: H5AC_unsettle_ring() + * + * Purpose: Advise the metadata cache that the specified free space + * manager ring is no longer settled (if it was on entry). + * + * If the target free space manager ring is already + * unsettled, do nothing, and return SUCCEED. + * + * If the target free space manager ring is settled, and + * we are not in the process of a file shutdown, mark + * the ring as unsettled, and return SUCCEED. + * + * If the target free space manager is settled, and we + * are in the process of a file shutdown, post an error + * message, and return FAIL. + * + * Note that this function simply passes the call on to + * the metadata cache proper, and returns the result. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 10/15/16 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_unsettle_ring(H5F_t * f, H5C_ring_t ring) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + if(FAIL == (ret_value = H5C_unsettle_ring(f, ring))) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_unsettle_ring() failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_unsettle_ring() */ + + +/*------------------------------------------------------------------------- * Function: H5AC_remove_entry() * * Purpose: Remove an entry from the cache. Must be not protected, pinned, diff --git a/src/H5ACdbg.c b/src/H5ACdbg.c index 8ca5102..6073288 100644 --- a/src/H5ACdbg.c +++ b/src/H5ACdbg.c @@ -101,6 +101,7 @@ H5AC_stats(const H5F_t *f) FUNC_LEAVE_NOAPI(SUCCEED) } /* H5AC_stats() */ +#ifndef NDEBUG /*------------------------------------------------------------------------- * Function: H5AC_dump_cache @@ -133,6 +134,7 @@ H5AC_dump_cache(const H5F_t *f) done: FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_dump_cache() */ +#endif /* NDEBUG */ /*------------------------------------------------------------------------- diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index 1fe6456..1f8299b 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -451,6 +451,8 @@ H5_DLL herr_t H5AC_validate_config(H5AC_cache_config_t *config_ptr); H5_DLL herr_t H5AC_load_cache_image_on_next_protect(H5F_t *f, haddr_t addr, hsize_t len, hbool_t rw); H5_DLL herr_t H5AC_validate_cache_image_config(H5AC_cache_image_config_t *config_ptr); +H5_DLL hbool_t H5AC_cache_image_pending(const H5F_t *f); +H5_DLL herr_t H5AC_force_cache_image_load(H5F_t * f, hid_t dxpl_id); /* Tag & Ring routines */ H5_DLL herr_t H5AC_tag(hid_t dxpl_id, haddr_t metadata_tag, haddr_t *prev_tag); @@ -464,6 +466,7 @@ H5_DLL herr_t H5AC_set_ring(hid_t dxpl_id, H5AC_ring_t ring, H5P_genplist_t **dx H5AC_ring_t *orig_ring); H5_DLL herr_t H5AC_reset_ring(H5P_genplist_t *dxpl, H5AC_ring_t orig_ring); H5_DLL herr_t H5AC_unsettle_entry_ring(void *entry); +H5_DLL herr_t H5AC_unsettle_ring(H5F_t * f, H5AC_ring_t ring); H5_DLL herr_t H5AC_expunge_tag_type_metadata(H5F_t *f, hid_t dxpl_id, haddr_t tag, int type_id, unsigned flags); H5_DLL herr_t H5AC_get_tag(const void *thing, /*OUT*/ haddr_t *tag); @@ -482,8 +485,8 @@ H5_DLL herr_t H5AC_add_candidate(H5AC_t * cache_ptr, haddr_t addr); /* Debugging functions */ H5_DLL herr_t H5AC_stats(const H5F_t *f); -H5_DLL herr_t H5AC_dump_cache(const H5F_t *f); #ifndef NDEBUG +H5_DLL herr_t H5AC_dump_cache(const H5F_t *f); H5_DLL herr_t H5AC_get_entry_ptr_from_addr(const H5F_t *f, haddr_t addr, void **entry_ptr_ptr); H5_DLL herr_t H5AC_flush_dependency_exists(H5F_t *f, haddr_t parent_addr, @@ -386,6 +386,7 @@ H5C_create(size_t max_cache_size, cache_ptr->cache_full = FALSE; cache_ptr->size_decreased = FALSE; cache_ptr->resize_in_progress = FALSE; + cache_ptr->msic_in_progress = FALSE; (cache_ptr->resize_ctl).version = H5C__CURR_AUTO_SIZE_CTL_VER; (cache_ptr->resize_ctl).rpt_fcn = NULL; @@ -731,7 +732,8 @@ herr_t H5C_prep_for_file_close(H5F_t *f, hid_t dxpl_id) { H5C_t * cache_ptr; - herr_t ret_value = SUCCEED; /* Return value */ + hbool_t image_generated = FALSE; /* Whether a cache image was generated */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -755,9 +757,48 @@ H5C_prep_for_file_close(H5F_t *f, hid_t dxpl_id) HDassert(cache_ptr->pl_len == 0); /* Prepare cache image */ - if(H5C__prep_image_for_file_close(f, dxpl_id) < 0) + if(H5C__prep_image_for_file_close(f, dxpl_id, &image_generated) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, FAIL, "can't create cache image") +#ifdef H5_HAVE_PARALLEL + if(!image_generated && cache_ptr->aux_ptr != NULL && f->shared->fs_persist) { + /* If persistent free space managers are enabled, flushing the + * metadata cache may result in the deletion, insertion, and/or + * dirtying of entries. + * + * This is a problem in PHDF5, as it breaks two invariants of + * our management of the metadata cache across all processes: + * + * 1) Entries will not be dirtied, deleted, inserted, or moved + * during flush in the parallel case. + * + * 2) All processes contain the same set of dirty metadata + * entries on entry to a sync point. + * + * To solve this problem for the persistent free space managers, + * serialize the metadata cache on all processes prior to the + * first sync point on file shutdown. The shutdown warning is + * a convenient location for this call. + * + * This is sufficient since: + * + * 1) FSM settle routines are only invoked on file close. Since + * serialization make the same settle calls as flush on file + * close, and since the close warning is issued after all + * non FSM related space allocations and just before the + * first sync point on close, this call will leave the caches + * in a consistant state across the processes if they were + * consistant before. + * + * 2) Since the FSM settle routines are only invoked once during + * file close, invoking them now will prevent their invocation + * during a flush, and thus avoid any resulting entrie dirties, + * deletions, insertion, or moves during the flush. + */ + if(H5C__serialize_cache(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTSERIALIZE, FAIL, "serialization of the cache failed") + } /* end if */ +#endif /* H5_HAVE_PARALLEL */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -925,12 +966,6 @@ H5C_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, HGOTO_ERROR(H5E_CACHE, H5E_CANTEXPUNGE, FAIL, "Target entry is protected") if(entry_ptr->is_pinned) HGOTO_ERROR(H5E_CACHE, H5E_CANTEXPUNGE, FAIL, "Target entry is pinned") -#ifdef H5_HAVE_PARALLEL - if(entry_ptr->coll_access) { - entry_ptr->coll_access = FALSE; - H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL) - } /* end if */ -#endif /* H5_HAVE_PARALLEL */ /* If we get this far, call H5C__flush_single_entry() with the * H5C__FLUSH_INVALIDATE_FLAG and the H5C__FLUSH_CLEAR_ONLY_FLAG. @@ -1465,10 +1500,15 @@ H5C_insert_entry(H5F_t * f, entry_ptr->prefetched = FALSE; entry_ptr->prefetch_type_id = 0; entry_ptr->age = 0; + entry_ptr->prefetched_dirty = FALSE; #ifndef NDEBUG /* debugging field */ entry_ptr->serialization_count = 0; #endif /* NDEBUG */ + entry_ptr->tl_next = NULL; + entry_ptr->tl_prev = NULL; + entry_ptr->tag_info = NULL; + /* Apply tag to newly inserted entry */ if(H5C__tag_entry(cache_ptr, entry_ptr, dxpl_id) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "Cannot tag metadata entry") @@ -2437,7 +2477,7 @@ H5C_protect(H5F_t * f, marked as collective, and is clean, it is possible that other processes will not have it in its cache and will expect a bcast of the entry from process 0. So process 0 - will bcast the entry to all other ranks. Ranks that do have + will bcast the entry to all other ranks. Ranks that _do_ have the entry in their cache still have to participate in the bcast. */ #ifdef H5_HAVE_PARALLEL @@ -3537,6 +3577,73 @@ done: /*------------------------------------------------------------------------- + * Function: H5C_unsettle_ring() + * + * Purpose: Advise the metadata cache that the specified free space + * manager ring is no longer settled (if it was on entry). + * + * If the target free space manager ring is already + * unsettled, do nothing, and return SUCCEED. + * + * If the target free space manager ring is settled, and + * we are not in the process of a file shutdown, mark + * the ring as unsettled, and return SUCCEED. + * + * If the target free space manager is settled, and we + * are in the process of a file shutdown, post an error + * message, and return FAIL. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 10/15/16 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_unsettle_ring(H5F_t * f, H5C_ring_t ring) +{ + H5C_t * cache_ptr; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->cache); + HDassert((H5C_RING_RDFSM == ring) || (H5C_RING_MDFSM == ring)); + cache_ptr = f->shared->cache; + HDassert(H5C__H5C_T_MAGIC == cache_ptr->magic); + + switch(ring) { + case H5C_RING_RDFSM: + if(cache_ptr->rdfsm_settled) { + if(cache_ptr->close_warning_received) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unexpected rdfsm ring unsettle") + cache_ptr->rdfsm_settled = FALSE; + } /* end if */ + break; + + case H5C_RING_MDFSM: + if(cache_ptr->mdfsm_settled) { + if(cache_ptr->close_warning_received) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unexpected mdfsm ring unsettle") + cache_ptr->mdfsm_settled = FALSE; + } /* end if */ + break; + + default: + HDassert(FALSE); /* this should be un-reachable */ + break; + } /* end switch */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_unsettle_ring() */ + + +/*------------------------------------------------------------------------- * Function: H5C_validate_resize_config() * * Purpose: Run a sanity check on the specified sections of the @@ -4586,7 +4693,7 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f, ( (entry_ptr->type)->id != H5AC_EPOCH_MARKER_ID ) && ( bytes_evicted < eviction_size_limit ) ) { - hbool_t corked = FALSE; + hbool_t skipping_entry = FALSE; HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert( ! (entry_ptr->is_protected) ); @@ -4600,9 +4707,11 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f, prev_is_dirty = prev_ptr->is_dirty; if(entry_ptr->is_dirty ) { + HDassert(!entry_ptr->prefetched_dirty); + /* dirty corked entry is skipped */ if(entry_ptr->tag_info && entry_ptr->tag_info->corked) - corked = TRUE; + skipping_entry = TRUE; else { /* reset entries_removed_counter and * last_entry_removed_ptr prior to the call to @@ -4622,16 +4731,22 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f, restart_scan = TRUE; } /* end else */ } /* end if */ - else { + else if(!entry_ptr->prefetched_dirty) { bytes_evicted += entry_ptr->size; if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0 ) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry") - } + } /* end else-if */ + else { + HDassert(!entry_ptr->is_dirty); + HDassert(entry_ptr->prefetched_dirty); + + skipping_entry = TRUE; + } /* end else */ if(prev_ptr != NULL) { - if(corked) /* dirty corked entry is skipped */ + if(skipping_entry) entry_ptr = prev_ptr; else if(restart_scan || (prev_ptr->is_dirty != prev_is_dirty) || (prev_ptr->next != next_ptr) @@ -4691,10 +4806,10 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f, prev_ptr = entry_ptr->prev; - if(!(entry_ptr->is_dirty)) { + if(!(entry_ptr->is_dirty) && !(entry_ptr->prefetched_dirty)) if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush clean entry") - } /* end if */ + /* just skip the entry if it is dirty, as we can't do * anything with it now since we can't write. * @@ -5999,6 +6114,7 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, hbool_t write_entry; /* internal flag */ hbool_t destroy_entry; /* internal flag */ hbool_t generate_image; /* internal flag */ + hbool_t update_page_buffer; /* internal flag */ hbool_t was_dirty; hbool_t suppress_image_entry_writes = FALSE; hbool_t suppress_image_entry_frees = FALSE; @@ -6024,6 +6140,7 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, del_from_slist_on_destroy = ((flags & H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) != 0); during_flush = ((flags & H5C__DURING_FLUSH_FLAG) != 0); generate_image = ((flags & H5C__GENERATE_IMAGE_FLAG) != 0); + update_page_buffer = ((flags & H5C__UPDATE_PAGE_BUFFER_FLAG) != 0); /* Set the flag for destroying the entry, based on the 'take ownership' * and 'destroy' flags @@ -6033,10 +6150,6 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, else destroy_entry = destroy; -#ifdef H5_HAVE_PARALLEL - HDassert(FALSE == entry_ptr->coll_access); -#endif - /* we will write the entry to disk if it exists, is dirty, and if the * clear only flag is not set. */ @@ -6232,9 +6345,11 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, * * 2) Delete it from the skip list if requested. * - * 3) Update the replacement policy for eviction + * 3) Delete it from the collective read access list. * - * 4) Remove it from the tag list for this object + * 4) Update the replacement policy for eviction + * + * 5) Remove it from the tag list for this object * * Finally, if the destroy_entry flag is set, discard the * entry. @@ -6244,6 +6359,14 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, if(entry_ptr->in_slist && del_from_slist_on_destroy) H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush) +#ifdef H5_HAVE_PARALLEL + /* Check for collective read access flag */ + if(entry_ptr->coll_access) { + entry_ptr->coll_access = FALSE; + H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL) + } /* end if */ +#endif /* H5_HAVE_PARALLEL */ + H5C__UPDATE_RP_FOR_EVICTION(cache_ptr, entry_ptr, FAIL) /* Remove entry from tag list */ @@ -6290,7 +6413,8 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag cleared") /* Propagate the clean flag up the flush dependency chain if appropriate */ - HDassert(entry_ptr->flush_dep_ndirty_children == 0); + if(entry_ptr->flush_dep_ndirty_children != 0) + HDassert(entry_ptr->flush_dep_ndirty_children == 0); if(entry_ptr->flush_dep_nparents > 0) if(H5C__mark_flush_dep_clean(entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep clean flag") @@ -6432,6 +6556,19 @@ H5C__flush_single_entry(H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, } /* end else */ } /* if (destroy) */ + /* Check if we have to update the page buffer with cleared entries + * so it doesn't go out of date + */ + if(update_page_buffer) { + /* Sanity check */ + HDassert(!destroy); + HDassert(entry_ptr->image_ptr); + + if(f->shared->page_buf && f->shared->page_buf->page_size >= entry_ptr->size) + if(H5PB_update_entry(f->shared->page_buf, entry_ptr->addr, entry_ptr->size, entry_ptr->image_ptr) > 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Failed to update PB with metadata cache") + } /* end if */ + if(cache_ptr->log_flush) if((cache_ptr->log_flush)(cache_ptr, entry_addr, was_dirty, flags) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "log_flush callback failed") @@ -6822,10 +6959,15 @@ H5C_load_entry(H5F_t * f, entry->prefetched = FALSE; entry->prefetch_type_id = 0; entry->age = 0; + entry->prefetched_dirty = FALSE; #ifndef NDEBUG /* debugging field */ entry->serialization_count = 0; #endif /* NDEBUG */ + entry->tl_next = NULL; + entry->tl_prev = NULL; + entry->tag_info = NULL; + H5C__RESET_CACHE_ENTRY_STATS(entry); ret_value = thing; @@ -6864,13 +7006,6 @@ done: * Thus the function simply does its best, returning success * unless an error is encountered. * - * The primary_dxpl_id and secondary_dxpl_id parameters - * specify the dxpl_ids used on the first write occasioned - * by the call (primary_dxpl_id), and on all subsequent - * writes (secondary_dxpl_id). This is useful in the metadata - * cache, but may not be needed elsewhere. If so, just use the - * same dxpl_id for both parameters. - * * Observe that this function cannot occasion a read. * * Return: Non-negative on success/Negative on failure. @@ -6886,11 +7021,13 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, H5C_t * cache_ptr = f->shared->cache; #if H5C_COLLECT_CACHE_STATS int32_t clean_entries_skipped = 0; + int32_t dirty_pf_entries_skipped = 0; int32_t total_entries_scanned = 0; #endif /* H5C_COLLECT_CACHE_STATS */ uint32_t entries_examined = 0; uint32_t initial_list_len; size_t empty_space; + hbool_t reentrant_call = FALSE; hbool_t prev_is_dirty = FALSE; hbool_t didnt_flush_entry = FALSE; hbool_t restart_scan; @@ -6908,6 +7045,18 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); HDassert(cache_ptr->index_size == (cache_ptr->clean_index_size + cache_ptr->dirty_index_size)); + /* check to see if cache_ptr->msic_in_progress is TRUE. If it, this + * is a re-entrant call via a client callback called in the make + * space in cache process. To avoid an infinite recursion, set + * reentrant_call to TRUE, and goto done. + */ + if(cache_ptr->msic_in_progress) { + reentrant_call = TRUE; + HGOTO_DONE(SUCCEED); + } /* end if */ + + cache_ptr->msic_in_progress = TRUE; + if ( write_permitted ) { restart_scan = FALSE; initial_list_len = cache_ptr->LRU_list_len; @@ -6954,7 +7103,8 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, didnt_flush_entry = TRUE; } else if ( ( (entry_ptr->type)->id != H5AC_EPOCH_MARKER_ID ) && - ( ! entry_ptr->flush_in_progress ) ) { + ( ! entry_ptr->flush_in_progress ) && + ( ! entry_ptr->prefetched_dirty ) ) { didnt_flush_entry = FALSE; @@ -6980,13 +7130,6 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, cache_ptr->entries_removed_counter = 0; cache_ptr->last_entry_removed_ptr = NULL; -#ifdef H5_HAVE_PARALLEL - if(TRUE == entry_ptr->coll_access) { - entry_ptr->coll_access = FALSE; - H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL) - } /* end if */ -#endif - if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__NO_FLAGS_SET) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry") @@ -7020,10 +7163,16 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, } else { - /* Skip epoch markers and entries that are in the process - * of being flushed. + /* Skip epoch markers, entries that are in the process + * of being flushed, and entries marked as prefetched_dirty + * (occurs in the R/O case only). */ didnt_flush_entry = TRUE; + +#if H5C_COLLECT_CACHE_STATS + if(entry_ptr->prefetched_dirty) + dirty_pf_entries_skipped++; +#endif /* H5C_COLLECT_CACHE_STATS */ } if ( prev_ptr != NULL ) { @@ -7088,6 +7237,7 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, cache_ptr->calls_to_msic++; cache_ptr->total_entries_skipped_in_msic += clean_entries_skipped; + cache_ptr->total_dirty_pf_entries_skipped_in_msic += dirty_pf_entries_skipped; cache_ptr->total_entries_scanned_in_msic += total_entries_scanned; if ( clean_entries_skipped > cache_ptr->max_entries_skipped_in_msic ) { @@ -7095,6 +7245,9 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, cache_ptr->max_entries_skipped_in_msic = clean_entries_skipped; } + if(dirty_pf_entries_skipped > cache_ptr->max_dirty_pf_entries_skipped_in_msic) + cache_ptr->max_dirty_pf_entries_skipped_in_msic = dirty_pf_entries_skipped; + if ( total_entries_scanned > cache_ptr->max_entries_scanned_in_msic ) { cache_ptr->max_entries_scanned_in_msic = total_entries_scanned; @@ -7163,6 +7316,12 @@ H5C__make_space_in_cache(H5F_t *f, hid_t dxpl_id, size_t space_needed, } done: + /* Sanity checks */ + HDassert(cache_ptr->msic_in_progress); + if(!reentrant_call) + cache_ptr->msic_in_progress = FALSE; + HDassert((!reentrant_call) || (cache_ptr->msic_in_progress)); + FUNC_LEAVE_NOAPI(ret_value) } /* H5C__make_space_in_cache() */ @@ -8723,12 +8882,21 @@ H5C_remove_entry(void *_entry) /* Update the cache internal data structures as appropriate for a destroy. * Specifically: * 1) Delete it from the index - * 2) Update the replacement policy for eviction - * 3) Remove it from the tag list for this object + * 2) Delete it from the collective read access list + * 3) Update the replacement policy for eviction + * 4) Remove it from the tag list for this object */ H5C__DELETE_FROM_INDEX(cache, entry, FAIL) +#ifdef H5_HAVE_PARALLEL + /* Check for collective read access flag */ + if(entry->coll_access) { + entry->coll_access = FALSE; + H5C__REMOVE_FROM_COLL_LIST(cache, entry, FAIL) + } /* end if */ +#endif /* H5_HAVE_PARALLEL */ + H5C__UPDATE_RP_FOR_EVICTION(cache, entry, FAIL) /* Remove entry from tag list */ diff --git a/src/H5Cdbg.c b/src/H5Cdbg.c index eb5f123..a955eaf 100644 --- a/src/H5Cdbg.c +++ b/src/H5Cdbg.c @@ -70,6 +70,7 @@ /*******************/ +#ifndef NDEBUG /*------------------------------------------------------------------------- * Function: H5C_dump_cache @@ -175,6 +176,85 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5C_dump_cache() */ +#endif /* NDEBUG */ + +#ifndef NDEBUG + +/*------------------------------------------------------------------------- + * Function: H5C_dump_cache_LRU + * + * Purpose: Print a summary of the contents of the metadata cache + * LRU for debugging purposes. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 10/10/10 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_dump_cache_LRU(H5C_t *cache_ptr, const char *cache_name) +{ + H5C_cache_entry_t * entry_ptr; + int i = 0; + + FUNC_ENTER_NOAPI_NOERR + + /* Sanity check */ + HDassert(cache_ptr != NULL); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(cache_name != NULL ); + + HDfprintf(stdout, "\n\nDump of metadata cache LRU \"%s\"\n", cache_name); + HDfprintf(stdout, "LRU len = %d, LRU size = %d\n", + cache_ptr->LRU_list_len, (int)(cache_ptr->LRU_list_size)); + HDfprintf(stdout, "index_size = %d, max_cache_size = %d, delta = %d\n\n", + (int)(cache_ptr->index_size), (int)(cache_ptr->max_cache_size), + (int)(cache_ptr->max_cache_size) - (int)(cache_ptr->index_size)); + + /* Print header */ + HDfprintf(stdout, "Entry "); + HDfprintf(stdout, "| Address "); + HDfprintf(stdout, "| Tag "); + HDfprintf(stdout, "| Size "); + HDfprintf(stdout, "| Ring "); + HDfprintf(stdout, "| Type "); + HDfprintf(stdout, "| Dirty"); + HDfprintf(stdout, "\n"); + + HDfprintf(stdout, "----------------------------------------------------------------------------------------------------------------\n"); + + entry_ptr = cache_ptr->LRU_head_ptr; + while(entry_ptr != NULL) { + HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + + /* Print entry */ + HDfprintf(stdout, "%s%5d ", cache_ptr->prefix, i); + HDfprintf(stdout, " 0x%16llx ", (long long)(entry_ptr->addr)); + + if(NULL == entry_ptr->tag_info) + HDfprintf(stdout, " %16s ", "N/A"); + else + HDfprintf(stdout, " 0x%16llx ", + (long long)(entry_ptr->tag_info->tag)); + + HDfprintf(stdout, " %5lld ", (long long)(entry_ptr->size)); + HDfprintf(stdout, " %d ", (int)(entry_ptr->ring)); + HDfprintf(stdout, " %2d %-32s ", (int)(entry_ptr->type->id), + (entry_ptr->type->name)); + HDfprintf(stdout, " %d", (int)(entry_ptr->is_dirty)); + HDfprintf(stdout, "\n"); + + i++; + entry_ptr = entry_ptr->next; + } /* end while */ + + HDfprintf(stdout, "----------------------------------------------------------------------------------------------------------------\n"); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5C_dump_cache_LRU() */ +#endif /* NDEBUG */ /*------------------------------------------------------------------------- @@ -385,6 +465,7 @@ H5C_stats(H5C_t * cache_ptr, double average_successful_search_depth = 0.0f; double average_failed_search_depth = 0.0f; double average_entries_skipped_per_calls_to_msic = 0.0f; + double average_dirty_pf_entries_skipped_per_call_to_msic = 0.0f; double average_entries_scanned_per_calls_to_msic = 0.0f; #endif /* H5C_COLLECT_CACHE_STATS */ herr_t ret_value = SUCCEED; /* Return value */ @@ -620,6 +701,17 @@ H5C_stats(H5C_t * cache_ptr, (long)(cache_ptr->max_entries_skipped_in_msic)); if(cache_ptr->calls_to_msic > 0) + average_dirty_pf_entries_skipped_per_call_to_msic = + (((double)(cache_ptr->total_dirty_pf_entries_skipped_in_msic)) / + ((double)(cache_ptr->calls_to_msic))); + + HDfprintf(stdout, + "%s MSIC: Average/max dirty pf entries skipped = %lf / %ld\n", + cache_ptr->prefix, + average_dirty_pf_entries_skipped_per_call_to_msic, + (long)(cache_ptr->max_dirty_pf_entries_skipped_in_msic)); + + if(cache_ptr->calls_to_msic > 0) average_entries_scanned_per_calls_to_msic = (((double)(cache_ptr->total_entries_scanned_in_msic)) / ((double)(cache_ptr->calls_to_msic))); @@ -887,12 +979,14 @@ H5C_stats__reset(H5C_t H5_ATTR_UNUSED * cache_ptr) cache_ptr->max_pel_len = 0; cache_ptr->max_pel_size = (size_t)0; - cache_ptr->calls_to_msic = 0; - cache_ptr->total_entries_skipped_in_msic = 0; - cache_ptr->total_entries_scanned_in_msic = 0; - cache_ptr->max_entries_skipped_in_msic = 0; - cache_ptr->max_entries_scanned_in_msic = 0; - cache_ptr->entries_scanned_to_make_space = 0; + cache_ptr->calls_to_msic = 0; + cache_ptr->total_entries_skipped_in_msic = 0; + cache_ptr->total_dirty_pf_entries_skipped_in_msic = 0; + cache_ptr->total_entries_scanned_in_msic = 0; + cache_ptr->max_entries_skipped_in_msic = 0; + cache_ptr->max_dirty_pf_entries_skipped_in_msic = 0; + cache_ptr->max_entries_scanned_in_msic = 0; + cache_ptr->entries_scanned_to_make_space = 0; cache_ptr->slist_scan_restarts = 0; cache_ptr->LRU_scan_restarts = 0; diff --git a/src/H5Cimage.c b/src/H5Cimage.c index 1da2545..fc58dac 100644 --- a/src/H5Cimage.c +++ b/src/H5Cimage.c @@ -147,6 +147,40 @@ H5FL_DEFINE(H5C_cache_entry_t); /*------------------------------------------------------------------------- + * + * Function: H5C_cache_image_pending() + * + * Purpose: Tests to see if the load of a metadata cache image + * load is pending (i.e. will be executed on the next + * protect or insert) + * + * Returns TRUE if a cache image load is pending, and FALSE + * if not. Throws an assertion failure on error. + * + * Return: TRUE if a cache image load is pending, and FALSE otherwise. + * + * Programmer: John Mainzer, 6/18/16 + * + *------------------------------------------------------------------------- + */ +hbool_t +H5C_cache_image_pending(const H5C_t *cache_ptr) +{ + hbool_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity checks */ + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + + ret_value = (cache_ptr->load_image && !cache_ptr->image_loaded); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_cache_image_pending() */ + + +/*------------------------------------------------------------------------- * Function: H5C_cache_image_status() * * Purpose: Examine the metadata cache associated with the supplied @@ -489,7 +523,6 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, */ HDassert(!((type->flags & H5C__CLASS_SKIP_READS) && (type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG))); - HDassert(H5F_addr_defined(addr)); HDassert(type->get_initial_load_size); HDassert(type->deserialize); @@ -498,6 +531,7 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, * relationships now. The client will restore the relationship(s) with * the deserialized entry if appropriate. */ + HDassert(pf_entry_ptr->fd_parent_count == pf_entry_ptr->flush_dep_nparents); for(i = (int)(pf_entry_ptr->fd_parent_count) - 1; i >= 0; i--) { HDassert(pf_entry_ptr->flush_dep_parent); HDassert(pf_entry_ptr->flush_dep_parent[i]); @@ -525,7 +559,7 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, */ if(pf_entry_ptr->fd_child_count > 0) { if(NULL == (fd_children = (H5C_cache_entry_t **)H5MM_calloc(sizeof(H5C_cache_entry_t **) * (size_t)(pf_entry_ptr->fd_child_count + 1)))) - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for fd child ptr array") + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for fd child ptr array") if(H5C__destroy_pf_entry_child_flush_deps(cache_ptr, pf_entry_ptr, fd_children) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTUNDEPEND, FAIL, "can't destroy pf entry child flush dependency(s).") @@ -544,7 +578,6 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, */ if(NULL == (thing = type->deserialize(pf_entry_ptr->image_ptr, len, udata, &dirty))) HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, FAIL, "Can't deserialize image") - ds_entry_ptr = (H5C_cache_entry_t *)thing; /* In general, an entry should be clean just after it is loaded. @@ -583,8 +616,7 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, ds_entry_ptr->image_ptr = pf_entry_ptr->image_ptr; ds_entry_ptr->image_up_to_date = !dirty; ds_entry_ptr->type = type; - ds_entry_ptr->is_dirty = dirty | - pf_entry_ptr->is_dirty; + ds_entry_ptr->is_dirty = dirty | pf_entry_ptr->is_dirty; ds_entry_ptr->dirtied = FALSE; ds_entry_ptr->is_protected = FALSE; ds_entry_ptr->is_read_only = FALSE; @@ -626,7 +658,7 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, pf_entry_ptr->coll_prev = NULL; #endif /* H5_HAVE_PARALLEL */ - /* initialize cache image related fields */ + /* Initialize cache image related fields */ ds_entry_ptr->include_in_image = FALSE; ds_entry_ptr->lru_rank = 0; ds_entry_ptr->image_dirty = FALSE; @@ -638,6 +670,10 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, ds_entry_ptr->prefetched = FALSE; ds_entry_ptr->prefetch_type_id = 0; ds_entry_ptr->age = 0; + ds_entry_ptr->prefetched_dirty = pf_entry_ptr->prefetched_dirty; +#ifndef NDEBUG /* debugging field */ + ds_entry_ptr->serialization_count = 0; +#endif /* NDEBUG */ H5C__RESET_CACHE_ENTRY_STATS(ds_entry_ptr); @@ -666,7 +702,7 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, * and H5C__FLUSH_CLEAR_ONLY_FLAG flags set. */ pf_entry_ptr->image_ptr = NULL; - if ( pf_entry_ptr->is_dirty ) { + if(pf_entry_ptr->is_dirty) { HDassert(pf_entry_ptr->in_slist); flush_flags |= H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG; } /* end if */ @@ -684,7 +720,6 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, H5C__INSERT_IN_INDEX(cache_ptr, ds_entry_ptr, FAIL) HDassert(!ds_entry_ptr->in_slist); - if(ds_entry_ptr->is_dirty) H5C__INSERT_ENTRY_IN_SLIST(cache_ptr, ds_entry_ptr, FAIL) @@ -699,16 +734,13 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, (ds_entry_ptr->type->notify)(H5C_NOTIFY_ACTION_AFTER_LOAD, ds_entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry loaded into cache") - /* restore flush dependencies with the flush dependency children of + /* Restore flush dependencies with the flush dependency children of * of the prefetched entry. Note that we must protect *ds_entry_ptr * before the call to avoid triggering sanity check failures, and * then unprotect it afterwards. */ i = 0; if(fd_children != NULL) { - int j; - hbool_t found; - H5C__UPDATE_RP_FOR_PROTECT(cache_ptr, ds_entry_ptr, FAIL) ds_entry_ptr->is_protected = TRUE; while(fd_children[i] != NULL) { @@ -718,15 +750,22 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, HDassert((fd_children[i])->fd_parent_count > 0); HDassert((fd_children[i])->fd_parent_addrs); - j = 0; - found = FALSE; - while((j < (int)((fd_children[i])->fd_parent_count)) && (!found)) { - if((fd_children[i])->fd_parent_addrs[j] == ds_entry_ptr->addr) - found = TRUE; +#ifndef NDEBUG + { + int j; + hbool_t found; + + j = 0; + found = FALSE; + while((j < (int)((fd_children[i])->fd_parent_count)) && (!found)) { + if((fd_children[i])->fd_parent_addrs[j] == ds_entry_ptr->addr) + found = TRUE; - j++; - } /* end while */ - HDassert(found); + j++; + } /* end while */ + HDassert(found); + } +#endif /* NDEBUG */ if(H5C_create_flush_dependency(ds_entry_ptr, fd_children[i]) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTDEPEND, FAIL, "Can't restore child flush dependency") @@ -735,7 +774,6 @@ H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr, } /* end while */ H5C__UPDATE_RP_FOR_UNPROTECT(cache_ptr, ds_entry_ptr, FAIL); - ds_entry_ptr->is_protected = FALSE; } /* end if ( fd_children != NULL ) */ HDassert((unsigned)i == ds_entry_ptr->fd_child_count); @@ -828,6 +866,54 @@ H5C__free_image_entries_array(H5C_t * cache_ptr) /*------------------------------------------------------------------------- + * Function: H5C_force_cache_image_load() + * + * Purpose: On rare occasions, it is necessary to run + * H5MF_tidy_self_referential_fsm_hack() prior to the first + * metadata cache access. This is a problem as if there is a + * cache image at the end of the file, that routine will + * discard it. + * + * We solve this issue by calling this function, which will + * load the cache image and then call + * H5MF_tidy_self_referential_fsm_hack() to discard it. + * + * Return: SUCCEED on success, and FAIL on failure. + * + * Programmer: John Mainzer + * 1/11/17 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_force_cache_image_load(H5F_t *f, hid_t dxpl_id) +{ + H5C_t *cache_ptr; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + cache_ptr = f->shared->cache; + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(cache_ptr->load_image); + + /* Load the cache image, if requested */ + if(cache_ptr->load_image) { + cache_ptr->load_image = FALSE; + if(H5C__load_cache_image(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, FAIL, "can't load cache image") + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_force_cache_image_load() */ + + +/*------------------------------------------------------------------------- * Function: H5C_get_cache_image_config * * Purpose: Copy the current configuration for cache image generation @@ -1235,7 +1321,7 @@ H5C__image_entry_cmp(const void *_entry1, const void *_entry2) *------------------------------------------------------------------------- */ herr_t -H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id) +H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id, hbool_t *image_generated) { H5C_t * cache_ptr = NULL; haddr_t eoa_frag_addr = HADDR_UNDEF; @@ -1251,6 +1337,7 @@ H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id) cache_ptr = f->shared->cache; HDassert(cache_ptr); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(image_generated); /* If the file is opened and closed without any access to * any group or data set, it is possible that the cache image (if @@ -1262,6 +1349,26 @@ H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id) HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, FAIL, "can't load cache image") } /* end if */ + /* Before we start to generate the cache image (if requested), verify + * that the superblock supports superblock extension messages, and + * silently cancel any request for a cache image if it does not. + * + * Ideally, we would do this when the cache image is requested, + * but the necessary information is not necessary available at that + * time -- hence this last minute check. + * + * Note that under some error conditions, the superblock will be + * undefined in this case as well -- if so, assume that the + * superblock does not support superblock extension messages. + */ + if((NULL == f->shared->sblock) || + (f->shared->sblock->super_vers < HDF5_SUPERBLOCK_VERSION_2)) { + H5C_cache_image_ctl_t default_image_ctl = H5C__DEFAULT_CACHE_IMAGE_CTL; + + cache_ptr->image_ctl = default_image_ctl; + HDassert(!(cache_ptr->image_ctl.generate_image)); + } /* end if */ + /* Generate the cache image, if requested */ if(cache_ptr->image_ctl.generate_image) { /* Create the cache image super block extension message. @@ -1355,6 +1462,15 @@ H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id) (hsize_t)(cache_ptr->image_data_len), &eoa_frag_addr, &eoa_frag_size))) HGOTO_ERROR(H5E_CACHE, H5E_NOSPACE, FAIL, "can't allocate file space for metadata cache image") + /* Make note of the eoa after allocation of the cache image + * block. This value is used for sanity checking when we + * shutdown the self referential free space managers after + * we destroy the metadata cache. + */ + HDassert(HADDR_UNDEF == f->shared->eoa_post_mdci_fsalloc); + if(HADDR_UNDEF == (f->shared->eoa_post_mdci_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)) ) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file size") + /* For now, drop any fragment left over from the allocation of the * image block on the ground. A fragment should only be returned * if the underlying file alignment is greater than 1. @@ -1386,7 +1502,7 @@ H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id) */ if(cache_ptr->image_ctl.flags & H5C_CI__GEN_MDC_IMAGE_BLK) if(H5C__write_cache_image_superblock_msg(f, dxpl_id, FALSE) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "update of cache image SB mesg failed.") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "update of cache image SB mesg failed") /* At this point: * @@ -1440,10 +1556,13 @@ H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id) */ if(cache_ptr->image_ctl.flags & H5C_CI__GEN_MDC_IMAGE_BLK) if(H5F_super_ext_remove_msg(f, dxpl_id, H5O_MDCI_MSG_ID) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove MDC image msg from superblock ext.") + HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove MDC image msg from superblock ext") cache_ptr->image_ctl.generate_image = FALSE; } /* end else */ + + /* Indicate that a cache image was generated */ + *image_generated = TRUE; } /* end if */ done: @@ -1458,6 +1577,18 @@ done: * image_ctl field of *cache_ptr. Make adjustments for * changes in configuration as required. * + * If the file is open read only, silently + * force the cache image configuration to its default + * (which disables construction of a cache image). + * + * Note that in addition to being inapplicable in the + * read only case, cache image is also inapplicable if + * the superblock does not support superblock extension + * messages. Unfortunately, this information need not + * be available at this point. Thus we check for this + * later, in H5C_prep_for_file_close() and cancel the + * cache image request if appropriate. + * * Fail if the new configuration is invalid. * * Return: SUCCEED on success, and FAIL on failure. @@ -1483,26 +1614,13 @@ H5C_set_cache_image_config(const H5F_t *f, H5C_t *cache_ptr, /* Check arguments */ if((cache_ptr == NULL) || (cache_ptr->magic != H5C__H5C_T_MAGIC)) HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "Bad cache_ptr on entry") - if(config_ptr == NULL) - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "NULL config_ptr on entry") - if(config_ptr->version != H5C__CURR_CACHE_IMAGE_CTL_VER) - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "Unknown config version") - /* check general configuration section of the config: */ + /* Validate the config: */ if(H5C_validate_cache_image_config(config_ptr) < 0) HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "invalid cache image configuration") - if(H5F_INTENT(f) & H5F_ACC_RDWR) /* file has been opened R/W */ - cache_ptr->image_ctl = *config_ptr; - else { /* file opened R/O -- suppress cache image silently */ - H5C_cache_image_ctl_t default_image_ctl = H5C__DEFAULT_CACHE_IMAGE_CTL; - - cache_ptr->image_ctl = default_image_ctl; - HDassert(!(cache_ptr->image_ctl.generate_image)); - } /* end else */ - #ifdef H5_HAVE_PARALLEL - /* the collective metadata write code is not currently compatible + /* The collective metadata write code is not currently compatible * with cache image. Until this is fixed, suppress cache image silently * if there is more than one process. * JRM -- 11/8/16 @@ -1513,6 +1631,30 @@ H5C_set_cache_image_config(const H5F_t *f, H5C_t *cache_ptr, cache_ptr->image_ctl = default_image_ctl; HDassert(!(cache_ptr->image_ctl.generate_image)); } /* end if */ + else { +#endif /* H5_HAVE_PARALLEL */ + /* A cache image can only be generated if the file is opened read / write + * and the superblock supports superblock extension messages. + * + * However, the superblock version is not available at this point -- + * hence we can only check the former requirement now. Do the latter + * check just before we construct the image.. + * + * If the file is opened read / write, apply the supplied configuration. + * + * If it is not, set the image configuration to the default, which has + * the effect of silently disabling the cache image if it was requested. + */ + if(H5F_INTENT(f) & H5F_ACC_RDWR) + cache_ptr->image_ctl = *config_ptr; + else { + H5C_cache_image_ctl_t default_image_ctl = H5C__DEFAULT_CACHE_IMAGE_CTL; + + cache_ptr->image_ctl = default_image_ctl; + HDassert(!(cache_ptr->image_ctl.generate_image)); + } /* end else */ +#ifdef H5_HAVE_PARALLEL + } /* end else */ #endif /* H5_HAVE_PARALLEL */ done: @@ -3186,7 +3328,13 @@ H5C__reconstruct_cache_entry(const H5F_t *f, H5C_t *cache_ptr, * this as otherwise the cache will attempt to write them on file * close. Since the file is R/O, the metadata cache image superblock * extension message and the cache image block will not be removed. - * Hence no danger in this. + * Hence no danger in this for subsequent opens. + * + * However, if the dirty entry (marked clean for purposes of the R/O + * file open) is evicted and then referred to, the cache will read + * either invalid or obsolete data from the file. Handle this by + * setting the prefetched_dirty field, and hiding such entries from + * the eviction candidate selection algorithm. */ pf_entry_ptr->is_dirty = (is_dirty && file_is_rw); @@ -3204,6 +3352,8 @@ H5C__reconstruct_cache_entry(const H5F_t *f, H5C_t *cache_ptr, /* Decode dirty dependency child count */ UINT16DECODE(p, pf_entry_ptr->fd_dirty_child_count); + if(!file_is_rw) + pf_entry_ptr->fd_dirty_child_count = 0; if(pf_entry_ptr->fd_dirty_child_count > pf_entry_ptr->fd_child_count) HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "invalid dirty flush dependency child count") @@ -3263,6 +3413,7 @@ H5C__reconstruct_cache_entry(const H5F_t *f, H5C_t *cache_ptr, pf_entry_ptr->image_up_to_date = TRUE; pf_entry_ptr->type = H5AC_PREFETCHED_ENTRY; pf_entry_ptr->prefetched = TRUE; + pf_entry_ptr->prefetched_dirty = is_dirty && (!file_is_rw); /* Sanity checks */ HDassert(pf_entry_ptr->size > 0 && pf_entry_ptr->size < H5C_MAX_ENTRY_SIZE); diff --git a/src/H5Cmpio.c b/src/H5Cmpio.c index ebb98b3..06ce714 100644 --- a/src/H5Cmpio.c +++ b/src/H5Cmpio.c @@ -65,6 +65,11 @@ /* Local Prototypes */ /********************/ static herr_t H5C__collective_write(H5F_t *f, hid_t dxpl_id); +static herr_t H5C__flush_candidate_entries(H5F_t *f, hid_t dxpl_id, + unsigned entries_to_flush[H5C_RING_NTYPES], + unsigned entries_to_clear[H5C_RING_NTYPES]); +static herr_t H5C__flush_candidates_in_ring(H5F_t *f, hid_t dxpl_id, + H5C_ring_t ring, unsigned entries_to_flush, unsigned entries_to_clear); /*********************/ @@ -175,31 +180,18 @@ H5C_apply_candidate_list(H5F_t * f, int mpi_rank, int mpi_size) { - hbool_t restart_scan; - hbool_t prev_is_dirty; int i; int m; int n; unsigned first_entry_to_flush; unsigned last_entry_to_flush; - unsigned entries_to_clear = 0; - unsigned entries_to_flush = 0; - unsigned entries_to_flush_or_clear_last = 0; - unsigned entries_to_flush_collectively = 0; - unsigned entries_cleared = 0; - unsigned entries_flushed = 0; - unsigned entries_delayed = 0; - unsigned entries_flushed_or_cleared_last = 0; - unsigned entries_flushed_collectively = 0; - unsigned entries_examined = 0; - unsigned initial_list_len; + unsigned total_entries_to_clear = 0; + unsigned total_entries_to_flush = 0; int * candidate_assignment_table = NULL; + unsigned entries_to_flush[H5C_RING_NTYPES]; + unsigned entries_to_clear[H5C_RING_NTYPES]; haddr_t addr; - H5C_cache_entry_t * clear_ptr = NULL; - H5C_cache_entry_t * next_ptr = NULL; H5C_cache_entry_t * entry_ptr = NULL; - H5C_cache_entry_t * flush_ptr = NULL; - H5C_cache_entry_t * delayed_ptr = NULL; #if H5C_DO_SANITY_CHECKS haddr_t last_addr; #endif /* H5C_DO_SANITY_CHECKS */ @@ -220,6 +212,10 @@ H5C_apply_candidate_list(H5F_t * f, HDassert(0 <= mpi_rank); HDassert(mpi_rank < mpi_size); + /* Initialize the entries_to_flush and entries_to_clear arrays */ + HDmemset(entries_to_flush, 0, sizeof(entries_to_flush)); + HDmemset(entries_to_clear, 0, sizeof(entries_to_clear)); + #if H5C_APPLY_CANDIDATE_LIST__DEBUG HDfprintf(stdout, "%s:%d: setting up candidate assignment table.\n", FUNC, mpi_rank); @@ -330,17 +326,26 @@ H5C_apply_candidate_list(H5F_t * f, */ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Listed entry is protected?!?!?") + /* Sanity checks */ + HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + HDassert(entry_ptr->ring >= H5C_RING_USER); + HDassert(entry_ptr->ring <= H5C_RING_SB); + HDassert(!entry_ptr->flush_immediately); + HDassert(!entry_ptr->clear_on_unprotect); + /* Determine whether the entry is to be cleared or flushed, * and mark it accordingly. We will scan the protected and * pinned list shortly, and clear or flush according to these * markings. */ if(u >= first_entry_to_flush && u <= last_entry_to_flush) { - entries_to_flush++; + total_entries_to_flush++; + entries_to_flush[entry_ptr->ring]++; entry_ptr->flush_immediately = TRUE; } /* end if */ else { - entries_to_clear++; + total_entries_to_clear++; + entries_to_clear[entry_ptr->ring]++; entry_ptr->clear_on_unprotect = TRUE; } /* end else */ @@ -356,372 +361,36 @@ H5C_apply_candidate_list(H5F_t * f, } /* end if */ } /* end for */ +#if H5C_DO_SANITY_CHECKS + m = 0; + n = 0; + for(i = 0; i < H5C_RING_NTYPES; i++) { + m += (int)entries_to_flush[i]; + n += (int)entries_to_clear[i]; + } /* end if */ + + HDassert((unsigned)m == total_entries_to_flush); + HDassert((unsigned)n == total_entries_to_clear); +#endif /* H5C_DO_SANITY_CHECKS */ + #if H5C_APPLY_CANDIDATE_LIST__DEBUG HDfprintf(stdout, "%s:%d: num candidates/to clear/to flush = %u/%u/%u.\n", - FUNC, mpi_rank, num_candidates, entries_to_clear, - entries_to_flush); + FUNC, mpi_rank, num_candidates, total_entries_to_clear, + total_entries_to_flush); #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ /* We have now marked all the entries on the candidate list for * either flush or clear -- now scan the LRU and the pinned list - * for these entries and do the deed. + * for these entries and do the deed. Do this via a call to + * H5C__flush_candidate_entries(). * * Note that we are doing things in this round about manner so as * to preserve the order of the LRU list to the best of our ability. * If we don't do this, my experiments indicate that we will have a * noticably poorer hit ratio as a result. */ - -#if H5C_APPLY_CANDIDATE_LIST__DEBUG - HDfprintf(stdout, "%s:%d: scanning LRU list. len = %d.\n", FUNC, mpi_rank, - (int)(cache_ptr->LRU_list_len)); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - /* ===================================================================== * - * Now scan the LRU and PEL lists, flushing or clearing entries as - * needed. - * - * The flush_me_last flag may dictate how or - * when some entries can be flushed, and should be addressed here. - * However, in their initial implementation, these flags only apply to the - * superblock, so there's only a relatively small change to this function - * to account for this one case where they come into play. If these flags - * are ever expanded upon, this function and the following flushing steps - * should be reworked to account for additional cases. - * ===================================================================== */ - - HDassert(entries_to_flush >= 0); - - restart_scan = FALSE; - entries_examined = 0; - initial_list_len = cache_ptr->LRU_list_len; - entry_ptr = cache_ptr->LRU_tail_ptr; - - /* Examine each entry in the LRU list */ - while ( ( entry_ptr != NULL ) - && - ( entries_examined <= (entries_to_flush + 1) * initial_list_len ) - && - ( (entries_cleared + entries_flushed) < num_candidates ) ) { - - if ( entry_ptr->prev != NULL ) - prev_is_dirty = entry_ptr->prev->is_dirty; - - /* If this process needs to clear this entry. */ - if(entry_ptr->clear_on_unprotect) { - - HDassert(entry_ptr->is_dirty); - - next_ptr = entry_ptr->next; - entry_ptr->clear_on_unprotect = FALSE; - clear_ptr = entry_ptr; - entry_ptr = entry_ptr->prev; - entries_cleared++; - -#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) - HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank, - (long long)clear_ptr->addr); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - /* reset entries_removed_counter and - * last_entry_removed_ptr prior to the call to - * H5C__flush_single_entry() so that we can spot - * unexpected removals of entries from the cache, - * and set the restart_scan flag if proceeding - * would be likely to cause us to scan an entry - * that is no longer in the cache. - * - * Note that as of this writing (April 2015) this - * case cannot occur in the parallel case. However - * Quincey is making noises about changing this, hence - * the insertion of this test. - * - * Note also that there is no test code to verify - * that this code actually works (although similar code - * in the serial version exists and is tested). - * - * Implementing a test will likely require implementing - * flush op like facilities in the parallel tests. At - * a guess this will not be terribly painful, but it - * will take a bit of time. - */ - cache_ptr->entries_removed_counter = 0; - cache_ptr->last_entry_removed_ptr = NULL; - - if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't clear entry.") - - if((cache_ptr->entries_removed_counter > 1) || - (cache_ptr->last_entry_removed_ptr == entry_ptr)) - restart_scan = TRUE; - } /* end if */ - - /* Else, if this process needs to flush this entry. */ - else if (entry_ptr->flush_immediately) { - - HDassert(entry_ptr->is_dirty); - - next_ptr = entry_ptr->next; - entry_ptr->flush_immediately = FALSE; - flush_ptr = entry_ptr; - entry_ptr = entry_ptr->prev; - entries_flushed++; - -#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) - HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank, - (long long)flush_ptr->addr); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - /* reset entries_removed_counter and - * last_entry_removed_ptr prior to the call to - * H5C__flush_single_entry() so that we can spot - * unexpected removals of entries from the cache, - * and set the restart_scan flag if proceeding - * would be likely to cause us to scan an entry - * that is no longer in the cache. - * - * Note that as of this writing (April 2015) this - * case cannot occur in the parallel case. However - * Quincey is making noises about changing this, hence - * the insertion of this test. - * - * Note also that there is no test code to verify - * that this code actually works (although similar code - * in the serial version exists and is tested). - * - * Implementing a test will likely require implementing - * flush op like facilities in the parallel tests. At - * a guess this will not be terribly painful, but it - * will take a bit of time. - */ - cache_ptr->entries_removed_counter = 0; - cache_ptr->last_entry_removed_ptr = NULL; - - /* Add this entry to the list of entries to collectively write */ - if(H5C__flush_single_entry(f, dxpl_id, flush_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush entry.") - - if((cache_ptr->entries_removed_counter > 1) || - (cache_ptr->last_entry_removed_ptr == entry_ptr)) - restart_scan = TRUE; - } /* end else-if */ - - /* Otherwise, no action to be taken on this entry. Grab the next. */ - else { - entry_ptr = entry_ptr->prev; - - if ( entry_ptr != NULL ) - next_ptr = entry_ptr->next; - - } /* end else */ - - if ( ( entry_ptr != NULL ) - && - ( ( restart_scan ) - || - ( entry_ptr->is_dirty != prev_is_dirty ) - || - ( entry_ptr->next != next_ptr ) - || - ( entry_ptr->is_protected ) - || - ( entry_ptr->is_pinned ) - ) - ) { - - /* something has happened to the LRU -- start over - * from the tail. - * - * Recall that this code should be un-reachable at present, - * as all the operations by entries on flush that could cause - * it to be reachable are disallowed in the parallel case at - * present. Hence the following assertion which should be - * removed if the above changes. - */ - - HDassert( ! restart_scan ); - HDassert( entry_ptr->is_dirty == prev_is_dirty ); - HDassert( entry_ptr->next == next_ptr ); - HDassert( ! entry_ptr->is_protected ); - HDassert( ! entry_ptr->is_pinned ); - - HDassert(FALSE); /* see comment above */ - - restart_scan = FALSE; - entry_ptr = cache_ptr->LRU_tail_ptr; -/* - H5C__UPDATE_STATS_FOR_LRU_SCAN_RESTART(cache_ptr) -*/ - } - - entries_examined++; - } /* end while */ - -#if H5C_APPLY_CANDIDATE_LIST__DEBUG - HDfprintf(stdout, "%s:%d: entries examined/cleared/flushed = %u/%u/%u.\n", - FUNC, mpi_rank, entries_examined, - entries_cleared, entries_flushed); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - /* It is also possible that some of the cleared entries are on the - * pinned list. Must scan that also. - * - * WARNING: - * - * As we now allow unpinning, and removal of other entries as a side - * effect of flushing an entry, it is possible that the next entry - * in a PEL scan could either be no longer pinned, or no longer in - * the cache by the time we get to it. - * - * At present, this is not possible in this case, as we disallow such - * operations in the parallel version of the library. However, Quincey - * has been making noises about relaxing this. If and when he does, - * we have a potential problem here. - * - * The same issue exists in the serial cache, and there are tests - * to detect this problem when it occurs, and adjust to it. As seen - * above in the LRU scan, I have ported such tests to the parallel - * code where a close cognate exists in the serial code. - * - * I haven't done so here, as there are no PEL scans where the problem - * can occur in the serial code. Needless to say, this will have to - * be repaired if the constraints on pre_serialize and serialize - * callbacks are relaxed in the parallel version of the metadata cache. - * - * JRM -- 4/1/15 - */ - -#if H5C_APPLY_CANDIDATE_LIST__DEBUG - HDfprintf(stdout, "%s:%d: scanning pinned entry list. len = %d\n", - FUNC, mpi_rank, (int)(cache_ptr->pel_len)); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - entry_ptr = cache_ptr->pel_head_ptr; - while((entry_ptr != NULL) && - ((entries_cleared + entries_flushed + entries_delayed) - < num_candidates)) { - - /* If entry is marked for flush or for clear */ - if((entry_ptr->clear_on_unprotect||entry_ptr->flush_immediately)) { - - /* If this entry needs to be flushed last */ - if (entry_ptr->flush_me_last) { - - /* At this time, only the superblock supports being - flushed last. Conveniently, it also happens to be the only - entry that supports being flushed collectively, as well. Also - conveniently, it's always pinned, so we only need to check - for it while scanning the PEL here. Finally, it's never - included in a candidate list that excludes other dirty - entries in a cache, so we can handle this relatively simple - case here. - - For now, this function asserts this and saves the entry - to flush it after scanning the rest of the PEL list. - - If there are ever more entries that either need to be - flushed last and/or flushed collectively, this whole routine - will need to be reworked to handle all additional cases. As - it is the simple case of a single pinned entry needing - flushed last and collectively is just a minor addition to - this routine, but signficantly buffing up the usage of - flush_me_last will require a more - intense rework of this function and potentially the function - of candidate lists as a whole. */ - - entries_to_flush_or_clear_last++; - entries_to_flush_collectively++; - HDassert(entries_to_flush_or_clear_last == 1); - HDassert(entries_to_flush_collectively == 1); - - /* Delay the entry. It will be flushed later. */ - delayed_ptr = entry_ptr; - entries_delayed++; - HDassert(entries_delayed == 1); - - } /* end if */ - - /* Else, this process needs to clear this entry. */ - else if (entry_ptr->clear_on_unprotect) { - HDassert(!entry_ptr->flush_immediately); - entry_ptr->clear_on_unprotect = FALSE; - clear_ptr = entry_ptr; - entry_ptr = entry_ptr->next; - entries_cleared++; - -#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) - HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank, - (long long)clear_ptr->addr); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't clear entry.") - } /* end else-if */ - - /* Else, if this process needs to independently flush this entry. */ - else if (entry_ptr->flush_immediately) { - entry_ptr->flush_immediately = FALSE; - flush_ptr = entry_ptr; - entry_ptr = entry_ptr->next; - entries_flushed++; - -#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) - HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank, - (long long)flush_ptr->addr); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - /* Add this entry to the list of entries to collectively write */ - if(H5C__flush_single_entry(f, dxpl_id, flush_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't clear entry.") - } /* end else-if */ - } /* end if */ - - /* Otherwise, this entry is not marked for flush or clear. Grab the next. */ - else { - entry_ptr = entry_ptr->next; - } /* end else */ - - } /* end while */ - -#if H5C_APPLY_CANDIDATE_LIST__DEBUG - HDfprintf(stdout, - "%s:%d: pel entries examined/cleared/flushed = %u/%u/%u.\n", - FUNC, mpi_rank, entries_examined, - entries_cleared, entries_flushed); - HDfprintf(stdout, "%s:%d: done.\n", FUNC, mpi_rank); - - HDfsync(stdout); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - /* ====================================================================== * - * Now, handle all delayed entries. * - * * - * This can *only* be the superblock at this time, so it's relatively * - * easy to deal with. We're collectively flushing the entry saved from * - * above. This will need to be handled differently if there are ever more * - * than one entry needing this special treatment.) * - * ====================================================================== */ - - if (delayed_ptr) { - - if (delayed_ptr->clear_on_unprotect) { - if(H5C__flush_single_entry(f, dxpl_id, delayed_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush entry.") - - entry_ptr->clear_on_unprotect = FALSE; - entries_cleared++; - } else if (delayed_ptr->flush_immediately) { - /* Add this entry to the list of entries to collectively write */ - if(H5C__flush_single_entry(f, dxpl_id, delayed_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush entry collectively.") - - entry_ptr->flush_immediately = FALSE; - entries_flushed++; - } /* end if */ - - entries_flushed_collectively++; - entries_flushed_or_cleared_last++; - } /* end if */ + if(H5C__flush_candidate_entries(f, dxpl_id, entries_to_flush, entries_to_clear) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "flush candidates failed") /* If we've deferred writing to do it collectively, take care of that now */ if(f->coll_md_write) { @@ -733,21 +402,6 @@ H5C_apply_candidate_list(H5F_t * f, HGOTO_ERROR(H5E_CACHE, H5E_WRITEERROR, FAIL, "can't write metadata collectively") } /* end if */ - /* ====================================================================== * - * Finished flushing everything. * - * ====================================================================== */ - - HDassert((entries_flushed == entries_to_flush)); - HDassert((entries_cleared == entries_to_clear)); - HDassert((entries_flushed_or_cleared_last == entries_to_flush_or_clear_last)); - HDassert((entries_flushed_collectively == entries_to_flush_collectively)); - - if((entries_flushed != entries_to_flush) || - (entries_cleared != entries_to_clear) || - (entries_flushed_or_cleared_last != entries_to_flush_or_clear_last) || - (entries_flushed_collectively != entries_to_flush_collectively)) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry count mismatch") - done: if(candidate_assignment_table != NULL) candidate_assignment_table = (int *)H5MM_xfree((void *)candidate_assignment_table); @@ -998,11 +652,13 @@ H5C_mark_entries_as_clean(H5F_t * f, { H5C_t * cache_ptr; unsigned entries_cleared; + unsigned pinned_entries_cleared; + hbool_t progress; unsigned entries_examined; unsigned initial_list_len; haddr_t addr; -#if H5C_DO_SANITY_CHECKS unsigned pinned_entries_marked = 0; +#if H5C_DO_SANITY_CHECKS unsigned protected_entries_marked = 0; unsigned other_entries_marked = 0; haddr_t last_addr; @@ -1086,19 +742,13 @@ H5C_mark_entries_as_clean(H5F_t * f, } /* end if */ entry_ptr->clear_on_unprotect = TRUE; + if(entry_ptr->is_pinned) + pinned_entries_marked++; #if H5C_DO_SANITY_CHECKS - if ( entry_ptr->is_protected ) { - + else if(entry_ptr->is_protected) protected_entries_marked++; - - } else if ( entry_ptr->is_pinned ) { - - pinned_entries_marked++; - - } else { - + else other_entries_marked++; - } #endif /* H5C_DO_SANITY_CHECKS */ } } @@ -1140,7 +790,8 @@ H5C_mark_entries_as_clean(H5F_t * f, entry_ptr = entry_ptr->prev; entries_cleared++; - if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) + if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, + (H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG | H5C__UPDATE_PAGE_BUFFER_FLAG)) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't clear entry") } /* end if */ else @@ -1155,19 +806,27 @@ H5C_mark_entries_as_clean(H5F_t * f, /* It is also possible that some of the cleared entries are on the * pinned list. Must scan that also. */ - entry_ptr = cache_ptr->pel_head_ptr; - while(entry_ptr != NULL) { - if(entry_ptr->clear_on_unprotect) { - entry_ptr->clear_on_unprotect = FALSE; - clear_ptr = entry_ptr; - entry_ptr = entry_ptr->next; - entries_cleared++; + pinned_entries_cleared = 0; + progress = TRUE; + while((pinned_entries_cleared < pinned_entries_marked) && progress) { + progress = FALSE; + entry_ptr = cache_ptr->pel_head_ptr; + while(entry_ptr != NULL) { + if(entry_ptr->clear_on_unprotect && entry_ptr->flush_dep_ndirty_children == 0) { + entry_ptr->clear_on_unprotect = FALSE; + clear_ptr = entry_ptr; + entry_ptr = entry_ptr->next; + entries_cleared++; + pinned_entries_cleared++; + progress = TRUE; - if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't clear entry") - } /* end if */ - else - entry_ptr = entry_ptr->next; + if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, + (H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG | H5C__UPDATE_PAGE_BUFFER_FLAG)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't clear entry") + } /* end if */ + else + entry_ptr = entry_ptr->next; + } /* end while */ } /* end while */ #if H5C_DO_SANITY_CHECKS @@ -1297,7 +956,6 @@ H5C__collective_write(H5F_t *f, hid_t dxpl_id) /* Get number of entries in collective write list */ count = (int)H5SL_count(cache_ptr->coll_write_list); - if(count > 0) { H5FD_mpio_xfer_t xfer_mode = H5FD_MPIO_COLLECTIVE; H5SL_node_t *node; @@ -1410,4 +1068,511 @@ done: FUNC_LEAVE_NOAPI(ret_value); } /* end H5C__collective_write() */ + + +/*------------------------------------------------------------------------- + * Function: H5C__flush_candidate_entries + * + * Purpose: Flush or clear (as indicated) the candidate entries that + * have been marked in the metadata cache. In so doing, + * observe rings and flush dependencies. + * + * Note that this function presumes that: + * + * 1) no candidate entries are protected, + * + * 2) all candidate entries are dirty, and + * + * 3) if a candidate entry has a dirty flush dependency + * child, that child is also a candidate entry. + * + * The function will fail if any of these preconditions are + * not met. + * + * Candidate entries are marked by setting either the + * flush_immediately or the clear_on_unprotect flags in the + * cache entry (but not both). Entries marked flush_immediately + * will be flushed, those marked clear_on_unprotect will be + * cleared. + * + * Note that this function is a modified version of + * H5C_flush_cache() -- any changes there may need to be + * reflected here and vise versa. + * + * Return: Non-negative on success/Negative on failure. + * + * Programmer: John Mainzer + * 2/10/17 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +static herr_t +H5C__flush_candidate_entries(H5F_t *f, hid_t dxpl_id, + unsigned entries_to_flush[H5C_RING_NTYPES], + unsigned entries_to_clear[H5C_RING_NTYPES]) +{ +#if H5C_DO_SANITY_CHECKS + int i; + uint32_t index_len = 0; + size_t index_size = (size_t)0; + size_t clean_index_size = (size_t)0; + size_t dirty_index_size = (size_t)0; + size_t slist_size = (size_t)0; + uint32_t slist_len = 0; +#endif /* H5C_DO_SANITY_CHECKS */ + H5C_ring_t ring; + H5C_t * cache_ptr; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + HDassert(f); + HDassert(f->shared); + + cache_ptr = f->shared->cache; + + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(cache_ptr->slist_ptr); + + HDassert(entries_to_flush[H5C_RING_UNDEFINED] == 0); + HDassert(entries_to_clear[H5C_RING_UNDEFINED] == 0); + +#if H5C_DO_SANITY_CHECKS + HDassert(cache_ptr->index_ring_len[H5C_RING_UNDEFINED] == 0); + HDassert(cache_ptr->index_ring_size[H5C_RING_UNDEFINED] == (size_t)0); + HDassert(cache_ptr->clean_index_ring_size[H5C_RING_UNDEFINED] == (size_t)0); + HDassert(cache_ptr->dirty_index_ring_size[H5C_RING_UNDEFINED] == (size_t)0); + HDassert(cache_ptr->slist_ring_len[H5C_RING_UNDEFINED] == 0); + HDassert(cache_ptr->slist_ring_size[H5C_RING_UNDEFINED] == (size_t)0); + + for(i = H5C_RING_USER; i < H5C_RING_NTYPES; i++) { + index_len += cache_ptr->index_ring_len[i]; + index_size += cache_ptr->index_ring_size[i]; + clean_index_size += cache_ptr->clean_index_ring_size[i]; + dirty_index_size += cache_ptr->dirty_index_ring_size[i]; + + slist_len += cache_ptr->slist_ring_len[i]; + slist_size += cache_ptr->slist_ring_size[i]; + } /* end for */ + + HDassert(cache_ptr->index_len == index_len); + HDassert(cache_ptr->index_size == index_size); + HDassert(cache_ptr->clean_index_size == clean_index_size); + HDassert(cache_ptr->dirty_index_size == dirty_index_size); + HDassert(cache_ptr->slist_len == slist_len); + HDassert(cache_ptr->slist_size == slist_size); +#endif /* H5C_DO_SANITY_CHECKS */ + +#if H5C_DO_EXTREME_SANITY_CHECKS + if(H5C_validate_protected_entry_list(cache_ptr) < 0 + || H5C_validate_pinned_entry_list(cache_ptr) < 0 + || H5C_validate_lru_list(cache_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") +#endif /* H5C_DO_EXTREME_SANITY_CHECKS */ + + cache_ptr->flush_in_progress = TRUE; + + /* flush each ring, starting from the outermost ring and + * working inward. + */ + ring = H5C_RING_USER; + while(ring < H5C_RING_NTYPES) { + if(H5C__flush_candidates_in_ring(f, dxpl_id, ring, entries_to_flush[ring], entries_to_clear[ring]) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "flush candidates in ring failed") + + ring++; + } /* end while */ + +done: + cache_ptr->flush_in_progress = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C__flush_candidate_entries() */ + + +/*------------------------------------------------------------------------- + * Function: H5C__flush_candidates_in_ring + * + * Purpose: Flush or clear (as indicated) the candidate entries + * contained in the specified cache and ring. All candidate + * entries in rings outside the specified ring must have been + * flushed (or cleared) on entry. + * + * Note that this function presumes that: + * + * 1) no candidate entries are protected, + * + * 2) all candidate entries are dirty, and + * + * 3) if a candidate entry has a dirty flush dependency + * child, that child is also a candidate entry. + * + * The function will fail if any of these preconditions are + * not met. + * + * Candidate entries are marked by setting either the + * flush_immediately or the clear_on_unprotect flags in the + * cache entry (but not both). Entries marked flush_immediately + * will be flushed, those marked clear_on_unprotect will be + * cleared. + * + * Candidate entries residing in the LRU must be flushed + * (or cleared) in LRU order to avoid performance issues. + * + * Return: Non-negative on success/Negative on failure. + * + * Programmer: John Mainzer + * 2/10/17 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5C__flush_candidates_in_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring, + unsigned entries_to_flush, unsigned entries_to_clear) +{ + H5C_t * cache_ptr; + hbool_t progress; + hbool_t restart_scan = FALSE; + unsigned entries_flushed = 0; + unsigned entries_cleared = 0; +#if H5C_DO_SANITY_CHECKS + unsigned init_index_len; +#endif /* H5C_DO_SANITY_CHECKS */ + unsigned clear_flags = H5C__FLUSH_CLEAR_ONLY_FLAG | + H5C__GENERATE_IMAGE_FLAG | + H5C__UPDATE_PAGE_BUFFER_FLAG; + unsigned flush_flags = H5C__NO_FLAGS_SET; + unsigned op_flags; + H5C_cache_entry_t *op_ptr; + H5C_cache_entry_t *entry_ptr; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + cache_ptr = f->shared->cache; + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(cache_ptr->slist_ptr); + HDassert(ring > H5C_RING_UNDEFINED); + HDassert(ring < H5C_RING_NTYPES); + +#if H5C_DO_EXTREME_SANITY_CHECKS + if((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || + (H5C_validate_lru_list(cache_ptr) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") +#endif /* H5C_DO_EXTREME_SANITY_CHECKS */ + +#if H5C_DO_SANITY_CHECKS + /* index len should not change */ + init_index_len = cache_ptr->index_len; +#endif /* H5C_DO_SANITY_CHECKS */ + + /* Examine entries in the LRU list, and flush or clear all entries + * so marked in the target ring. + * + * With the current implementation of flush dependencies, no entry + * in the LRU can have flush dependency children -- thus one pass + * through the LRU will be sufficient. + * + * It is possible that this will change -- hence the assertion. + */ + restart_scan = FALSE; + entry_ptr = cache_ptr->LRU_tail_ptr; + while(((entries_flushed < entries_to_flush) || (entries_cleared < entries_to_clear)) + && (entry_ptr != NULL)) { + hbool_t prev_is_dirty = FALSE; + H5C_cache_entry_t *next_ptr; + + /* Entries in the LRU must not have flush dependency children */ + HDassert(entry_ptr->flush_dep_nchildren == 0); + + /* Remember dirty state of entry to advance to */ + if(entry_ptr->prev != NULL) + prev_is_dirty = entry_ptr->prev->is_dirty; + + /* If the entry is in the ring */ + if(entry_ptr->ring == ring) { + /* If this process needs to clear this entry. */ + if(entry_ptr->clear_on_unprotect) { + HDassert(entry_ptr->is_dirty); + + /* Set entry and flags for operation */ + op_ptr = entry_ptr; + op_flags = clear_flags; + + /* Set next entry appropriately */ + next_ptr = entry_ptr->next; + + /* Reset entry flag */ + entry_ptr->clear_on_unprotect = FALSE; + entries_cleared++; + } /* end if */ + else if(entry_ptr->flush_immediately) { + HDassert(entry_ptr->is_dirty); + + /* Set entry and flags for operation */ + op_ptr = entry_ptr; + op_flags = flush_flags; + + /* Set next entry appropriately */ + next_ptr = entry_ptr->next; + + /* Reset entry flag */ + entry_ptr->flush_immediately = FALSE; + entries_flushed++; + } /* end else-if */ + else { + /* No operation for this entry */ + op_ptr = NULL; + + /* Set next entry appropriately */ + next_ptr = entry_ptr; + } /* end else */ + + /* Advance to next entry */ + entry_ptr = entry_ptr->prev; + + /* Check for operation */ + if(op_ptr) { + /* reset entries_removed_counter and + * last_entry_removed_ptr prior to the call to + * H5C__flush_single_entry() so that we can spot + * unexpected removals of entries from the cache, + * and set the restart_scan flag if proceeding + * would be likely to cause us to scan an entry + * that is no longer in the cache. + * + * Note that as of this writing, this + * case cannot occur in the parallel case. + * + * Note also that there is no test code to verify + * that this code actually works (although similar code + * in the serial version exists and is tested). + */ + cache_ptr->entries_removed_counter = 0; + cache_ptr->last_entry_removed_ptr = NULL; + + if(H5C__flush_single_entry(f, dxpl_id, op_ptr, op_flags) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't flush entry") + + if(cache_ptr->entries_removed_counter != 0 + || cache_ptr->last_entry_removed_ptr != NULL) + restart_scan = TRUE; + } /* end if */ + } /* end if */ + else { + /* Remember "next" pointer (after advancing entries) */ + next_ptr = entry_ptr; + + /* Advance to next entry */ + entry_ptr = entry_ptr->prev; + } /* end else */ + + /* Check for restarts, etc. */ + if((entry_ptr != NULL) && + (restart_scan || (entry_ptr->is_dirty != prev_is_dirty) + || (entry_ptr->next != next_ptr) || entry_ptr->is_protected + || entry_ptr->is_pinned)) { + + /* Something has happened to the LRU -- start over + * from the tail. + * + * Recall that this code should be un-reachable at present, + * as all the operations by entries on flush that could cause + * it to be reachable are disallowed in the parallel case at + * present. Hence the following assertion which should be + * removed if the above changes. + */ + HDassert(!restart_scan); + HDassert(entry_ptr->is_dirty == prev_is_dirty); + HDassert(entry_ptr->next == next_ptr); + HDassert(!entry_ptr->is_protected); + HDassert(!entry_ptr->is_pinned); + + HDassert(FALSE); /* see comment above */ + + restart_scan = FALSE; + entry_ptr = cache_ptr->LRU_tail_ptr; + + H5C__UPDATE_STATS_FOR_LRU_SCAN_RESTART(cache_ptr) + } /* end if */ + } /* end while */ + + /* It is also possible that some of the cleared entries are on the + * pinned list. Must scan that also. + * + * Observe that in the case of the pinned entry list, most of the + * entries will have flush dependency children. As entries with + * flush dependency children may not be flushed until all of their + * children are clean, multiple passes throguh the pinned entry list + * may be required. + * + * WARNING: + * + * As we now allow unpinning, and removal of other entries as a side + * effect of flushing an entry, it is possible that the next entry + * in a PEL scan could either be no longer pinned, or no longer in + * the cache by the time we get to it. + * + * At present, this should not be possible in this case, as we disallow + * such operations in the parallel version of the library. However, + * this may change, and to that end, I have included code to detect + * such changes and cause this function to fail if they are detected. + */ + progress = TRUE; + while(progress && ((entries_flushed < entries_to_flush) || (entries_cleared < entries_to_clear))) { + progress = FALSE; + entry_ptr = cache_ptr->pel_head_ptr; + while((entry_ptr != NULL) && + ((entries_flushed < entries_to_flush) || (entries_cleared < entries_to_clear))) { + H5C_cache_entry_t *prev_ptr; + hbool_t next_is_dirty = FALSE; + + HDassert(entry_ptr->is_pinned); + + /* Remember dirty state of entry to advance to */ + if(entry_ptr->next != NULL) + next_is_dirty = entry_ptr->next->is_dirty; + + if(entry_ptr->ring == ring && entry_ptr->flush_dep_ndirty_children == 0) { + if(entry_ptr->clear_on_unprotect) { + HDassert(entry_ptr->is_dirty); + + /* Set entry and flags for operation */ + op_ptr = entry_ptr; + op_flags = clear_flags; + + /* Reset entry flag */ + entry_ptr->clear_on_unprotect = FALSE; + entries_cleared++; + progress = TRUE; + } /* end if */ + else if(entry_ptr->flush_immediately) { + HDassert(entry_ptr->is_dirty); + + /* Set entry and flags for operation */ + op_ptr = entry_ptr; + op_flags = flush_flags; + + /* Reset entry flag */ + entry_ptr->flush_immediately = FALSE; + entries_flushed++; + progress = TRUE; + } /* end else-if */ + else + /* No operation for this entry */ + op_ptr = NULL; + + /* Check for operation */ + if(op_ptr) { + /* reset entries_removed_counter and + * last_entry_removed_ptr prior to the call to + * H5C__flush_single_entry() so that we can spot + * unexpected removals of entries from the cache, + * and set the restart_scan flag if proceeding + * would be likely to cause us to scan an entry + * that is no longer in the cache. + * + * Note that as of this writing, this + * case cannot occur in the parallel case. + * + * Note also that there is no test code to verify + * that this code actually works (although similar code + * in the serial version exists and is tested). + */ + cache_ptr->entries_removed_counter = 0; + cache_ptr->last_entry_removed_ptr = NULL; + + /* Add this entry to the list of entries to collectively write + * + * This comment is misleading -- the entry will be added to the + * collective write list only if said list exists. + * + * JRM -- 2/9/17 + */ + if(H5C__flush_single_entry(f, dxpl_id, op_ptr, op_flags) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't flush entry") + + if(cache_ptr->entries_removed_counter != 0 + || cache_ptr->last_entry_removed_ptr != NULL) + restart_scan = TRUE; + } /* end if */ + } /* end if */ + + /* Remember "previous" pointer (after advancing entries) */ + prev_ptr = entry_ptr; + + /* Advance to next entry */ + entry_ptr = entry_ptr->next; + + /* Check for restarts, etc. */ + if((entry_ptr != NULL) && + (restart_scan || (entry_ptr->is_dirty != next_is_dirty) + || (entry_ptr->prev != prev_ptr) || entry_ptr->is_protected + || !entry_ptr->is_pinned)) { + /* Something has happened to the pinned entry list -- start + * over from the head. + * + * Recall that this code should be un-reachable at present, + * as all the operations by entries on flush that could cause + * it to be reachable are disallowed in the parallel case at + * present. Hence the following assertion which should be + * removed if the above changes. + */ + + HDassert(!restart_scan); + HDassert(entry_ptr->is_dirty == next_is_dirty); + HDassert(entry_ptr->prev == prev_ptr); + HDassert(!entry_ptr->is_protected); + HDassert(entry_ptr->is_pinned); + + HDassert(FALSE); /* see comment above */ + + restart_scan = FALSE; + + entry_ptr = cache_ptr->pel_head_ptr; + + /* we don't keeps stats for pinned entry list scan + * restarts. If this code ever becomes reachable, + * define the necessary field, and implement the + * the following macro: + * + * H5C__UPDATE_STATS_FOR_PEL_SCAN_RESTART(cache_ptr) + */ + } /* end if */ + } /* end while ( ( entry_ptr != NULL ) && + * ( ( entries_flushed > entries_to_flush ) || + * ( entries_cleared > entries_to_clear ) ) ) + */ + } /* end while ( ( ( entries_flushed > entries_to_flush ) || + * ( entries_cleared > entries_to_clear ) ) && + * ( progress ) ) + */ + +#if H5C_DO_SANITY_CHECKS + HDassert(init_index_len == cache_ptr->index_len); +#endif /* H5C_DO_SANITY_CHECKS */ + + if(entries_flushed != entries_to_flush || entries_cleared != entries_to_clear) { + entry_ptr = cache_ptr->il_head; + while(entry_ptr != NULL) { + HDassert(!entry_ptr->clear_on_unprotect || (entry_ptr->ring > ring)); + HDassert(!entry_ptr->flush_immediately || (entry_ptr->ring > ring)); + entry_ptr = entry_ptr->il_next; + } /* end while */ + + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "can't flush/clear all entries") + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C__flush_candidates_in_ring() */ #endif /* H5_HAVE_PARALLEL */ + diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h index 5b923e9..69e8145 100644 --- a/src/H5Cpkg.h +++ b/src/H5Cpkg.h @@ -4104,6 +4104,23 @@ typedef struct H5C_tag_info_t { * of changes to the file driver info superblock extension * management code needed to support rings. * + * msic_in_progress: As the metadata cache has become re-entrant, and as + * the free space manager code has become more tightly + * integrated with the metadata cache, it is possible that + * a call to H5C_insert_entry() may trigger a call to + * H5C_make_space_in_cache(), which, via H5C__flush_single_entry() + * and client callbacks, may trigger an infinite regression + * of calls to H5C_make_space_in_cache(). + * + * The msic_in_progress boolean flag is used to detect this, + * and prevent the infinite regression that would otherwise + * occur. + * + * Note that this is issue is not hypothetical -- this field + * was added 2/16/17 to address this issue when it was + * exposed by modifications to test/fheap.c to cause it to + * use paged allocation. + * * resize_ctl: Instance of H5C_auto_size_ctl_t containing configuration * data for automatic cache resizing. * @@ -4500,12 +4517,22 @@ typedef struct H5C_tag_info_t { * total_entries_skipped_in_msic: Number of clean entries skipped while * enforcing the min_clean_fraction in H5C__make_space_in_cache(). * + * total_dirty_pf_entries_skipped_in_msic: Number of dirty prefetched entries + * skipped in H5C__make_space_in_cache(). Note that this can + * only occur when a file is opened R/O with a cache image + * containing dirty entries. + * * total_entries_scanned_in_msic: Number of clean entries skipped while * enforcing the min_clean_fraction in H5C__make_space_in_cache(). * * max_entries_skipped_in_msic: Maximum number of clean entries skipped * in any one call to H5C__make_space_in_cache(). * + * max_dirty_pf_entries_skipped_in_msic: Maximum number of dirty prefetched + * entries skipped in any one call to H5C__make_space_in_cache(). + * Note that this can only occur when the file is opened + * R/O with a cache image containing dirty entries. + * * max_entries_scanned_in_msic: Maximum number of entries scanned over * in any one call to H5C__make_space_in_cache(). * @@ -4733,6 +4760,7 @@ struct H5C_t { hbool_t cache_full; hbool_t size_decreased; hbool_t resize_in_progress; + hbool_t msic_in_progress; H5C_auto_size_ctl_t resize_ctl; /* Fields for epoch markers used in automatic cache size adjustment */ @@ -4822,8 +4850,10 @@ struct H5C_t { /* Fields for tracking 'make space in cache' (msic) operations */ int64_t calls_to_msic; int64_t total_entries_skipped_in_msic; + int64_t total_dirty_pf_entries_skipped_in_msic; int64_t total_entries_scanned_in_msic; int32_t max_entries_skipped_in_msic; + int32_t max_dirty_pf_entries_skipped_in_msic; int32_t max_entries_scanned_in_msic; int64_t entries_scanned_to_make_space; @@ -4871,7 +4901,8 @@ typedef int (*H5C_tag_iter_cb_t)(H5C_cache_entry_t *entry, void *ctx); /******************************/ /* Package Private Prototypes */ /******************************/ -H5_DLL herr_t H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id); +H5_DLL herr_t H5C__prep_image_for_file_close(H5F_t *f, hid_t dxpl_id, + hbool_t *image_generated); H5_DLL herr_t H5C__deserialize_prefetched_entry(H5F_t *f, hid_t dxpl_id, H5C_t * cache_ptr, H5C_cache_entry_t** entry_ptr_ptr, const H5C_class_t * type, haddr_t addr, void * udata); diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index 28eacf2..539dece 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -185,6 +185,7 @@ * H5C__TAKE_OWNERSHIP_FLAG * H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG * H5C__GENERATE_IMAGE_FLAG + * H5C__UPDATE_PAGE_BUFFER_FLAG */ #define H5C__NO_FLAGS_SET 0x00000 #define H5C__SET_FLUSH_MARKER_FLAG 0x00001 @@ -205,6 +206,7 @@ #define H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG 0x08000 #define H5C__DURING_FLUSH_FLAG 0x10000 /* Set when the entire cache is being flushed */ #define H5C__GENERATE_IMAGE_FLAG 0x20000 /* Set during parallel I/O */ +#define H5C__UPDATE_PAGE_BUFFER_FLAG 0x40000 /* Set during parallel I/O */ /* Debugging/sanity checking/statistics settings */ #ifndef NDEBUG @@ -1066,9 +1068,9 @@ typedef int H5C_ring_t; * * is_read_only: Boolean flag that is only meaningful if is_protected is * TRUE. In this circumstance, it indicates whether the - * entry has been protected read only, or read/write. + * entry has been protected read-only, or read/write. * - * If the entry has been protected read only (i.e. is_protected + * If the entry has been protected read-only (i.e. is_protected * and is_read_only are both TRUE), we allow the entry to be * protected more than once. * @@ -1078,7 +1080,7 @@ typedef int H5C_ring_t; * the entry is actually unprotected. * * ro_ref_count: Integer field used to maintain a count of the number of - * outstanding read only protects on this entry. This field + * outstanding read-only protects on this entry. This field * must be zero whenever either is_protected or is_read_only * are TRUE. * @@ -1458,13 +1460,13 @@ typedef int H5C_ring_t; * the load of a cache image block, although other scenarios * are contemplated for the use of this feature. Note that * unlike the usual prefetch situation, this means that a - * pre fetched entry can be dirty, and/or can be a party to + * prefetched entry can be dirty, and/or can be a party to * flush dependency relationship(s). This complicates matters * somewhat. * - * The essential feature of a pre-fetched entry is that it + * The essential feature of a prefetched entry is that it * consists only of a buffer containing the on disk image of - * the entry. Thus it must be deserialized before it can + * the entry. Thus it must be deserialized before it can * be passed back to the library on a protect call. This * task is handled by H5C_deserialized_prefetched_entry(). * In essence, this routine calls the deserialize callback @@ -1475,7 +1477,7 @@ typedef int H5C_ring_t; * * Further, if the prefetched entry is a flush dependency parent, * all its flush dependency children (which must also be - * pre-fetched entries), must be tranfered to the new cache + * prefetched entries), must be tranfered to the new cache * entry returned by the deserailization callback. * * Finally, if the prefetched entry is a flush dependency child, @@ -1511,6 +1513,46 @@ typedef int H5C_ring_t; * * This field must be zero if prefetched is FALSE. * + * prefetched_dirty: Boolean field that must be set to FALSE unless the + * following conditions hold: + * + * 1) The file has been opened R/O. + * + * 2) The entry is either a prefetched entry, or was + * re-constructed from a prefetched entry. + * + * 3) The base prefetched entry was marked dirty. + * + * This field exists to solve the following problem with + * files containing cache images that are opened R/O. + * + * If the cache image contains a dirty entry, that entry + * must be marked clean when it is inserted into the cache + * in the read-only case, as otherwise the metadata cache + * will attempt to flush it on file close -- which is poor + * form in the read-only case. + * + * However, since the entry is marked clean, it is possible + * that the metadata cache will evict it if the size of the + * metadata in the file exceeds the size of the metadata cache, + * and the application visits much of this data. + * + * If this happens, and the metadata cache is then asked for + * this entry, it will attempt to read it from file, and will + * obtain either obsolete or invalid data depending on whether + * the entry has ever been written to it assigned location in + * the file. + * + * With this background, the purpose of this field should be + * obvious -- when set, it allows the eviction candidate + * selection code to skip over the entry, thus avoiding the + * issue. + * + * Since the issue only arises in the R/O case, there is + * no possible interaction with SWMR. There are also + * potential interactions with Evict On Close -- at present, + * we deal with this by disabling EOC in the R/O case. + * * serialization_count: Integer field used to maintain a count of the * number of times each entry is serialized during cache * serialization. While no entry should be serialized more than @@ -1627,6 +1669,7 @@ typedef struct H5C_cache_entry_t { hbool_t prefetched; int prefetch_type_id; int32_t age; + hbool_t prefetched_dirty; #ifndef NDEBUG /* debugging field */ int serialization_count; @@ -2208,6 +2251,7 @@ H5_DLL herr_t H5C_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, haddr_t addr, unsigned flags); H5_DLL herr_t H5C_flush_cache(H5F_t *f, hid_t dxpl_id, unsigned flags); H5_DLL herr_t H5C_flush_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag); +H5_DLL herr_t H5C_force_cache_image_load(H5F_t * f, hid_t dxpl_id); H5_DLL herr_t H5C_evict_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag, hbool_t match_global); H5_DLL herr_t H5C_expunge_tag_type_metadata(H5F_t *f, hid_t dxpl_id, haddr_t tag, int type_id, unsigned flags); H5_DLL herr_t H5C_get_tag(const void *thing, /*OUT*/ haddr_t *tag); @@ -2259,7 +2303,6 @@ H5_DLL herr_t H5C_set_trace_file_ptr(H5C_t *cache_ptr, FILE *trace_file_ptr); H5_DLL herr_t H5C_stats(H5C_t *cache_ptr, const char *cache_name, hbool_t display_detailed_stats); H5_DLL void H5C_stats__reset(H5C_t *cache_ptr); -H5_DLL herr_t H5C_dump_cache(H5C_t *cache_ptr, const char *cache_name); H5_DLL herr_t H5C_unpin_entry(void *thing); H5_DLL herr_t H5C_destroy_flush_dependency(void *parent_thing, void *child_thing); H5_DLL herr_t H5C_unprotect(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *thing, @@ -2273,9 +2316,11 @@ H5_DLL herr_t H5C_retag_entries(H5C_t * cache_ptr, haddr_t src_tag, haddr_t dest H5_DLL herr_t H5C_cork(H5C_t *cache_ptr, haddr_t obj_addr, unsigned action, hbool_t *corked); H5_DLL herr_t H5C_get_entry_ring(const H5F_t *f, haddr_t addr, H5C_ring_t *ring); H5_DLL herr_t H5C_unsettle_entry_ring(void *thing); +H5_DLL herr_t H5C_unsettle_ring(H5F_t * f, H5C_ring_t ring); H5_DLL herr_t H5C_remove_entry(void *thing); H5_DLL herr_t H5C_cache_image_status(H5F_t * f, hbool_t *load_ci_ptr, hbool_t *write_ci_ptr); +H5_DLL hbool_t H5C_cache_image_pending(const H5C_t *cache_ptr); #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5C_apply_candidate_list(H5F_t *f, hid_t dxpl_id, @@ -2289,6 +2334,8 @@ H5_DLL herr_t H5C_mark_entries_as_clean(H5F_t *f, hid_t dxpl_id, unsigned ce_arr #endif /* H5_HAVE_PARALLEL */ #ifndef NDEBUG /* debugging functions */ +H5_DLL herr_t H5C_dump_cache(H5C_t *cache_ptr, const char *cache_name); +H5_DLL herr_t H5C_dump_cache_LRU(H5C_t *cache_ptr, const char *cache_name); H5_DLL hbool_t H5C_get_serialization_in_progress(const H5C_t *cache_ptr); H5_DLL hbool_t H5C_cache_is_clean(const H5C_t *cache_ptr, H5C_ring_t inner_ring); H5_DLL herr_t H5C_dump_cache_skip_list(H5C_t *cache_ptr, char *calling_fcn); diff --git a/src/H5Ctag.c b/src/H5Ctag.c index 157a838..a9bcca1 100644 --- a/src/H5Ctag.c +++ b/src/H5Ctag.c @@ -465,7 +465,7 @@ H5C__evict_tagged_entries_cb(H5C_cache_entry_t *entry, void *_ctx) entry and we'll loop back around again (as evicting other entries will hopefully unpin this entry) */ ctx->pinned_entries_need_evicted = TRUE; - else { + else if(!entry->prefetched_dirty) { /* Evict the Entry */ if(H5C__flush_single_entry(ctx->f, ctx->dxpl_id, entry, H5C__FLUSH_INVALIDATE_FLAG | H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, H5_ITER_ERROR, "Entry eviction failed.") @@ -406,7 +406,7 @@ H5Fis_hdf5(const char *name) HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "no file name specified") /* call the private is_HDF5 function */ - if((ret_value = H5F_is_hdf5(name, H5AC_ind_read_dxpl_id)) < 0) + if((ret_value = H5F__is_hdf5(name, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id)) < 0) HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable open file") done: @@ -535,7 +535,7 @@ done: * Modifications: * Robb Matzke, 1997-07-18 * File struct creation and destruction is through H5F_new() and - * H5F_dest(). Reading the root symbol table entry is done with + * H5F__dest(). Reading the root symbol table entry is done with * H5G_decode(). * * Robb Matzke, 1997-09-23 @@ -713,12 +713,12 @@ H5Fflush(hid_t object_id, H5F_scope_t scope) /* Flush other files, depending on scope */ if(H5F_SCOPE_GLOBAL == scope) { /* Call the flush routine for mounted file hierarchies */ - if(H5F_flush_mounts(f, H5AC_ind_read_dxpl_id) < 0) + if(H5F_flush_mounts(f, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush mounted file hierarchy") } /* end if */ else { /* Call the flush routine, for this file */ - if(H5F_flush(f, H5AC_ind_read_dxpl_id, FALSE) < 0) + if(H5F__flush(f, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file's cached information") } /* end else */ } /* end if */ @@ -773,7 +773,7 @@ H5Fclose(hid_t file_id) if((nref = H5I_get_ref(file_id, FALSE)) < 0) HGOTO_ERROR(H5E_ATOM, H5E_CANTGET, FAIL, "can't get ID ref count") if(nref == 1) - if(H5F_flush(f, H5AC_ind_read_dxpl_id, FALSE) < 0) + if(H5F__flush(f, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache") } /* end if */ @@ -838,7 +838,7 @@ H5Freopen(hid_t file_id) done: if(ret_value < 0 && new_file) - if(H5F_dest(new_file, H5AC_ind_read_dxpl_id, FALSE) < 0) + if(H5F__dest(new_file, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, FALSE) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "can't close file") FUNC_LEAVE_API(ret_value) @@ -1042,7 +1042,7 @@ H5Fget_file_image(hid_t file_id, void *buf_ptr, size_t buf_len) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a file ID") /* call private get_file_image function */ - if((ret_value = H5F_get_file_image(file, buf_ptr, buf_len, H5AC_ind_read_dxpl_id)) < 0) + if((ret_value = H5F_get_file_image(file, buf_ptr, buf_len, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id)) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file image") done: @@ -1619,7 +1619,7 @@ H5Fstart_swmr_write(hid_t file_id) H5G_name_t *obj_paths=NULL; /* Group hierarchy path */ size_t u; /* Local index variable */ hbool_t setup = FALSE; /* Boolean flag to indicate whether SWMR setting is enabled */ - H5F_io_info_t fio_info; /* I/O info for operation */ + H5F_io_info2_t fio_info; /* I/O info for operation */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) @@ -1650,7 +1650,7 @@ H5Fstart_swmr_write(hid_t file_id) HGOTO_ERROR(H5E_FILE, H5E_UNSUPPORTED, FAIL, "can't have both SWMR and MDC cache image") /* Flush data buffers */ - if(H5F_flush(file, H5AC_ind_read_dxpl_id, FALSE) < 0) + if(H5F__flush(file, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file's cached information") /* Get the # of opened named datatypes and attributes */ @@ -1704,7 +1704,9 @@ H5Fstart_swmr_write(hid_t file_id) /* Set up I/O info for operation */ fio_info.f = file; - if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") /* Flush and reset the accumulator */ @@ -1923,7 +1925,7 @@ H5Fset_latest_format(hid_t file_id, hbool_t latest_format) latest_flags = H5F_USE_LATEST_FLAGS(f, H5F_LATEST_ALL_FLAGS); if(latest_format != (H5F_LATEST_ALL_FLAGS == latest_flags)) { /* Call the flush routine, for this file */ - if(H5F_flush(f, H5AC_ind_read_dxpl_id, FALSE) < 0) + if(H5F__flush(f, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file's cached information") /* Toggle the 'latest format' flag */ @@ -1973,7 +1975,9 @@ H5Fformat_convert(hid_t fid) /* Check for persistent freespace manager, which needs to be downgraded */ if(!(f->shared->fs_strategy == H5F_FILE_SPACE_STRATEGY_DEF && - f->shared->fs_threshold == H5F_FREE_SPACE_THRESHOLD_DEF)) { + f->shared->fs_persist == H5F_FREE_SPACE_PERSIST_DEF && + f->shared->fs_threshold == H5F_FREE_SPACE_THRESHOLD_DEF && + f->shared->fs_page_size == H5F_FILE_SPACE_PAGE_SIZE_DEF)) { /* Check to remove free-space manager info message from superblock extension */ if(H5F_addr_defined(f->shared->sblock->ext_addr)) if(H5F_super_ext_remove_msg(f, H5AC_ind_read_dxpl_id, H5O_FSINFO_ID) < 0) @@ -1985,7 +1989,9 @@ H5Fformat_convert(hid_t fid) /* Set non-persistent freespace manager */ f->shared->fs_strategy = H5F_FILE_SPACE_STRATEGY_DEF; + f->shared->fs_persist = H5F_FREE_SPACE_PERSIST_DEF; f->shared->fs_threshold = H5F_FREE_SPACE_THRESHOLD_DEF; + f->shared->fs_page_size = H5F_FILE_SPACE_PAGE_SIZE_DEF; /* Indicate that the superblock should be marked dirty */ mark_dirty = TRUE; @@ -2004,3 +2010,79 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Fformat_convert() */ + +/*------------------------------------------------------------------------- + * Function: H5Freset_page_buffering_stats + * + * Purpose: Resets statistics for the page buffer layer. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5Freset_page_buffering_stats(hid_t file_id) +{ + H5F_t *file; /* File to reset stats on */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE1("e", "i", file_id); + + /* Check args */ + if(NULL == (file = (H5F_t *)H5I_object(file_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid file identifier") + if(NULL == file->shared->page_buf) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "page buffering not enabled on file") + + /* Reset the statistics */ + if(H5PB_reset_stats(file->shared->page_buf) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't reset stats for page buffering") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Freset_page_buffering_stats() */ + + +/*------------------------------------------------------------------------- + * Function: H5Fget_page_buffering_stats + * + * Purpose: Retrieves statistics for the page buffer layer. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5Fget_page_buffering_stats(hid_t file_id, unsigned accesses[2], unsigned hits[2], + unsigned misses[2], unsigned evictions[2], unsigned bypasses[2]) +{ + H5F_t *file; /* File object for file ID */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE6("e", "i*Iu*Iu*Iu*Iu*Iu", file_id, accesses, hits, misses, evictions, + bypasses); + + /* Check args */ + if(NULL == (file = (H5F_t *)H5I_object_verify(file_id, H5I_FILE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a file ID") + if(NULL == file->shared->page_buf) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "page buffering not enabled on file") + if(NULL == accesses || NULL == hits || NULL == misses || NULL == evictions || NULL == bypasses) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL input parameters for stats") + + /* Get the statistics */ + if(H5PB_get_stats(file->shared->page_buf, accesses, hits, misses, evictions, bypasses) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't retrieve stats for page buffering") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Fget_page_buffering_stats() */ + @@ -1530,7 +1530,7 @@ herr_t H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, void *buf/*out*/) { - H5P_genplist_t *dxpl; /* DXPL object */ + H5FD_io_info_t fdio_info; /* File driver I/O object */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) @@ -1549,13 +1549,24 @@ H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size if(!buf) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null result buffer") - /* Get the DXPL plist object for DXPL ID */ - if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + /* Set up the file driver I/O info object */ + fdio_info.file = file; + if(H5FD_MEM_DRAW == type) { + if(NULL == (fdio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end if */ + else { + if(NULL == (fdio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end else */ /* Do the real work */ /* (Note compensating for base address addition in internal routine) */ - if(H5FD_read(file, dxpl, type, addr - file->base_addr, size, buf) < 0) + if(H5FD_read(&fdio_info, type, addr - file->base_addr, size, buf) < 0) HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "file read request failed") done: @@ -1584,7 +1595,7 @@ herr_t H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, const void *buf) { - H5P_genplist_t *dxpl; /* DXPL object */ + H5FD_io_info_t fdio_info; /* File driver I/O object */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) @@ -1602,13 +1613,24 @@ H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t siz if(!buf) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null buffer") - /* Get the DXPL plist object for DXPL ID */ - if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + /* Set up the file driver I/O info object */ + fdio_info.file = file; + if(H5FD_MEM_DRAW == type) { + if(NULL == (fdio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end if */ + else { + if(NULL == (fdio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end else */ /* The real work */ /* (Note compensating for base address addition in internal routine) */ - if(H5FD_write(file, dxpl, type, addr - file->base_addr, size, buf) < 0) + if(H5FD_write(&fdio_info, type, addr - file->base_addr, size, buf) < 0) HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "file write request failed") done: @@ -2032,3 +2054,27 @@ H5FD_get_base_addr(const H5FD_t *file) FUNC_LEAVE_NOAPI(file->base_addr) } /* end H5FD_get_base_addr() */ + +/*-------------------------------------------------------------------------- + * Function: H5FD_set_paged_aggr + * + * Purpose: Set "paged_aggr" for the file. + * + * Return: Non-negative if succeed; negative if fails. + * + * Programmer: Vailin Choi; April 2013 + * + *-------------------------------------------------------------------------- + */ +herr_t +H5FD_set_paged_aggr(H5FD_t *file, hbool_t paged) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(file); + + /* Indicate whether paged aggregation for handling file space is enabled or not */ + file->paged_aggr = paged; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD_set_paged_aggr() */ diff --git a/src/H5FDint.c b/src/H5FDint.c index 744c3d1..0809ac8 100644 --- a/src/H5FDint.c +++ b/src/H5FDint.c @@ -93,12 +93,9 @@ *------------------------------------------------------------------------- */ herr_t -H5FD_locate_signature(H5FD_t *file, -#ifndef H5_DEBUG_BUILD -const -#endif /* H5_DEBUG_BUILD */ -H5P_genplist_t *dxpl, haddr_t *sig_addr) +H5FD_locate_signature(H5FD_io_info_t *fdio_info, haddr_t *sig_addr) { + H5FD_t *file; haddr_t addr, eoa, eof; uint8_t buf[H5F_SIGNATURE_LEN]; unsigned n, maxpow; @@ -106,6 +103,10 @@ H5P_genplist_t *dxpl, haddr_t *sig_addr) FUNC_ENTER_NOAPI_NOINIT + HDassert(fdio_info); + file = fdio_info->file; + HDassert(file); + /* Find the least N such that 2^N is larger than the file size */ eof = H5FD_get_eof(file, H5FD_MEM_SUPER); eoa = H5FD_get_eoa(file, H5FD_MEM_SUPER); @@ -124,7 +125,7 @@ H5P_genplist_t *dxpl, haddr_t *sig_addr) addr = (8 == n) ? 0 : (haddr_t)1 << n; if(H5FD_set_eoa(file, H5FD_MEM_SUPER, addr + H5F_SIGNATURE_LEN) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to set EOA value for file signature") - if(H5FD_read(file, dxpl, H5FD_MEM_SUPER, addr, (size_t)H5F_SIGNATURE_LEN, buf) < 0) + if(H5FD_read(fdio_info, H5FD_MEM_SUPER, addr, (size_t)H5F_SIGNATURE_LEN, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to read file signature") if(!HDmemcmp(buf, H5F_SIGNATURE, (size_t)H5F_SIGNATURE_LEN)) break; @@ -162,29 +163,36 @@ done: *------------------------------------------------------------------------- */ herr_t -H5FD_read(H5FD_t *file, -#ifndef H5_DEBUG_BUILD -const -#endif /* H5_DEBUG_BUILD */ -H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t addr, +H5FD_read(H5FD_io_info_t *fdio_info, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/) { + H5FD_t *file; + H5P_genplist_t *io_dxpl; haddr_t eoa = HADDR_UNDEF; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) + HDassert(fdio_info); + file = fdio_info->file; HDassert(file && file->cls); - HDassert(TRUE == H5P_class_isa(H5P_CLASS(dxpl), H5P_CLS_DATASET_XFER_g)); + HDassert(TRUE == H5P_class_isa(H5P_CLASS(fdio_info->meta_dxpl), H5P_CLS_DATASET_XFER_g)); + HDassert(TRUE == H5P_class_isa(H5P_CLASS(fdio_info->raw_dxpl), H5P_CLS_DATASET_XFER_g)); HDassert(buf); + /* Set up proper DXPL for I/O */ + if(H5FD_MEM_DRAW == type) + io_dxpl = fdio_info->raw_dxpl; + else + io_dxpl = fdio_info->meta_dxpl; + /* Sanity check the dxpl type against the mem type */ #ifdef H5_DEBUG_BUILD { H5FD_dxpl_type_t dxpl_type; /* Property indicating the type of the internal dxpl */ /* get the dxpl type */ - if(H5P_get(dxpl, H5FD_DXPL_TYPE_NAME, &dxpl_type) < 0) + if(H5P_get(io_dxpl, H5FD_DXPL_TYPE_NAME, &dxpl_type) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't retrieve dxpl type") /* we shouldn't be here if the dxpl is labeled with NO I/O */ @@ -219,7 +227,7 @@ H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t addr, HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, addr = %llu, size = %llu, eoa = %llu", (unsigned long long)(addr + file->base_addr), (unsigned long long)size, (unsigned long long)eoa) /* Dispatch to driver */ - if((file->cls->read)(file, type, H5P_PLIST_ID(dxpl), addr + file->base_addr, size, buf) < 0) + if((file->cls->read)(file, type, H5P_PLIST_ID(io_dxpl), addr + file->base_addr, size, buf) < 0) HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "driver read request failed") done: @@ -241,29 +249,36 @@ done: *------------------------------------------------------------------------- */ herr_t -H5FD_write(H5FD_t *file, -#ifndef H5_DEBUG_BUILD -const -#endif /* H5_DEBUG_BUILD */ -H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t addr, +H5FD_write(const H5FD_io_info_t *fdio_info, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf) { + H5FD_t *file; + H5P_genplist_t *io_dxpl; haddr_t eoa = HADDR_UNDEF; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) + HDassert(fdio_info); + file = fdio_info->file; HDassert(file && file->cls); - HDassert(TRUE == H5P_class_isa(H5P_CLASS(dxpl), H5P_CLS_DATASET_XFER_g)); + HDassert(TRUE == H5P_class_isa(H5P_CLASS(fdio_info->meta_dxpl), H5P_CLS_DATASET_XFER_g)); + HDassert(TRUE == H5P_class_isa(H5P_CLASS(fdio_info->raw_dxpl), H5P_CLS_DATASET_XFER_g)); HDassert(buf); + /* Set up proper DXPL for I/O */ + if(H5FD_MEM_DRAW == type) + io_dxpl = fdio_info->raw_dxpl; + else + io_dxpl = fdio_info->meta_dxpl; + /* Sanity check the dxpl type against the mem type */ #ifdef H5_DEBUG_BUILD { H5FD_dxpl_type_t dxpl_type; /* Property indicating the type of the internal dxpl */ /* get the dxpl type */ - if(H5P_get(dxpl, H5FD_DXPL_TYPE_NAME, &dxpl_type) < 0) + if(H5P_get(io_dxpl, H5FD_DXPL_TYPE_NAME, &dxpl_type) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't retrieve dxpl type") /* we shouldn't be here if the dxpl is labeled with NO I/O */ @@ -291,7 +306,7 @@ H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t addr, (unsigned long long)(addr+ file->base_addr), (unsigned long long)size, (unsigned long long)eoa) /* Dispatch to driver */ - if((file->cls->write)(file, type, H5P_PLIST_ID(dxpl), addr + file->base_addr, size, buf) < 0) + if((file->cls->write)(file, type, H5P_PLIST_ID(io_dxpl), addr + file->base_addr, size, buf) < 0) HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "driver write request failed") done: diff --git a/src/H5FDlog.c b/src/H5FDlog.c index 03228d2..f320946 100644 --- a/src/H5FDlog.c +++ b/src/H5FDlog.c @@ -934,13 +934,7 @@ H5FD_log_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hsi /* Compute the address for the block to allocate */ addr = file->eoa; - /* Check if we need to align this block */ - if(size >= file->pub.threshold) { - /* Check for an already aligned block */ - if(addr % file->pub.alignment != 0) - addr = ((addr / file->pub.alignment) + 1) * file->pub.alignment; - } /* end if */ - + /* Extend the end-of-allocated space address */ file->eoa = addr + size; /* Retain the (first) flavor of the information written to the file */ diff --git a/src/H5FDmulti.c b/src/H5FDmulti.c index 7e12869..0ab7fc5 100644 --- a/src/H5FDmulti.c +++ b/src/H5FDmulti.c @@ -1194,6 +1194,8 @@ H5FD_multi_query(const H5FD_t *_f, unsigned long *flags /* out */) *flags = 0; *flags |= H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */ *flags |= H5FD_FEAT_AGGREGATE_SMALLDATA; /* OK to aggregate "small" raw data allocations */ + *flags |= H5FD_FEAT_USE_ALLOC_SIZE; /* OK just pass the allocation size to the alloc callback */ + *flags |= H5FD_FEAT_PAGED_AGGR; /* OK special file space mapping for paged aggregation */ } /* end if */ return(0); @@ -1538,6 +1540,14 @@ H5FD_multi_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size) mmt = file->fa.memb_map[type]; if (H5FD_MEM_DEFAULT==mmt) mmt = type; + /* XXX: NEED to work on this again */ + if(file->pub.paged_aggr) { + ALL_MEMBERS(mt) { + if(file->memb[mt]) + file->memb[mt]->paged_aggr = file->pub.paged_aggr; + } END_MEMBERS; + } + if (HADDR_UNDEF==(addr=H5FDalloc(file->memb[mmt], mmt, dxpl_id, size))) H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "member file can't alloc", HADDR_UNDEF) addr += file->fa.memb_addr[mmt]; diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h index 45f0187..c64ec30 100644 --- a/src/H5FDprivate.h +++ b/src/H5FDprivate.h @@ -115,6 +115,19 @@ typedef enum { #define H5FD_DXPL_TYPE_NAME "H5P_dxpl_type" #endif /* H5_DEBUG_BUILD */ +/* I/O Info for an operation */ +typedef struct H5FD_io_info_t { + H5FD_t *file; /* File driver object */ +#ifndef H5_DEBUG_BUILD + const +#endif /* H5_DEBUG_BUILD */ + H5P_genplist_t *meta_dxpl; /* Metadata DXPL object */ +#ifndef H5_DEBUG_BUILD + const +#endif /* H5_DEBUG_BUILD */ + H5P_genplist_t *raw_dxpl; /* Raw data DXPL object */ +} H5FD_io_info_t; + /*****************************/ /* Library Private Variables */ @@ -126,15 +139,10 @@ typedef enum { /******************************/ /* Forward declarations for prototype arguments */ -struct H5P_genplist_t; struct H5F_t; H5_DLL int H5FD_term_interface(void); -H5_DLL herr_t H5FD_locate_signature(H5FD_t *file, -#ifndef H5_DEBUG_BUILD -const -#endif /* H5_DEBUG_BUILD */ -H5P_genplist_t *dxpl, haddr_t *sig_addr); +H5_DLL herr_t H5FD_locate_signature(H5FD_io_info_t *fdio_info, haddr_t *sig_addr); H5_DLL H5FD_class_t *H5FD_get_class(hid_t id); H5_DLL hsize_t H5FD_sb_size(H5FD_t *file); H5_DLL herr_t H5FD_sb_encode(H5FD_t *file, char *name/*out*/, uint8_t *buf); @@ -146,8 +154,8 @@ H5_DLL H5FD_t *H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); H5_DLL herr_t H5FD_close(H5FD_t *file); H5_DLL int H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2); -H5_DLL haddr_t H5FD_alloc(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, struct H5F_t *f, - hsize_t size, haddr_t *align_addr, hsize_t *align_size); +H5_DLL haddr_t H5FD_alloc(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, + struct H5F_t *f, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size); H5_DLL herr_t H5FD_free(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, struct H5F_t *f, haddr_t addr, hsize_t size); H5_DLL htri_t H5FD_try_extend(H5FD_t *file, H5FD_mem_t type, struct H5F_t *f, @@ -159,17 +167,9 @@ H5_DLL haddr_t H5FD_get_maxaddr(const H5FD_t *file); H5_DLL herr_t H5FD_get_feature_flags(const H5FD_t *file, unsigned long *feature_flags); H5_DLL herr_t H5FD_set_feature_flags(H5FD_t *file, unsigned long feature_flags); H5_DLL herr_t H5FD_get_fs_type_map(const H5FD_t *file, H5FD_mem_t *type_map); -H5_DLL herr_t H5FD_read(H5FD_t *file, -#ifndef H5_DEBUG_BUILD -const -#endif /* H5_DEBUG_BUILD */ -H5P_genplist_t *dxpl, H5FD_mem_t type, +H5_DLL herr_t H5FD_read(H5FD_io_info_t *fdio_info, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/); -H5_DLL herr_t H5FD_write(H5FD_t *file, -#ifndef H5_DEBUG_BUILD -const -#endif /* H5_DEBUG_BUILD */ -H5P_genplist_t *dxpl, H5FD_mem_t type, +H5_DLL herr_t H5FD_write(const H5FD_io_info_t *fdio_info, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf); H5_DLL herr_t H5FD_flush(H5FD_t *file, hid_t dxpl_id, hbool_t closing); H5_DLL herr_t H5FD_truncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing); @@ -179,6 +179,7 @@ H5_DLL herr_t H5FD_get_fileno(const H5FD_t *file, unsigned long *filenum); H5_DLL herr_t H5FD_get_vfd_handle(H5FD_t *file, hid_t fapl, void** file_handle); H5_DLL herr_t H5FD_set_base_addr(H5FD_t *file, haddr_t base_addr); H5_DLL haddr_t H5FD_get_base_addr(const H5FD_t *file); +H5_DLL herr_t H5FD_set_paged_aggr(H5FD_t *file, hbool_t paged); /* Function prototypes for MPI based VFDs*/ #ifdef H5_HAVE_PARALLEL diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h index 436be10..883b28d 100644 --- a/src/H5FDpublic.h +++ b/src/H5FDpublic.h @@ -239,6 +239,19 @@ typedef enum H5F_mem_t H5FD_mem_t; * driver supports the single-writer/multiple-readers I/O pattern. */ #define H5FD_FEAT_SUPPORTS_SWMR_IO 0x00001000 + /* + * Defining H5FD_FEAT_USE_ALLOC_SIZE for a VFL driver + * means that the library will just pass the allocation size to the + * the driver's allocation callback which will eventually handle alignment. + * This is specifically used for the multi/split driver. + */ +#define H5FD_FEAT_USE_ALLOC_SIZE 0x00002000 + /* + * Defining H5FD_FEAT_PAGED_AGGR for a VFL driver + * means that the driver needs special file space mapping for paged aggregation. + * This is specifically used for the multi/split driver. + */ +#define H5FD_FEAT_PAGED_AGGR 0x00004000 /* Forward declaration */ typedef struct H5FD_t H5FD_t; @@ -307,6 +320,7 @@ struct H5FD_t { /* Space allocation management fields */ hsize_t threshold; /* Threshold for alignment */ hsize_t alignment; /* Allocation alignment */ + hbool_t paged_aggr; /* Paged aggregation for file space is enabled or not */ }; /* Define enum for the source of file image callbacks */ diff --git a/src/H5FDspace.c b/src/H5FDspace.c index fcddecf..0ad3cf0 100644 --- a/src/H5FDspace.c +++ b/src/H5FDspace.c @@ -99,7 +99,7 @@ H5FL_DEFINE(H5FD_free_t); *------------------------------------------------------------------------- */ static haddr_t -H5FD_extend(H5FD_t *file, H5FD_mem_t type, hbool_t new_block, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size) +H5FD_extend(H5FD_t *file, H5FD_mem_t type, hsize_t size) { hsize_t orig_size = size; /* Original allocation size */ haddr_t eoa; /* Address of end-of-allocated space */ @@ -117,40 +117,18 @@ H5FD_extend(H5FD_t *file, H5FD_mem_t type, hbool_t new_block, hsize_t size, hadd /* Get current end-of-allocated space address */ eoa = file->cls->get_eoa(file, type); - /* Compute extra space to allocate, if this is a new block and should be aligned */ - extra = 0; - if(new_block && file->alignment > 1 && orig_size >= file->threshold) { - hsize_t mis_align; /* Amount EOA is misaligned */ - - /* Check for EOA already aligned */ - if((mis_align = (eoa % file->alignment)) > 0) { - extra = file->alignment - mis_align; - if(frag_addr) - *frag_addr = eoa - file->base_addr; /* adjust for file's base address */ - if(frag_size) - *frag_size = extra; - } /* end if */ - } /* end if */ - - /* Add in extra allocation amount */ - size += extra; - /* Check for overflow when extending */ if(H5F_addr_overflow(eoa, size) || (eoa + size) > file->maxaddr) HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "file allocation request failed") - /* Set the [possibly aligned] address to return */ - ret_value = eoa + extra; + /* Set the [NOT aligned] address to return */ + ret_value = eoa; /* Extend the end-of-allocated space address */ eoa += size; if(file->cls->set_eoa(file, type, eoa) < 0) HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "file allocation request failed") - /* Post-condition sanity check */ - if(new_block && file->alignment && orig_size >= file->threshold) - HDassert(!(ret_value % file->alignment)); - done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_extend() */ @@ -160,6 +138,8 @@ done: * Function: H5FD_alloc_real * * Purpose: Allocate space in the file with the VFD + * Note: the handling of alignment is moved up from each driver to + * this routine. * * Return: Success: The format address of the new file memory. * Failure: The undefined address HADDR_UNDEF @@ -170,8 +150,14 @@ done: *------------------------------------------------------------------------- */ haddr_t -H5FD_alloc_real(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size) +H5FD_alloc_real(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, hsize_t size, + haddr_t *frag_addr, hsize_t *frag_size) { + hsize_t orig_size = size; /* Original allocation size */ + haddr_t eoa; /* Address of end-of-allocated space */ + hsize_t extra; /* Extra space to allocate, to align request */ + unsigned long flags = 0; /* Driver feature flags */ + hbool_t use_alloc_size; /* Just pass alloc size to the driver */ haddr_t ret_value = HADDR_UNDEF; /* Return value */ FUNC_ENTER_NOAPI(HADDR_UNDEF) @@ -185,16 +171,53 @@ HDfprintf(stderr, "%s: type = %u, size = %Hu\n", FUNC, (unsigned)type, size); HDassert(type >= H5FD_MEM_DEFAULT && type < H5FD_MEM_NTYPES); HDassert(size > 0); + /* Check for query driver and call it */ + if(file->cls->query) + (file->cls->query)(file, &flags); + + /* Check for the driver feature flag */ + use_alloc_size = flags & H5FD_FEAT_USE_ALLOC_SIZE; + + /* Get current end-of-allocated space address */ + eoa = file->cls->get_eoa(file, type); + + /* Compute extra space to allocate, if this should be aligned */ + extra = 0; + if(!file->paged_aggr && file->alignment > 1 && orig_size >= file->threshold) { + hsize_t mis_align; /* Amount EOA is misaligned */ + + /* Check for EOA already aligned */ + if((mis_align = (eoa % file->alignment)) > 0) { + extra = file->alignment - mis_align; + if(frag_addr) + *frag_addr = eoa - file->base_addr; /* adjust for file's base address */ + if(frag_size) + *frag_size = extra; + } /* end if */ + } /* end if */ + /* Dispatch to driver `alloc' callback or extend the end-of-address marker */ + /* For the multi/split driver: the size passed down to the alloc callback is the original size from H5FD_alloc() */ + /* For all other drivers: the size passed down to the alloc callback is the size + [possibly] alignment size */ if(file->cls->alloc) { - if((ret_value = (file->cls->alloc)(file, type, dxpl_id, size)) == HADDR_UNDEF) + ret_value = (file->cls->alloc)(file, type, dxpl_id, use_alloc_size ? size : size + extra); + if(!H5F_addr_defined(ret_value)) HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "driver allocation request failed") } /* end if */ else { - if((ret_value = H5FD_extend(file, type, TRUE, size, frag_addr, frag_size)) == HADDR_UNDEF) + ret_value = H5FD_extend(file, type, size + extra); + if(!H5F_addr_defined(ret_value)) HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, HADDR_UNDEF, "driver eoa update request failed") } /* end else */ + /* Set the [possibly aligned] address to return */ + if(!use_alloc_size) + ret_value += extra; + + /* Post-condition sanity check */ + if(!file->paged_aggr && file->alignment > 1 && orig_size >= file->threshold) + HDassert(!(ret_value % file->alignment)); + /* Convert absolute file offset to relative address */ ret_value -= file->base_addr; @@ -420,7 +443,7 @@ H5FD_try_extend(H5FD_t *file, H5FD_mem_t type, H5F_t *f, hid_t dxpl_id, /* Check if the block is exactly at the end of the file */ if(H5F_addr_eq(blk_end, eoa)) { /* Extend the object by extending the underlying file */ - if(HADDR_UNDEF == H5FD_extend(file, type, FALSE, extra_requested, NULL, NULL)) + if(HADDR_UNDEF == H5FD_extend(file, type, extra_requested)) HGOTO_ERROR(H5E_VFL, H5E_CANTEXTEND, FAIL, "driver extend request failed") /* Mark EOA info dirty in cache, so change will get encoded */ diff --git a/src/H5FDstdio.c b/src/H5FDstdio.c index 4c62bcc..13f728e 100644 --- a/src/H5FDstdio.c +++ b/src/H5FDstdio.c @@ -601,13 +601,6 @@ H5FD_stdio_alloc(H5FD_t *_file, H5FD_mem_t /*UNUSED*/ type, hid_t /*UNUSED*/ dxp /* Compute the address for the block to allocate */ addr = file->eoa; - /* Check if we need to align this block */ - if(size >= file->pub.threshold) { - /* Check for an already aligned block */ - if((addr % file->pub.alignment) != 0) - addr = ((addr / file->pub.alignment) + 1) * file->pub.alignment; - } /* end if */ - file->eoa = addr + size; return addr; @@ -135,7 +135,7 @@ HDfprintf(stderr, "%s: Creating free space manager, nclasses = %Zu\n", FUNC, ncl fspace->swmr_write = (H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) > 0; fspace->alignment = alignment; - fspace->threshold = threshold; + fspace->align_thres = threshold; /* Check if the free space tracker is supposed to be persistant */ if(fs_addr) { @@ -227,7 +227,7 @@ HDfprintf(stderr, "%s: fspace->rc = %u\n", FUNC, fspace->rc); HGOTO_ERROR(H5E_FSPACE, H5E_CANTINC, NULL, "unable to increment ref. count on free space header") fspace->alignment = alignment; - fspace->threshold = threshold; + fspace->align_thres = threshold; /* Unlock free space header */ if(H5AC_unprotect(f, dxpl_id, H5AC_FSPACE_HDR, fs_addr, fspace, H5AC__NO_FLAGS_SET) < 0) @@ -873,10 +873,7 @@ H5FS_alloc_sect(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id) HDassert(fspace); if(!H5F_addr_defined(fspace->sect_addr) && fspace->sinfo && fspace->serial_sect_count > 0) { - /* Allocate space for section info from aggregator/vfd (or temp. address space) */ - /* (The original version called H5MF_alloc(), but that may cause sect_size to change again) */ - /* (This routine is only called during file close operations, so don't allocate from temp. address space) */ - if(HADDR_UNDEF == (fspace->sect_addr = H5MF_aggr_vfd_alloc(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, fspace->sect_size))) + if(HADDR_UNDEF == (fspace->sect_addr = H5MF_alloc(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, fspace->sect_size))) HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "file allocation failed for section info") fspace->alloc_sect_size = fspace->sect_size; @@ -901,169 +898,6 @@ done: /*------------------------------------------------------------------------- - * Function: H5FS_alloc_vfd_alloc_hdr_and_section_info - * - * Purpose: This function is part of a hack to patch over a design - * flaw in the free space managers for file space allocation. - * Specifically, if a free space manager allocates space for - * its own section info, it is possible for it to - * go into an infinite loop as it: - * - * 1) computes the size of the section info - * - * 2) allocates file space for the section info - * - * 3) notices that the size of the section info - * has changed - * - * 4) deallocates the section info file space and - * returns to 1) above. - * - * Similarly, if it allocates space for its own header, it - * can go into an infinte loop as it: - * - * 1) allocates space for the header - * - * 2) notices that the free space manager is empty - * and thus should not have file space allocated - * to its header - * - * 3) frees the space allocated to the header - * - * 4) notices that the free space manager is not - * empty and thus must have file space allocated - * to it, and thus returns to 1) above. - * - * In a nutshell, the solution applied in this hack is to - * deallocate file space for the free space manager(s) that - * handle FSM header and/or section info file space allocations, - * wait until all other allocation/deallocation requests have - * been handled, and then test to see if the free space manager(s) - * in question are empty. If they are, do nothing. If they - * are not, allocate space for them at end of file bypassing the - * usual file space allocation calls, and thus avoiding the - * potential infinite loops. - * - * The purpose of this function is to allocate file space for - * the header and section info of the target free space manager - * directly from the VFD if needed. In this case the function - * also re-inserts the header and section info in the metadata - * cache with this allocation. - * - * Note that if f->shared->alignment > 1, and EOA is not a - * multiple of the alignment, it is possible that performing - * this allocation may generate a fragment of file space in - * addition to the space allocated for the section info. - * - * At present we deal with this by screaming and dying. - * Obviously, this is not acceptatable, but it should do - * for now. - * - * - * Return: Success: non-negative - * Failure: negative - * - * Programmer: John Mainzer - * 6/6/16 - * - *------------------------------------------------------------------------- - */ -herr_t -H5FS_alloc_vfd_alloc_hdr_and_section_info(H5F_t *f, hid_t dxpl_id, - H5FS_t *fspace, haddr_t *fs_addr_ptr) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) - - /* Check arguments */ - HDassert(f); - HDassert(fspace); - HDassert(fs_addr_ptr); - - /* The section info should be unlocked */ - HDassert(fspace->sinfo_lock_count == 0); - - /* No space should be allocated */ - HDassert(*fs_addr_ptr == HADDR_UNDEF); - HDassert(fspace->addr == HADDR_UNDEF); - HDassert(fspace->sect_addr == HADDR_UNDEF); - HDassert(fspace->alloc_sect_size == 0); - - /* Check if any space will be needed */ - if(fspace->serial_sect_count > 0) { - haddr_t sect_addr; /* Address of sinfo */ - - /* The section info is floating, so fspace->sinfo should be defined */ - HDassert(fspace->sinfo); - - /* Start by allocating file space for the header */ - if(HADDR_UNDEF == (fspace->addr = H5MF_vfd_alloc(f, dxpl_id, H5FD_MEM_FSPACE_HDR, - (hsize_t)H5FS_HEADER_SIZE(f), FALSE))) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space for hdr") - - /* Cache the new free space header (pinned) */ - if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_HDR, fspace->addr, fspace, H5AC__PIN_ENTRY_FLAG) < 0) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't add free space header to cache") - - - /* Now allocate file space for the section info */ - if(HADDR_UNDEF == (sect_addr = H5MF_vfd_alloc(f, dxpl_id, H5FD_MEM_FSPACE_SINFO, - fspace->sect_size, FALSE))) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space") - - /* Update fspace->alloc_sect_size and fspace->sect_addr to reflect - * the allocation - */ - fspace->alloc_sect_size = fspace->sect_size; - fspace->sect_addr = sect_addr; - - /* We have changed the sinfo address -- Mark free space header dirty */ - if(H5AC_mark_entry_dirty(fspace) < 0) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty") - - /* Insert the new section info into the metadata cache. */ - - /* Question: Do we need to worry about this insertion causing an - * eviction from the metadata cache? Think on this. If so, add a - * flag to H5AC_insert() to force it to skip the call to make space in - * cache. - * - * On reflection, no. - * - * On a regular file close, any eviction will not change the - * the contents of the free space manger(s), as all entries - * should have correct file space allocated by the time this - * function is called. - * - * In the cache image case, the selection of entries for inclusion - * in the cache image will not take place until after this call. - * (Recall that this call is made during the metadata fsm settle - * routine, which is called during the serialization routine in - * the cache image case. Entries are not selected for inclusion - * in the image until after the cache is serialized.) - * - * JRM -- 11/4/16 - */ - if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_SINFO, sect_addr, fspace->sinfo, H5AC__NO_FLAGS_SET) < 0) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't add free space sinfo to cache") - - /* Since space has been allocated for the section info and the sinfo - * has been inserted into the cache, relinquish owership (i.e. float) - * the section info. - */ - fspace->sinfo = NULL; - - /* Set the address of the free space header, on success */ - *fs_addr_ptr = fspace->addr; - } /* end if */ - -done: - FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* H5FS_alloc_vfd_alloc_hdr_and_section_info() */ - - -/*------------------------------------------------------------------------- * Function: H5FS_free() * * Purpose: Free space for free-space manager header and section info header @@ -1076,7 +910,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id) +H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id, hbool_t free_file_space) { haddr_t saved_addr; /* Previous address of item */ unsigned cache_flags; /* Flags for unprotecting cache entries */ @@ -1123,7 +957,8 @@ H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id) /* Free space for the free-space manager section info */ if(!H5F_IS_TMP_ADDR(f, saved_addr)) { - if(H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, saved_addr, saved_size) < 0) + if(free_file_space && + H5MF_xfree(f, H5FD_MEM_FSPACE_SINFO, dxpl_id, saved_addr, saved_size) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to release free space sections") } /* end if */ @@ -1165,7 +1000,8 @@ H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id) fspace->addr = HADDR_UNDEF; /* Free space for the free-space manager header */ - if(H5MF_xfree(f, H5FD_MEM_FSPACE_HDR, dxpl_id, saved_addr, (hsize_t)H5FS_HEADER_SIZE(f)) < 0) + if(free_file_space && + H5MF_xfree(f, H5FD_MEM_FSPACE_HDR, dxpl_id, saved_addr, (hsize_t)H5FS_HEADER_SIZE(f)) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTFREE, FAIL, "unable to free free space header") } /* end if */ @@ -1339,6 +1175,23 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FS_sinfo_dest() */ +herr_t +H5FS_get_sect_count(const H5FS_t *frsp, hsize_t *tot_sect_count) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check arguments. */ + HDassert(frsp); + HDassert(tot_sect_count); + + /* Report statistics for free space */ + *tot_sect_count = frsp->serial_sect_count; + + FUNC_LEAVE_NOAPI(ret_value) +} + #ifdef H5FS_DEBUG_ASSERT /*------------------------------------------------------------------------- diff --git a/src/H5FScache.c b/src/H5FScache.c index dcc8122..ddc66fd 100644 --- a/src/H5FScache.c +++ b/src/H5FScache.c @@ -37,6 +37,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5ACprivate.h" /* Metadata cache */ #include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File */ #include "H5FSpkg.h" /* File free space */ #include "H5MFprivate.h" /* File memory management */ #include "H5VMprivate.h" /* Vectors and arrays */ @@ -296,7 +297,7 @@ H5FS__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, /* # of section classes */ /* (only check if we actually have some classes) */ UINT16DECODE(image, nclasses); - if(fspace->nclasses > 0 && fspace->nclasses != nclasses) + if(fspace->nclasses > 0 && nclasses > fspace->nclasses) HGOTO_ERROR(H5E_FSPACE, H5E_CANTLOAD, NULL, "section class count mismatch") /* Shrink percent */ @@ -596,9 +597,11 @@ H5FS__cache_hdr_pre_serialize(H5F_t *f, hid_t dxpl_id, void *_thing, * real file space lest the header be written to file with * a nonsense section info address. */ - HDassert(fspace->serial_sect_count > 0); - HDassert(fspace->sect_size > 0); - HDassert(fspace->alloc_sect_size == (size_t)fspace->sect_size); + if(!H5F_POINT_OF_NO_RETURN(f)) { + HDassert(fspace->serial_sect_count > 0); + HDassert(fspace->sect_size > 0); + HDassert(fspace->alloc_sect_size == (size_t)fspace->sect_size); + } /* end if */ if(H5F_IS_TMP_ADDR(f, fspace->sect_addr)) { unsigned sect_status = 0; @@ -702,10 +705,12 @@ H5FS__cache_hdr_serialize(const H5F_t *f, void *_image, size_t len, * The following asserts are a cursory check on this. */ HDassert((! H5F_addr_defined(fspace->sect_addr)) || (! H5F_IS_TMP_ADDR(f, fspace->sect_addr))); - HDassert((! H5F_addr_defined(fspace->sect_addr)) || - ((fspace->serial_sect_count > 0) && - (fspace->sect_size > 0) && - (fspace->alloc_sect_size == (size_t)fspace->sect_size))); + + if(!H5F_POINT_OF_NO_RETURN(f)) + HDassert((! H5F_addr_defined(fspace->sect_addr)) || + ((fspace->serial_sect_count > 0) && + (fspace->sect_size > 0) && + (fspace->alloc_sect_size == (size_t)fspace->sect_size))); /* Magic number */ HDmemcpy(image, H5FS_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC); @@ -1061,7 +1066,7 @@ H5FS__cache_sinfo_deserialize(const void *_image, size_t len, void *_udata, /* Insert section in free space manager, unless requested not to */ if(!(des_flags & H5FS_DESERIALIZE_NO_ADD)) - if(H5FS_sect_add(udata->f, udata->dxpl_id, udata->fspace, new_sect, H5FS_ADD_DESERIALIZING, NULL) < 0) + if(H5FS_sect_add(udata->f, udata->dxpl_id, udata->fspace, new_sect, H5FS_ADD_DESERIALIZING, udata) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, NULL, "can't add section to free space manager") } /* end for */ } while(image < (((const uint8_t *)_image + old_sect_size) - H5FS_SIZEOF_CHKSUM)); @@ -1177,8 +1182,9 @@ H5FS__cache_sinfo_pre_serialize(H5F_t *f, hid_t dxpl_id, void *_thing, HDassert(new_len); HDassert(flags); - /* we shouldn't be called if the section info is empty */ - HDassert(fspace->serial_sect_count > 0); + /* we shouldn't be called if the section info is empty, unless we hit the point of no return. */ + if(!H5F_POINT_OF_NO_RETURN(f)) + HDassert(fspace->serial_sect_count > 0); sinfo_addr = addr; /* this will change if we relocate the section data */ diff --git a/src/H5FSpkg.h b/src/H5FSpkg.h index 4411236..ab97955 100644 --- a/src/H5FSpkg.h +++ b/src/H5FSpkg.h @@ -187,8 +187,9 @@ struct H5FS_t { /* must be either H5C__NO_FLAGS_SET (i.e r/w) */ /* or H5AC__READ_ONLY_FLAG (i.e. r/o). */ size_t max_cls_serial_size; /* Max. additional size of serialized form of section */ - hsize_t threshold; /* Threshold for alignment */ hsize_t alignment; /* Alignment */ + hsize_t align_thres; /* Threshold for alignment */ + /* Memory data structures (not stored directly) */ H5FS_section_class_t *sect_cls; /* Array of section classes for this free list */ diff --git a/src/H5FSprivate.h b/src/H5FSprivate.h index 1816a6e..74f9dcb 100644 --- a/src/H5FSprivate.h +++ b/src/H5FSprivate.h @@ -67,6 +67,12 @@ * managed sections is in flux) */ +#define H5FS_PAGE_END_NO_ADD 0x08 /* For "small" page fs: + * Don't add section to free space: + * when the section is at page end and + * when the section size is <= "small" + */ + /* Flags for deserialize callback */ #define H5FS_DESERIALIZE_NO_ADD 0x01 /* Don't add section to free space * manager after it's deserialized @@ -98,11 +104,11 @@ typedef struct H5FS_section_class_t { herr_t (*term_cls)(struct H5FS_section_class_t *); /* Routine to terminate class-specific settings */ /* Object methods */ - herr_t (*add)(H5FS_section_info_t *, unsigned *, void *); /* Routine called when section is about to be added to manager */ + herr_t (*add)(H5FS_section_info_t **, unsigned *, void *); /* Routine called when section is about to be added to manager */ herr_t (*serialize)(const struct H5FS_section_class_t *, const H5FS_section_info_t *, uint8_t *); /* Routine to serialize a "live" section into a buffer */ H5FS_section_info_t *(*deserialize)(const struct H5FS_section_class_t *, hid_t dxpl_id, const uint8_t *, haddr_t, hsize_t, unsigned *); /* Routine to deserialize a buffer into a "live" section */ htri_t (*can_merge)(const H5FS_section_info_t *, const H5FS_section_info_t *, void *); /* Routine to determine if two nodes are mergable */ - herr_t (*merge)(H5FS_section_info_t *, H5FS_section_info_t *, void *); /* Routine to merge two nodes */ + herr_t (*merge)(H5FS_section_info_t **, H5FS_section_info_t *, void *); /* Routine to merge two nodes */ htri_t (*can_shrink)(const H5FS_section_info_t *, void *); /* Routine to determine if node can shrink container */ herr_t (*shrink)(H5FS_section_info_t **, void *); /* Routine to shrink container */ herr_t (*free)(H5FS_section_info_t *); /* Routine to free node */ @@ -182,9 +188,8 @@ H5_DLL herr_t H5FS_delete(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr); H5_DLL herr_t H5FS_close(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace); H5_DLL herr_t H5FS_alloc_hdr(H5F_t *f, H5FS_t *fspace, haddr_t *fs_addr, hid_t dxpl_id); H5_DLL herr_t H5FS_alloc_sect(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id); -H5_DLL herr_t H5FS_alloc_vfd_alloc_hdr_and_section_info(H5F_t *f, hid_t dxpl_id, - H5FS_t *fspace, haddr_t *fs_addr_ptr); -H5_DLL herr_t H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id); +H5_DLL herr_t H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id, + hbool_t free_file_space); /* Free space section routines */ H5_DLL herr_t H5FS_sect_add(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, @@ -192,7 +197,7 @@ H5_DLL herr_t H5FS_sect_add(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5_DLL htri_t H5FS_sect_try_merge(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5FS_section_info_t *sect, unsigned flags, void *op_data); H5_DLL htri_t H5FS_sect_try_extend(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, - haddr_t addr, hsize_t size, hsize_t extra_requested); + haddr_t addr, hsize_t size, hsize_t extra_requested, unsigned flags, void *op_data); H5_DLL herr_t H5FS_sect_remove(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5FS_section_info_t *node); H5_DLL htri_t H5FS_sect_find(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, @@ -202,11 +207,18 @@ H5_DLL herr_t H5FS_sect_stats(const H5FS_t *fspace, hsize_t *tot_space, hsize_t *nsects); H5_DLL herr_t H5FS_sect_change_class(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5FS_section_info_t *sect, uint16_t new_class); -H5_DLL htri_t H5FS_sect_try_shrink_eoa(const H5F_t *f, hid_t dxpl_id, const H5FS_t *fspace, void *op_data); -H5_DLL herr_t H5FS_sect_query_last_sect(const H5FS_t *fspace, haddr_t *sect_addr, hsize_t *sect_size); +H5_DLL htri_t H5FS_sect_try_shrink_eoa(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, + void *op_data); /* Statistics routine */ H5_DLL herr_t H5FS_stat_info(const H5F_t *f, const H5FS_t *frsp, H5FS_stat_t *stats); +H5_DLL herr_t H5FS_get_sect_count(const H5FS_t *frsp, hsize_t *tot_sect_count); + +/* free space manager settling routines */ +H5_DLL herr_t H5FS_vfd_alloc_hdr_and_section_info_if_needed(H5F_t *f, + hid_t dxpl_id, + H5FS_t *fspace, + haddr_t *fs_addr_ptr); /* Debugging routines for dumping file structures */ H5_DLL herr_t H5FS_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, diff --git a/src/H5FSsection.c b/src/H5FSsection.c index efe0031..49ff003 100644 --- a/src/H5FSsection.c +++ b/src/H5FSsection.c @@ -25,6 +25,8 @@ /* Module Setup */ /****************/ +#define H5F_FRIEND /*suppress error about including H5Fpkg */ + #include "H5FSmodule.h" /* This source code file is part of the H5FS module */ @@ -33,6 +35,7 @@ /***********/ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* File access */ #include "H5FSpkg.h" /* File free space */ #include "H5MFprivate.h" /* File memory management */ #include "H5VMprivate.h" /* Vectors and arrays */ @@ -1210,11 +1213,13 @@ H5FS_sect_merge(H5FS_t *fspace, H5FS_section_info_t **sect, void *op_data) HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't remove section from internal data structures") /* Merge the two sections together */ - if((*tmp_sect_cls->merge)(tmp_sect, *sect, op_data) < 0) + if((*tmp_sect_cls->merge)(&tmp_sect, *sect, op_data) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't merge two sections") /* Retarget section pointer to 'less than' node that was merged into */ *sect = tmp_sect; + if(*sect == NULL) + HGOTO_DONE(ret_value); /* Indicate successful merge occurred */ modified = TRUE; @@ -1254,9 +1259,13 @@ H5FS_sect_merge(H5FS_t *fspace, H5FS_section_info_t **sect, void *op_data) HGOTO_ERROR(H5E_FSPACE, H5E_CANTRELEASE, FAIL, "can't remove section from internal data structures") /* Merge the two sections together */ - if((*sect_cls->merge)(*sect, tmp_sect, op_data) < 0) + if((*sect_cls->merge)(sect, tmp_sect, op_data) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't merge two sections") + /* It's possible that the merge caused the section to be deleted (particularly in the paged allocation case) */ + if(*sect == NULL) + HGOTO_DONE(ret_value); + /* Indicate successful merge occurred */ modified = TRUE; } /* end if */ @@ -1383,7 +1392,7 @@ HDfprintf(stderr, "%s: *sect = {%a, %Hu, %u, %s}\n", FUNC, sect->addr, sect->siz /* Call "add" section class callback, if there is one */ cls = &fspace->sect_cls[sect->type]; if(cls->add) { - if((*cls->add)(sect, &flags, op_data) < 0) + if((*cls->add)(§, &flags, op_data) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "'add' section class callback failed") } /* end if */ @@ -1411,7 +1420,7 @@ HDfprintf(stderr, "%s: fspace->tot_space = %Hu\n", FUNC, fspace->tot_space); #endif /* H5FS_SINFO_DEBUG */ /* Mark free space sections as changed */ /* (if adding sections while deserializing sections, don't set the flag) */ - if(!(flags & H5FS_ADD_DESERIALIZING)) + if(!(flags & (H5FS_ADD_DESERIALIZING | H5FS_PAGE_END_NO_ADD))) sinfo_modified = TRUE; done: @@ -1444,7 +1453,7 @@ HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value); */ htri_t H5FS_sect_try_extend(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, haddr_t addr, - hsize_t size, hsize_t extra_requested) + hsize_t size, hsize_t extra_requested, unsigned flags, void *op_data) { hbool_t sinfo_valid = FALSE; /* Whether the section info is valid */ hbool_t sinfo_modified = FALSE; /* Whether the section info was modified */ @@ -1528,10 +1537,16 @@ if(_section_) /* Adjust section by amount requested */ sect->addr += extra_requested; sect->size -= extra_requested; - - /* Re-add adjusted section to free sections data structures */ - if(H5FS_sect_link(fspace, sect, 0) < 0) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't insert free space section into skip list") + if(cls->add) + if((*cls->add)(§, &flags, op_data) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "'add' section class callback failed") + + /* Re-adding the section could cause it to disappear (particularly when paging) */ + if(sect) { + /* Re-add adjusted section to free sections data structures */ + if(H5FS_sect_link(fspace, sect, 0) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTINSERT, FAIL, "can't insert free space section into skip list") + } /* end if */ } /* end if */ else { /* Sanity check */ @@ -1667,7 +1682,7 @@ HDfprintf(stderr, "%s: fspace->sinfo->nbins = %u\n", FUNC, fspace->sinfo->nbins) HDfprintf(stderr, "%s: bin = %u\n", FUNC, bin); #endif /* QAK */ alignment = fspace->alignment; - if(!((alignment > 1) && (request >= fspace->threshold))) + if(!((alignment > 1) && (request >= fspace->align_thres))) alignment = 0; /* no alignment */ do { @@ -2349,7 +2364,7 @@ HDfprintf(stderr, "%s: sect->size = %Hu, sect->addr = %a, sect->type = %u\n", "H *------------------------------------------------------------------------- */ htri_t -H5FS_sect_try_shrink_eoa(const H5F_t *f, hid_t dxpl_id, const H5FS_t *fspace, void *op_data) +H5FS_sect_try_shrink_eoa(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, void *op_data) { hbool_t sinfo_valid = FALSE; /* Whether the section info is valid */ hbool_t section_removed = FALSE; /* Whether a section was removed */ @@ -2406,42 +2421,264 @@ done: /*------------------------------------------------------------------------- - * Function: H5FS_sect_query_last_sect - * - * Purpose: Retrieve info about the last section on the merge list - * - * Return: Success: non-negative - * Failure: negative + * Function: H5FS_vfd_alloc_hdr_and_section_info_if_needed + * + * Purpose: This function is part of a hack to patch over a design + * flaw in the free space managers for file space allocation. + * Specifically, if a free space manager allocates space for + * its own section info, it is possible for it to + * go into an infinite loop as it: + * + * 1) computes the size of the section info + * + * 2) allocates file space for the section info + * + * 3) notices that the size of the section info + * has changed + * + * 4) deallocates the section info file space and + * returns to 1) above. + * + * Similarly, if it allocates space for its own header, it + * can go into an infinte loop as it: + * + * 1) allocates space for the header + * + * 2) notices that the free space manager is empty + * and thus should not have file space allocated + * to its header + * + * 3) frees the space allocated to the header + * + * 4) notices that the free space manager is not + * empty and thus must have file space allocated + * to it, and thus returns to 1) above. + * + * In a nutshell, the solution applied in this hack is to + * deallocate file space for the free space manager(s) that + * handle FSM header and/or section info file space allocations, + * wait until all other allocation/deallocation requests have + * been handled, and then test to see if the free space manager(s) + * in question are empty. If they are, do nothing. If they + * are not, allocate space for them at end of file bypassing the + * usual file space allocation calls, and thus avoiding the + * potential infinite loops. + * + * The purpose of this function is to allocate file space for + * the header and section info of the target free space manager + * directly from the VFD if needed. In this case the function + * also re-inserts the header and section info in the metadata + * cache with this allocation. + * + * When paged allocation is not enabled, allocation of space + * for the free space manager header and section info is + * straight forward -- we simply allocate the space directly + * from file driver. + * + * Note that if f->shared->alignment > 1, and EOA is not a + * multiple of the alignment, it is possible that performing + * these allocations may generate a fragment of file space in + * addition to the space allocated for the section info. This + * excess space is dropped on the floor. As shall be seen, + * it will usually be reclaimed later. + * + * When page allocation is enabled, things are more difficult, + * as there is the possibility that page buffering will be + * enabled when the free space managers are read. To allow + * for this, we must ensure that space allocated for the + * free space manager header and section info is either larger + * than a page, or resides completely withing a page. + * + * Do this by allocating space for the free space header and + * section info starting at page boundaries, and extending + * allocation to the next page boundary. This of course wastes + * space, but see below. + * + * On the first free space allocation / deallocation after the + * next file open, we will read the self referential free space + * managers, float them and reduce the EOA to its value prior + * to allocation of file space for the self referential free + * space managers on the preceeding file close. This EOA value + * is stored in the free space manager superblock extension + * message. + * + * Return: Success: non-negative + * Failure: negative * - * Programmer: Vailin Choi + * Programmer: John Mainzer + * 6/6/16 * *------------------------------------------------------------------------- */ herr_t -H5FS_sect_query_last_sect(const H5FS_t *fspace, haddr_t *sect_addr, hsize_t *sect_size) +H5FS_vfd_alloc_hdr_and_section_info_if_needed(H5F_t *f, hid_t dxpl_id, + H5FS_t *fspace, haddr_t *fs_addr_ptr) { - FUNC_ENTER_NOAPI_NOINIT_NOERR + hsize_t hdr_alloc_size; + hsize_t sinfo_alloc_size; + haddr_t sect_addr = HADDR_UNDEF; /* address of sinfo */ + haddr_t eoa_frag_addr = HADDR_UNDEF; /* Address of fragment at EOA */ + hsize_t eoa_frag_size = 0; /* Size of fragment at EOA */ + haddr_t eoa = HADDR_UNDEF; /* Initial EOA for the file */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT /* Check arguments. */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); HDassert(fspace); + HDassert(fs_addr_ptr); - if(fspace->sinfo && fspace->sinfo->merge_list) { - H5SL_node_t *last_node; /* Last node in merge list */ + /* the section info should be unlocked */ + HDassert(fspace->sinfo_lock_count == 0); - /* Check for last node in the merge list */ - if(NULL != (last_node = H5SL_last(fspace->sinfo->merge_list))) { - H5FS_section_info_t *tmp_sect; /* Temporary free space section */ + /* no space should be allocated */ + HDassert(*fs_addr_ptr == HADDR_UNDEF); + HDassert(fspace->addr == HADDR_UNDEF); + HDassert(fspace->sect_addr == HADDR_UNDEF); + HDassert(fspace->alloc_sect_size == 0); - /* Get the pointer to the last section, from the last node */ - tmp_sect = (H5FS_section_info_t *)H5SL_item(last_node); - HDassert(tmp_sect); - if(sect_addr) - *sect_addr = tmp_sect->addr; - if(sect_size) - *sect_size = tmp_sect->size; - } /* end if */ + /* persistant free space managers must be enabled */ + HDassert(f->shared->fs_persist); + + /* At present, all free space strategies enable the free space managers. + * This will probably change -- at which point this assertion should + * be revisited. + */ + /* Updated: Only the following two strategies enable the free-space managers */ + HDassert((f->shared->fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR) || + (f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE)); + + if(fspace->serial_sect_count > 0) { + /* the section info is floating, so space->sinfo should be defined */ + HDassert(fspace->sinfo); + + /* start by allocating file space for the header */ + + /* Get the EOA for the file -- need for sanity check below */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_FSPACE_HDR))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "Unable to get eoa") + + /* check for overlap into temporary allocation space */ + if(H5F_IS_TMP_ADDR(f, (eoa + fspace->sect_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, FAIL, "hdr file space alloc will overlap into 'temporary' file space") + + hdr_alloc_size = H5FS_HEADER_SIZE(f); + + /* if page allocation is enabled, extend the hdr_alloc_size to the + * next page boundary. + */ + if(H5F_PAGED_AGGR(f)) { + HDassert(0 == (eoa % f->shared->fs_page_size)); + + hdr_alloc_size = ((hdr_alloc_size / f->shared->fs_page_size) + 1) * f->shared->fs_page_size; + + HDassert(hdr_alloc_size >= H5FS_HEADER_SIZE(f)); + HDassert((hdr_alloc_size % f->shared->fs_page_size) == 0); + } /* end if */ + + /* allocate space for the hdr */ + if(HADDR_UNDEF == (fspace->addr = H5FD_alloc(f->shared->lf, dxpl_id, + H5FD_MEM_FSPACE_HDR, f, + hdr_alloc_size, + &eoa_frag_addr, + &eoa_frag_size))) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space for hdr") + + /* if the file alignement is 1, there should be no + * eoa fragment. Otherwise, drop any fragment on the floor. + */ + HDassert((eoa_frag_size == 0) || (f->shared->alignment != 1)); + + /* Cache the new free space header (pinned) */ + if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_HDR, fspace->addr, fspace, H5AC__PIN_ENTRY_FLAG) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space header to cache") + + *fs_addr_ptr = fspace->addr; + + /* now allocate file space for the section info */ + + /* Get the EOA for the file -- need for sanity check below */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_FSPACE_SINFO))) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTGET, FAIL, "Unable to get eoa") + + /* check for overlap into temporary allocation space */ + if(H5F_IS_TMP_ADDR(f, (eoa + fspace->sect_size))) + HGOTO_ERROR(H5E_FSPACE, H5E_BADRANGE, FAIL, "sinfo file space alloc will overlap into 'temporary' file space") + + sinfo_alloc_size = fspace->sect_size; + + /* if paged allocation is enabled, extend the sinfo_alloc_size to the + * next page boundary. + */ + if(H5F_PAGED_AGGR(f)) { + HDassert(0 == (eoa % f->shared->fs_page_size)); + + sinfo_alloc_size = ((sinfo_alloc_size / f->shared->fs_page_size) + 1) * f->shared->fs_page_size; + + HDassert(sinfo_alloc_size >= fspace->sect_size); + HDassert((sinfo_alloc_size % f->shared->fs_page_size) == 0); + } /* end if */ + + /* allocate space for the section info */ + if(HADDR_UNDEF == (sect_addr = H5FD_alloc(f->shared->lf, dxpl_id, + H5FD_MEM_FSPACE_SINFO, f, + sinfo_alloc_size, + &eoa_frag_addr, + &eoa_frag_size))) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTALLOC, FAIL, "can't allocate file space") + + /* if the file alignement is 1, there should be no + * eoa fragment. Otherwise, drop the fragment on the floor. + */ + HDassert((eoa_frag_size == 0) || (f->shared->alignment != 1)); + + /* update fspace->alloc_sect_size and fspace->sect_addr to reflect + * the allocation + */ + fspace->alloc_sect_size = fspace->sect_size; + fspace->sect_addr = sect_addr; + + /* insert the new section info into the metadata cache. */ + + /* Question: Do we need to worry about this insertion causing an + * eviction from the metadata cache? Think on this. If so, add a + * flag to H5AC_insert() to force it to skip the call to make space in + * cache. + * + * On reflection, no. + * + * On a regular file close, any eviction will not change the + * the contents of the free space manger(s), as all entries + * should have correct file space allocated by the time this + * function is called. + * + * In the cache image case, the selection of entries for inclusion + * in the cache image will not take place until after this call. + * (Recall that this call is made during the metadata fsm settle + * routine, which is called during the serialization routine in + * the cache image case. Entries are not selected for inclusion + * in the image until after the cache is serialized.) + * + * JRM -- 11/4/16 + */ + if(H5AC_insert_entry(f, dxpl_id, H5AC_FSPACE_SINFO, sect_addr, fspace->sinfo, H5AC__NO_FLAGS_SET) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space sinfo to cache") + + /* We have changed the sinfo address -- Mark free space header dirty */ + if(H5AC_mark_entry_dirty(fspace) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTMARKDIRTY, FAIL, "unable to mark free space header as dirty") + + /* since space has been allocated for the section info and the sinfo + * has been inserted into the cache, relinquish owership (i.e. float) + * the section info. + */ + fspace->sinfo = NULL; } /* end if */ - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5FS_sect_query_last_sect() */ +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FS_vfd_alloc_hdr_and_section_info_if_needed() */ diff --git a/src/H5Faccum.c b/src/H5Faccum.c index 48f9bdd..7f3bb39 100644 --- a/src/H5Faccum.c +++ b/src/H5Faccum.c @@ -112,18 +112,25 @@ H5FL_BLK_DEFINE_STATIC(meta_accum); *------------------------------------------------------------------------- */ herr_t -H5F__accum_read(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t addr, +H5F__accum_read(const H5F_io_info2_t *fio_info, H5FD_mem_t map_type, haddr_t addr, size_t size, void *buf/*out*/) { + H5FD_io_info_t fdio_info; /* File driver I/O info */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE HDassert(fio_info); HDassert(fio_info->f); - HDassert(fio_info->dxpl); + HDassert(fio_info->meta_dxpl); + HDassert(fio_info->raw_dxpl); HDassert(buf); + /* Translate to file driver I/O info object */ + fdio_info.file = fio_info->f->shared->lf; + fdio_info.meta_dxpl = fio_info->meta_dxpl; + fdio_info.raw_dxpl = fio_info->raw_dxpl; + /* Check if this information is in the metadata accumulator */ if((fio_info->f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && map_type != H5FD_MEM_DRAW) { H5F_meta_accum_t *accum; /* Alias for file's metadata accumulator */ @@ -178,7 +185,7 @@ H5F__accum_read(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t addr accum->dirty_off += amount_before; /* Dispatch to driver */ - if(H5FD_read(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, amount_before, accum->buf) < 0) + if(H5FD_read(&fdio_info, map_type, addr, amount_before, accum->buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") } /* end if */ else @@ -192,7 +199,7 @@ H5F__accum_read(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t addr H5_CHECKED_ASSIGN(amount_after, size_t, ((addr + size) - (accum->loc + accum->size)), hsize_t); /* Dispatch to driver */ - if(H5FD_read(fio_info->f->shared->lf, fio_info->dxpl, map_type, (accum->loc + accum->size), amount_after, (accum->buf + accum->size + amount_before)) < 0) + if(H5FD_read(&fdio_info, map_type, (accum->loc + accum->size), amount_after, (accum->buf + accum->size + amount_before)) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") } /* end if */ @@ -206,13 +213,13 @@ H5F__accum_read(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t addr /* Current read doesn't overlap with metadata accumulator, read it from file */ else { /* Dispatch to driver */ - if(H5FD_read(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, size, buf) < 0) + if(H5FD_read(&fdio_info, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") } /* end else */ } /* end if */ else { /* Read the data */ - if(H5FD_read(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, size, buf) < 0) + if(H5FD_read(&fdio_info, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") /* Check for overlap w/dirty accumulator */ @@ -255,7 +262,7 @@ H5F__accum_read(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t addr } /* end if */ else { /* Read the data */ - if(H5FD_read(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, size, buf) < 0) + if(H5FD_read(&fdio_info, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") } /* end else */ @@ -278,7 +285,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5F__accum_adjust(H5F_meta_accum_t *accum, const H5F_io_info_t *fio_info, +H5F__accum_adjust(H5F_meta_accum_t *accum, const H5FD_io_info_t *fdio_info, H5F_accum_adjust_t adjust, size_t size) { herr_t ret_value = SUCCEED; /* Return value */ @@ -286,7 +293,7 @@ H5F__accum_adjust(H5F_meta_accum_t *accum, const H5F_io_info_t *fio_info, FUNC_ENTER_STATIC HDassert(accum); - HDassert(fio_info); + HDassert(fdio_info); HDassert(H5F_ACCUM_APPEND == adjust || H5F_ACCUM_PREPEND == adjust); HDassert(size > 0); HDassert(size <= H5F_ACCUM_MAX_SIZE); @@ -344,7 +351,7 @@ H5F__accum_adjust(H5F_meta_accum_t *accum, const H5F_io_info_t *fio_info, /* Check if the dirty region overlaps the region to eliminate from the accumulator */ if((accum->size - shrink_size) < (accum->dirty_off + accum->dirty_len)) { /* Write out the dirty region from the metadata accumulator, with dispatch to driver */ - if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, (accum->loc + accum->dirty_off), accum->dirty_len, (accum->buf + accum->dirty_off)) < 0) + if(H5FD_write(fdio_info, H5FD_MEM_DEFAULT, (accum->loc + accum->dirty_off), accum->dirty_len, (accum->buf + accum->dirty_off)) < 0) HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "file write failed") /* Reset accumulator dirty flag */ @@ -355,7 +362,7 @@ H5F__accum_adjust(H5F_meta_accum_t *accum, const H5F_io_info_t *fio_info, /* Check if the dirty region overlaps the region to eliminate from the accumulator */ if(shrink_size > accum->dirty_off) { /* Write out the dirty region from the metadata accumulator, with dispatch to driver */ - if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, (accum->loc + accum->dirty_off), accum->dirty_len, (accum->buf + accum->dirty_off)) < 0) + if(H5FD_write(fdio_info, H5FD_MEM_DEFAULT, (accum->loc + accum->dirty_off), accum->dirty_len, (accum->buf + accum->dirty_off)) < 0) HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "file write failed") /* Reset accumulator dirty flag */ @@ -417,9 +424,10 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t addr, +H5F__accum_write(const H5F_io_info2_t *fio_info, H5FD_mem_t map_type, haddr_t addr, size_t size, const void *buf) { + H5FD_io_info_t fdio_info; /* File driver I/O info */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -427,9 +435,15 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add HDassert(fio_info); HDassert(fio_info->f); HDassert(H5F_INTENT(fio_info->f) & H5F_ACC_RDWR); - HDassert(fio_info->dxpl); + HDassert(fio_info->meta_dxpl); + HDassert(fio_info->raw_dxpl); HDassert(buf); + /* Translate to file driver I/O info object */ + fdio_info.file = fio_info->f->shared->lf; + fdio_info.meta_dxpl = fio_info->meta_dxpl; + fdio_info.raw_dxpl = fio_info->raw_dxpl; + /* Check for accumulating metadata */ if((fio_info->f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && map_type != H5FD_MEM_DRAW) { H5F_meta_accum_t *accum; /* Alias for file's metadata accumulator */ @@ -446,7 +460,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add /* Check if the new metadata adjoins the beginning of the current accumulator */ if((addr + size) == accum->loc) { /* Check if we need to adjust accumulator size */ - if(H5F__accum_adjust(accum, fio_info, H5F_ACCUM_PREPEND, size) < 0) + if(H5F__accum_adjust(accum, &fdio_info, H5F_ACCUM_PREPEND, size) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator") /* Move the existing metadata to the proper location */ @@ -471,7 +485,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add /* Check if the new metadata adjoins the end of the current accumulator */ else if(addr == (accum->loc + accum->size)) { /* Check if we need to adjust accumulator size */ - if(H5F__accum_adjust(accum, fio_info, H5F_ACCUM_APPEND, size) < 0) + if(H5F__accum_adjust(accum, &fdio_info, H5F_ACCUM_APPEND, size) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator") /* Copy the new metadata to the end */ @@ -531,7 +545,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add H5_CHECKED_ASSIGN(add_size, size_t, (accum->loc - addr), hsize_t); /* Check if we need to adjust accumulator size */ - if(H5F__accum_adjust(accum, fio_info, H5F_ACCUM_PREPEND, add_size) < 0) + if(H5F__accum_adjust(accum, &fdio_info, H5F_ACCUM_PREPEND, add_size) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator") /* Calculate the proper offset of the existing metadata */ @@ -571,7 +585,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add H5_CHECKED_ASSIGN(add_size, size_t, (addr + size) - (accum->loc + accum->size), hsize_t); /* Check if we need to adjust accumulator size */ - if(H5F__accum_adjust(accum, fio_info, H5F_ACCUM_APPEND, add_size) < 0) + if(H5F__accum_adjust(accum, &fdio_info, H5F_ACCUM_APPEND, add_size) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator") /* Compute offset of dirty region (after adjusting accumulator) */ @@ -637,7 +651,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add else { /* Write out the existing metadata accumulator, with dispatch to driver */ if(accum->dirty) { - if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, accum->loc + accum->dirty_off, accum->dirty_len, accum->buf + accum->dirty_off) < 0) + if(H5FD_write(&fdio_info, H5FD_MEM_DEFAULT, accum->loc + accum->dirty_off, accum->dirty_len, accum->buf + accum->dirty_off) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") /* Reset accumulator dirty flag */ @@ -733,7 +747,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add HGOTO_ERROR(H5E_IO, H5E_CANTRESET, FAIL, "can't reset accumulator") /* Write the data */ - if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, size, buf) < 0) + if(H5FD_write(&fdio_info, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") /* Check for overlap w/accumulator */ @@ -818,7 +832,7 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add } /* end if */ else { /* Write the data */ - if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, size, buf) < 0) + if(H5FD_write(&fdio_info, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") } /* end else */ @@ -842,10 +856,11 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F__accum_free(const H5F_io_info_t *fio_info, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr, +H5F__accum_free(const H5F_io_info2_t *fio_info, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr, hsize_t size) { H5F_meta_accum_t *accum; /* Alias for file's metadata accumulator */ + H5FD_io_info_t fdio_info; /* File driver I/O info */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -853,11 +868,17 @@ H5F__accum_free(const H5F_io_info_t *fio_info, H5FD_mem_t H5_ATTR_UNUSED type, h /* check arguments */ HDassert(fio_info); HDassert(fio_info->f); - HDassert(fio_info->dxpl); + HDassert(fio_info->meta_dxpl); + HDassert(fio_info->raw_dxpl); /* Set up alias for file's metadata accumulator info */ accum = &fio_info->f->shared->accum; + /* Translate to file driver I/O info object */ + fdio_info.file = fio_info->f->shared->lf; + fdio_info.meta_dxpl = fio_info->meta_dxpl; + fdio_info.raw_dxpl = fio_info->raw_dxpl; + /* Adjust the metadata accumulator to remove the freed block, if it overlaps */ if((fio_info->f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && H5F_addr_overlap(addr, size, accum->loc, accum->size)) { @@ -930,7 +951,7 @@ H5F__accum_free(const H5F_io_info_t *fio_info, H5FD_mem_t H5_ATTR_UNUSED type, h /* Check if block to free is entirely before dirty region */ if(H5F_addr_le(tail_addr, dirty_start)) { /* Write out the entire dirty region of the accumulator */ - if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, dirty_start, accum->dirty_len, accum->buf + accum->dirty_off) < 0) + if(H5FD_write(&fdio_info, H5FD_MEM_DEFAULT, dirty_start, accum->dirty_len, accum->buf + accum->dirty_off) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") } /* end if */ /* Block to free overlaps with some/all of dirty region */ @@ -945,7 +966,7 @@ H5F__accum_free(const H5F_io_info_t *fio_info, H5FD_mem_t H5_ATTR_UNUSED type, h HDassert(write_size > 0); /* Write out the unfreed dirty region of the accumulator */ - if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, dirty_start + dirty_delta, write_size, accum->buf + accum->dirty_off + dirty_delta) < 0) + if(H5FD_write(&fdio_info, H5FD_MEM_DEFAULT, dirty_start + dirty_delta, write_size, accum->buf + accum->dirty_off + dirty_delta) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") } /* end if */ @@ -965,7 +986,7 @@ H5F__accum_free(const H5F_io_info_t *fio_info, H5FD_mem_t H5_ATTR_UNUSED type, h HDassert(write_size > 0); /* Write out the unfreed end of the dirty region of the accumulator */ - if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, dirty_start + dirty_delta, write_size, accum->buf + accum->dirty_off + dirty_delta) < 0) + if(H5FD_write(&fdio_info, H5FD_MEM_DEFAULT, dirty_start + dirty_delta, write_size, accum->buf + accum->dirty_off + dirty_delta) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") } /* end if */ @@ -1006,7 +1027,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F__accum_flush(const H5F_io_info_t *fio_info) +H5F__accum_flush(const H5F_io_info2_t *fio_info) { herr_t ret_value = SUCCEED; /* Return value */ @@ -1014,12 +1035,20 @@ H5F__accum_flush(const H5F_io_info_t *fio_info) HDassert(fio_info); HDassert(fio_info->f); - HDassert(fio_info->dxpl); + HDassert(fio_info->meta_dxpl); + HDassert(fio_info->raw_dxpl); /* Check if we need to flush out the metadata accumulator */ if((fio_info->f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && fio_info->f->shared->accum.dirty) { + H5FD_io_info_t fdio_info; /* File driver I/O info */ + + /* Translate to file driver I/O info object */ + fdio_info.file = fio_info->f->shared->lf; + fdio_info.meta_dxpl = fio_info->meta_dxpl; + fdio_info.raw_dxpl = fio_info->raw_dxpl; + /* Flush the metadata contents */ - if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, H5FD_MEM_DEFAULT, fio_info->f->shared->accum.loc + fio_info->f->shared->accum.dirty_off, fio_info->f->shared->accum.dirty_len, fio_info->f->shared->accum.buf + fio_info->f->shared->accum.dirty_off) < 0) + if(H5FD_write(&fdio_info, H5FD_MEM_DEFAULT, fio_info->f->shared->accum.loc + fio_info->f->shared->accum.dirty_off, fio_info->f->shared->accum.dirty_len, fio_info->f->shared->accum.buf + fio_info->f->shared->accum.dirty_off) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") /* Reset the dirty flag */ @@ -1045,7 +1074,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F__accum_reset(const H5F_io_info_t *fio_info, hbool_t flush) +H5F__accum_reset(const H5F_io_info2_t *fio_info, hbool_t flush) { herr_t ret_value = SUCCEED; /* Return value */ @@ -1053,7 +1082,6 @@ H5F__accum_reset(const H5F_io_info_t *fio_info, hbool_t flush) HDassert(fio_info); HDassert(fio_info->f); - HDassert(fio_info->dxpl); /* Flush any dirty data in accumulator, if requested */ if(flush) diff --git a/src/H5Fint.c b/src/H5Fint.c index d122357..794be50 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -76,8 +76,8 @@ typedef struct H5F_olist_t { static int H5F_get_objects_cb(void *obj_ptr, hid_t obj_id, void *key); static herr_t H5F_build_actual_name(const H5F_t *f, const H5P_genplist_t *fapl, const char *name, char ** /*out*/ actual_name);/* Declare a free list to manage the H5F_t struct */ -static herr_t H5F__flush_phase1(H5F_t *f, hid_t dxpl_id); -static herr_t H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing); +static herr_t H5F__flush_phase1(H5F_t *f, hid_t meta_dxpl_id); +static herr_t H5F__flush_phase2(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t closing); /*********************/ @@ -182,6 +182,14 @@ H5F_get_access_plist(H5F_t *f, hbool_t app_ref) efc_size = H5F_efc_max_nfiles(f->shared->efc); if(H5P_set(new_plist, H5F_ACS_EFC_SIZE_NAME, &efc_size) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set elink file cache size") + if(f->shared->page_buf != NULL) { + if(H5P_set(new_plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &(f->shared->page_buf->max_size)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set page buffer size") + if(H5P_set(new_plist, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, &(f->shared->page_buf->min_meta_perc)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set minimum metadata fraction of page buffer") + if(H5P_set(new_plist, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, &(f->shared->page_buf->min_raw_perc)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set minimum raw data fraction of page buffer") + } /* end if */ #ifdef H5_HAVE_PARALLEL if(H5P_set(new_plist, H5_COLL_MD_READ_FLAG_NAME, &(f->coll_md_read)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set collective metadata read flag") @@ -495,7 +503,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5F_is_hdf5 + * Function: H5F__is_hdf5 * * Purpose: Check the file signature to detect an HDF5 file. * @@ -509,17 +517,14 @@ done: * * Programmer: Unknown * - * Modifications: - * Robb Matzke, 1999-08-02 - * Rewritten to use the virtual file layer. *------------------------------------------------------------------------- */ htri_t -H5F_is_hdf5(const char *name, hid_t dxpl_id) +H5F__is_hdf5(const char *name, hid_t meta_dxpl_id, hid_t raw_dxpl_id) { H5FD_t *file = NULL; /* Low-level file struct */ + H5FD_io_info_t fdio_info; /* File driver I/O info */ haddr_t sig_addr; /* Addess of hdf5 file signature */ - H5P_genplist_t *xfer_plist= NULL; /* Dataset transfer property list object */ htri_t ret_value = FAIL; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -528,12 +533,15 @@ H5F_is_hdf5(const char *name, hid_t dxpl_id) if(NULL == (file = H5FD_open(name, H5F_ACC_RDONLY, H5P_FILE_ACCESS_DEFAULT, HADDR_UNDEF))) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to open file") - /* Get the property list object */ - if(NULL == (xfer_plist = (H5P_genplist_t *)H5I_object(dxpl_id))) + /* Set up the file driver info */ + fdio_info.file = file; + if(NULL == (fdio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id))) + HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get new property list object") + if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(raw_dxpl_id))) HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get new property list object") /* The file is an hdf5 file if the hdf5 file signature can be found */ - if(H5FD_locate_signature(file, xfer_plist, &sig_addr) < 0) + if(H5FD_locate_signature(&fdio_info, &sig_addr) < 0) HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable to locate file signature") ret_value = (HADDR_UNDEF != sig_addr); @@ -544,7 +552,7 @@ done: HDONE_ERROR(H5E_IO, H5E_CANTCLOSEFILE, FAIL, "unable to close file") FUNC_LEAVE_NOAPI(ret_value) -} /* end H5F_is_hdf5() */ +} /* end H5F__is_hdf5() */ /*------------------------------------------------------------------------- @@ -597,11 +605,26 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t f->shared->flags = flags; f->shared->sohm_addr = HADDR_UNDEF; f->shared->sohm_vers = HDF5_SHAREDHEADER_VERSION; - for(u = 0; u < NELMTS(f->shared->fs_addr); u++) - f->shared->fs_addr[u] = HADDR_UNDEF; f->shared->accum.loc = HADDR_UNDEF; f->shared->lf = lf; + /* Initialization for handling file space */ + for(u = 0; u < NELMTS(f->shared->fs_addr); u++) { + f->shared->fs_state[u] = H5F_FS_STATE_CLOSED; + f->shared->fs_addr[u] = HADDR_UNDEF; + f->shared->fs_man[u] = NULL; + } /* end for */ + f->shared->first_alloc_dealloc = FALSE; + f->shared->eoa_pre_fsm_fsalloc = HADDR_UNDEF; + f->shared->eoa_post_fsm_fsalloc = HADDR_UNDEF; + f->shared->eoa_post_mdci_fsalloc = HADDR_UNDEF; + + /* Initialization for handling file space (for paged aggregation) */ + f->shared->pgend_meta_thres = H5F_FILE_SPACE_PGEND_META_THRES; + + /* intialize point of no return */ + f->shared->point_of_no_return = FALSE; + /* * Copy the file creation and file access property lists into the * new file handle. We do this early because some values might need @@ -621,8 +644,19 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t HDassert(f->shared->sohm_nindexes < 255); if(H5P_get(plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &f->shared->fs_strategy) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get file space strategy") + if(H5P_get(plist, H5F_CRT_FREE_SPACE_PERSIST_NAME, &f->shared->fs_persist) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get file space persisting status") if(H5P_get(plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &f->shared->fs_threshold) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get free-space section threshold") + if(H5P_get(plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, &f->shared->fs_page_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get file space page size") + HDassert(f->shared->fs_page_size >= H5F_FILE_SPACE_PAGE_SIZE_MIN); + + /* Temporary for multi/split drivers: fail file creation + when persisting free-space or using paged aggregation strategy */ + if(H5F_HAS_FEATURE(f, H5FD_FEAT_PAGED_AGGR)) + if(f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE || f->shared->fs_persist) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't open with this strategy or persistent fs") /* Get the FAPL values to cache */ if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) @@ -796,7 +830,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5F_dest + * Function: H5F__dest * * Purpose: Destroys a file structure. This function flushes the cache * but doesn't do any other cleanup other than freeing memory @@ -812,11 +846,11 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) +H5F__dest(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t flush) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_PACKAGE /* Sanity check */ HDassert(f); @@ -824,14 +858,14 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) if(1 == f->shared->nrefs) { int actype; /* metadata cache type (enum value) */ - H5F_io_info_t fio_info; /* I/O info for operation */ + H5F_io_info2_t fio_info; /* I/O info for operation */ /* Flush at this point since the file will be closed (phase 1). * Only try to flush the file if it was opened with write access, and if * the caller requested a flush. */ if((H5F_ACC_RDWR & H5F_INTENT(f)) && flush) - if(H5F__flush_phase1(f, dxpl_id) < 0) + if(H5F__flush_phase1(f, meta_dxpl_id) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush cached data (phase 1)") @@ -839,7 +873,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) * This allows the cache to set up for creating a metadata cache * image if this has been requested. */ - if(H5AC_prep_for_file_close(f, dxpl_id) < 0) + if(H5AC_prep_for_file_close(f, meta_dxpl_id) < 0) /* Push error, but keep going */ HDONE_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "metadata cache prep for close failed") @@ -848,7 +882,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) * the caller requested a flush. */ if((H5F_ACC_RDWR & H5F_INTENT(f)) && flush) - if(H5F__flush_phase2(f, dxpl_id, TRUE) < 0) + if(H5F__flush_phase2(f, meta_dxpl_id, raw_dxpl_id, TRUE) < 0) /* Push error, but keep going */ HDONE_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush cached data (phase 2)") @@ -895,7 +929,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) * -- JRM */ if(H5F_ACC_RDWR & H5F_INTENT(f)) { - if(H5MF_close(f, dxpl_id) < 0) + if(H5MF_close(f, meta_dxpl_id) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't release file free space info") @@ -913,7 +947,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) f->shared->sblock->status_flags &= (uint8_t)(~H5F_SUPER_SWMR_WRITE_ACCESS); /* Mark EOA info dirty in cache, so change will get encoded */ - if(H5F_eoa_dirty(f, dxpl_id) < 0) + if(H5F_eoa_dirty(f, meta_dxpl_id) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty") @@ -924,12 +958,12 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) * At most, this should change the superblock or the * superblock extension messages. */ - if(H5MF_free_aggrs(f, dxpl_id) < 0) + if(H5MF_free_aggrs(f, meta_dxpl_id) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't release file space") /* Truncate the file to the current allocated size */ - if(H5FD_truncate(f->shared->lf, dxpl_id, TRUE) < 0) + if(H5FD_truncate(f->shared->lf, meta_dxpl_id, TRUE) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "low level truncate failed") @@ -968,10 +1002,15 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing file") /* Shutdown the metadata cache */ - if(H5AC_dest(f, dxpl_id)) + if(H5AC_dest(f, meta_dxpl_id)) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing file") + /* Shutdown the page buffer cache */ + if(H5PB_dest(f) < 0) + /* Push error, but keep going*/ + HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing page buffer cache") + /* Clean up the metadata cache log location string */ if(f->shared->mdc_log_location) f->shared->mdc_log_location = (char *)H5MM_xfree(f->shared->mdc_log_location); @@ -990,7 +1029,9 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) /* Set up I/O info for operation */ fio_info.f = f; - if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id))) + HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id))) HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") /* Destroy other components of the file */ @@ -1130,7 +1171,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) */ H5F_t * H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, - hid_t dxpl_id) + hid_t meta_dxpl_id) { H5F_t *file = NULL; /*the success return value */ H5F_file_t *shared = NULL; /*shared part of `file' */ @@ -1139,6 +1180,10 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_class_t *drvr; /*file driver class info */ H5P_genplist_t *a_plist; /*file access property list */ H5F_close_degree_t fc_degree; /*file close degree */ + hid_t raw_dxpl_id = H5AC_rawdata_dxpl_id; /* Raw data dxpl used by library */ + size_t page_buf_size; + unsigned page_buf_min_meta_perc; + unsigned page_buf_min_raw_perc; hbool_t set_flag = FALSE; /*set the status_flags in the superblock */ hbool_t clear = FALSE; /*clear the status_flags */ hbool_t evict_on_close; /* evict on close value from plist */ @@ -1289,6 +1334,25 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, if(NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list") + /* Check if page buffering is enabled */ + if(H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &page_buf_size) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get page buffer size") + if(page_buf_size) { +#ifdef H5_HAVE_PARALLEL + /* Collective metadata writes are not supported with page buffering */ + if(file->coll_md_write) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "collective metadata writes are not supported with page buffering") + + /* Temporary: fail file create when page buffering feature is enabled for parallel */ + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "page buffering is disabled for parallel") +#endif /* H5_HAVE_PARALLEL */ + /* Query for other page buffer cache properties */ + if(H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, &page_buf_min_meta_perc) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get minimum metadata fraction of page buffer") + if(H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, &page_buf_min_raw_perc) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get minimum raw data fraction of page buffer") + } /* end if */ + /* * Read or write the file superblock, depending on whether the file is * empty or not. @@ -1299,26 +1363,35 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, * to create & write the superblock. */ + /* Create the page buffer before initializing the superblock */ + if(page_buf_size) + if(H5PB_create(file, page_buf_size, page_buf_min_meta_perc, page_buf_min_raw_perc) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create page buffer") + /* Initialize information about the superblock and allocate space for it */ /* (Writes superblock extension messages, if there are any) */ - if(H5F__super_init(file, dxpl_id) < 0) + if(H5F__super_init(file, meta_dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to allocate file superblock") /* Create and open the root group */ /* (This must be after the space for the superblock is allocated in * the file, since the superblock must be at offset 0) */ - if(H5G_mkroot(file, dxpl_id, TRUE) < 0) + if(H5G_mkroot(file, meta_dxpl_id, TRUE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create/open root group") } /* end if */ else if (1 == shared->nrefs) { - /* Read the superblock if it hasn't been read before. */ - if(H5F__super_read(file, dxpl_id, TRUE) < 0) + if(H5F__super_read(file, meta_dxpl_id, raw_dxpl_id, TRUE) < 0) HGOTO_ERROR(H5E_FILE, H5E_READERROR, NULL, "unable to read superblock") + /* Create the page buffer before initializing the superblock */ + if(page_buf_size) + if(H5PB_create(file, page_buf_size, page_buf_min_meta_perc, page_buf_min_raw_perc) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create page buffer") + /* Open the root group */ - if(H5G_mkroot(file, dxpl_id, FALSE) < 0) + if(H5G_mkroot(file, meta_dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to read root group") } /* end if */ @@ -1360,12 +1433,11 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, */ if(H5P_get(a_plist, H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME, &evict_on_close) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get evict on close value") - - if(shared->nrefs == 1) { + if(shared->nrefs == 1) shared->evict_on_close = evict_on_close; - } else if(shared->nrefs > 1) { + else if(shared->nrefs > 1) { if(shared->evict_on_close != evict_on_close) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "file evict-on-close value doesn't match") + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "file evict-on-close value doesn't match") } /* end if */ /* Formulate the absolute path for later search of target file for external links */ @@ -1393,7 +1465,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, /* Flush the superblock */ if(H5F_super_dirty(file) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, NULL, "unable to mark superblock as dirty") - if(H5F_flush_tagged_metadata(file, H5AC__SUPERBLOCK_TAG, H5AC_ind_read_dxpl_id) < 0) + if(H5F_flush_tagged_metadata(file, H5AC__SUPERBLOCK_TAG, meta_dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, NULL, "unable to flush superblock") /* Remove the file lock for SWMR_WRITE */ @@ -1405,7 +1477,6 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, else { /* H5F_ACC_RDONLY: check consistency of status_flags */ /* Skip check of status_flags for file with < superblock version 3 */ if(file->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_3) { - if(H5F_INTENT(file) & H5F_ACC_SWMR_READ) { if((file->shared->sblock->status_flags & H5F_SUPER_WRITE_ACCESS && !(file->shared->sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS)) @@ -1413,12 +1484,10 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, (!(file->shared->sblock->status_flags & H5F_SUPER_WRITE_ACCESS) && file->shared->sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS)) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "file is not already open for SWMR writing") - } /* end if */ else if((file->shared->sblock->status_flags & H5F_SUPER_WRITE_ACCESS) || (file->shared->sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS)) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "file is already open for write (may use <h5clear file> to clear file consistency flags)") - } /* version 3 superblock */ } /* end else */ } /* end if set_flag */ @@ -1428,7 +1497,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, done: if((NULL == ret_value) && file) - if(H5F_dest(file, dxpl_id, FALSE) < 0) + if(H5F__dest(file, meta_dxpl_id, raw_dxpl_id, FALSE) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, NULL, "problems closing file") FUNC_LEAVE_NOAPI(ret_value) } /* end H5F_open() */ @@ -1448,7 +1517,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5F__flush_phase1(H5F_t *f, hid_t dxpl_id) +H5F__flush_phase1(H5F_t *f, hid_t meta_dxpl_id) { herr_t ret_value = SUCCEED; /* Return value */ @@ -1458,7 +1527,7 @@ H5F__flush_phase1(H5F_t *f, hid_t dxpl_id) HDassert(f); /* Flush any cached dataset storage raw data */ - if(H5D_flush(f, dxpl_id) < 0) + if(H5D_flush(f, meta_dxpl_id) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush dataset cache") @@ -1468,7 +1537,7 @@ H5F__flush_phase1(H5F_t *f, hid_t dxpl_id) /* (needs to happen before cache flush, with superblock write, since the * 'eoa' value is written in superblock -QAK) */ - if(H5MF_free_aggrs(f, dxpl_id) < 0) + if(H5MF_free_aggrs(f, meta_dxpl_id) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't release file space") @@ -1490,9 +1559,9 @@ H5F__flush_phase1(H5F_t *f, hid_t dxpl_id) *------------------------------------------------------------------------- */ static herr_t -H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing) +H5F__flush_phase2(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t closing) { - H5F_io_info_t fio_info; /* I/O info for operation */ + H5F_io_info2_t fio_info; /* I/O info for operation */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -1501,23 +1570,26 @@ H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing) HDassert(f); /* Flush the entire metadata cache */ - if(H5AC_flush(f, dxpl_id) < 0) + if(H5AC_flush(f, meta_dxpl_id) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush metadata cache") /* Truncate the file to the current allocated size */ - if(H5FD_truncate(f->shared->lf, dxpl_id, closing) < 0) + if(H5FD_truncate(f->shared->lf, meta_dxpl_id, closing) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "low level truncate failed") /* Flush the entire metadata cache again since the EOA could have changed in the truncate call. */ - if(H5AC_flush(f, dxpl_id) < 0) + if(H5AC_flush(f, meta_dxpl_id) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush metadata cache") /* Set up I/O info for operation */ fio_info.f = f; - if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id))) + /* Push error, but keep going*/ + HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(raw_dxpl_id))) /* Push error, but keep going*/ HDONE_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") @@ -1526,8 +1598,13 @@ H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing) /* Push error, but keep going*/ HDONE_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "unable to flush metadata accumulator") + /* Flush the page buffer */ + if(H5PB_flush(&fio_info) < 0) + /* Push error, but keep going*/ + HDONE_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "page buffer flush failed") + /* Flush file buffers to disk. */ - if(H5FD_flush(f->shared->lf, dxpl_id, closing) < 0) + if(H5FD_flush(f->shared->lf, meta_dxpl_id, closing) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "low level flush failed") @@ -1536,7 +1613,7 @@ H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing) /*------------------------------------------------------------------------- - * Function: H5F_flush + * Function: H5F__flush * * Purpose: Flushes cached data. * @@ -1549,28 +1626,27 @@ H5F__flush_phase2(H5F_t *f, hid_t dxpl_id, hbool_t closing) *------------------------------------------------------------------------- */ herr_t -H5F_flush(H5F_t *f, hid_t dxpl_id, hbool_t closing) +H5F__flush(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t closing) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_PACKAGE /* Sanity check arguments */ HDassert(f); /* First phase of flushing data */ - if(H5F__flush_phase1(f, dxpl_id) < 0) + if(H5F__flush_phase1(f, meta_dxpl_id) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush file data") /* Second phase of flushing data */ - if(H5F__flush_phase2(f, dxpl_id, closing) < 0) + if(H5F__flush_phase2(f, meta_dxpl_id, raw_dxpl_id, closing) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush file data") -done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5F_flush() */ +} /* end H5F__flush() */ /*------------------------------------------------------------------------- @@ -1795,7 +1871,7 @@ H5F_try_close(H5F_t *f, hbool_t *was_closed /*out*/) if(H5F_efc_try_close(f) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "can't attempt to close EFC") - /* Delay flush until the shared file struct is closed, in H5F_dest. If the + /* Delay flush until the shared file struct is closed, in H5F__dest. If the * application called H5Fclose, it would have been flushed in that function * (unless it will have been flushed in H5F_dest anyways). */ @@ -1804,7 +1880,7 @@ H5F_try_close(H5F_t *f, hbool_t *was_closed /*out*/) * shared H5F_file_t struct. If the reference count for the H5F_file_t * struct reaches zero then destroy it also. */ - if(H5F_dest(f, H5AC_ind_read_dxpl_id, TRUE) < 0) + if(H5F__dest(f, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id, TRUE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "problems closing file") /* Since we closed the file, this should be set to TRUE */ @@ -2362,7 +2438,8 @@ H5F_set_store_msg_crt_idx(H5F_t *f, hbool_t flag) *------------------------------------------------------------------------- */ ssize_t -H5F_get_file_image(H5F_t *file, void *buf_ptr, size_t buf_len, hid_t dxpl_id) +H5F_get_file_image(H5F_t *file, void *buf_ptr, size_t buf_len, hid_t meta_dxpl_id, + hid_t raw_dxpl_id) { H5FD_t *fd_ptr; /* file driver */ haddr_t eoa; /* End of file address */ @@ -2429,10 +2506,10 @@ H5F_get_file_image(H5F_t *file, void *buf_ptr, size_t buf_len, hid_t dxpl_id) /* test to see if a buffer was provided -- if not, we are done */ if(buf_ptr != NULL) { + H5FD_io_info_t fdio_info; /* File driver I/O info */ size_t space_needed; /* size of file image */ hsize_t tmp; size_t tmp_size; - H5P_genplist_t *xfer_plist= NULL; /* Dataset transfer property list object */ /* Check for buffer too small */ if((haddr_t)buf_len < eoa) @@ -2440,13 +2517,16 @@ H5F_get_file_image(H5F_t *file, void *buf_ptr, size_t buf_len, hid_t dxpl_id) space_needed = (size_t)eoa; - /* Get the property list object */ - if(NULL == (xfer_plist = (H5P_genplist_t *)H5I_object(dxpl_id))) - HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get new property list object") + /* Set up file driver I/O info object */ + fdio_info.file = fd_ptr; + if(NULL == (fdio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id))) + HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get property list object") + if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(raw_dxpl_id))) + HGOTO_ERROR(H5E_CACHE, H5E_BADATOM, FAIL, "can't get property list object") /* read in the file image */ /* (Note compensation for base address addition in internal routine) */ - if(H5FD_read(fd_ptr, xfer_plist, H5FD_MEM_DEFAULT, 0, space_needed, buf_ptr) < 0) + if(H5FD_read(&fdio_info, H5FD_MEM_DEFAULT, 0, space_needed, buf_ptr) < 0) HGOTO_ERROR(H5E_FILE, H5E_READERROR, FAIL, "file image read request failed") /* Offset to "status_flags" in the superblock */ @@ -2648,6 +2728,38 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F__set_eoa() */ + +/*------------------------------------------------------------------------- + * Function: H5F__set_paged_aggr + * + * Purpose: Quick and dirty routine to set the file's paged_aggr mode + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol <koziol@hdfgroup.org> + * June 19, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F__set_paged_aggr(const H5F_t *f, hbool_t paged) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity check */ + HDassert(f); + HDassert(f->shared); + + /* Dispatch to driver */ + if(H5FD_set_paged_aggr(f->shared->lf, paged) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "driver set paged aggr mode failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__set_paged_aggr() */ + #ifdef H5_HAVE_PARALLEL /*------------------------------------------------------------------------- diff --git a/src/H5Fio.c b/src/H5Fio.c index afe1278..6d23995 100644 --- a/src/H5Fio.c +++ b/src/H5Fio.c @@ -39,6 +39,7 @@ #include "H5Fpkg.h" /* File access */ #include "H5FDprivate.h" /* File drivers */ #include "H5Iprivate.h" /* IDs */ +#include "H5PBprivate.h" /* Page Buffer */ /****************/ @@ -96,15 +97,11 @@ herr_t H5F_block_read(const H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, hid_t dxpl_id, void *buf/*out*/) { - H5F_io_info_t fio_info; /* I/O info for operation */ + H5F_io_info2_t fio_info; /* I/O info for operation */ H5FD_mem_t map_type; /* Mapped memory type */ - hid_t my_dxpl_id = dxpl_id; /* transfer property to use for I/O */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) -#ifdef QAK -HDfprintf(stderr, "%s: read from addr = %a, size = %Zu\n", FUNC, addr, size); -#endif /* QAK */ HDassert(f); HDassert(f->shared); @@ -118,20 +115,24 @@ HDfprintf(stderr, "%s: read from addr = %a, size = %Zu\n", FUNC, addr, size); /* Treat global heap as raw data */ map_type = (type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type; -#ifdef H5_DEBUG_BUILD - /* GHEAP type is treated as RAW, so update the dxpl type property too */ - if(H5FD_MEM_GHEAP == type) - my_dxpl_id = H5AC_rawdata_dxpl_id; -#endif /* H5_DEBUG_BUILD */ - - /* Set up I/O info for operation */ + /* Set up the I/O info object */ fio_info.f = f; - if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(my_dxpl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(H5FD_MEM_DRAW == type) { + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end if */ + else { + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end else */ - /* Pass through metadata accumulator layer */ - if(H5F__accum_read(&fio_info, map_type, addr, size, buf) < 0) - HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "read through metadata accumulator failed") + /* Pass through page buffer layer */ + if(H5PB_read(&fio_info, map_type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "read through page buffer failed") done: FUNC_LEAVE_NOAPI(ret_value) @@ -157,15 +158,11 @@ herr_t H5F_block_write(const H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, hid_t dxpl_id, const void *buf) { - H5F_io_info_t fio_info; /* I/O info for operation */ + H5F_io_info2_t fio_info; /* I/O info for operation */ H5FD_mem_t map_type; /* Mapped memory type */ - hid_t my_dxpl_id = dxpl_id; /* transfer property to use for I/O */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) -#ifdef QAK -HDfprintf(stderr, "%s: write to addr = %a, size = %Zu\n", FUNC, addr, size); -#endif /* QAK */ HDassert(f); HDassert(f->shared); @@ -180,20 +177,24 @@ HDfprintf(stderr, "%s: write to addr = %a, size = %Zu\n", FUNC, addr, size); /* Treat global heap as raw data */ map_type = (type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type; -#ifdef H5_DEBUG_BUILD - /* GHEAP type is treated as RAW, so update the dxpl type property too */ - if(H5FD_MEM_GHEAP == type) - my_dxpl_id = H5AC_rawdata_dxpl_id; -#endif /* H5_DEBUG_BUILD */ - - /* Set up I/O info for operation */ + /* Set up the I/O info object */ fio_info.f = f; - if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(my_dxpl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(H5FD_MEM_DRAW == type) { + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end if */ + else { + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end else */ - /* Pass through metadata accumulator layer */ - if(H5F__accum_write(&fio_info, map_type, addr, size, buf) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write through metadata accumulator failed") + /* Pass through page buffer layer */ + if(H5PB_write(&fio_info, map_type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write through page buffer failed") done: FUNC_LEAVE_NOAPI(ret_value) @@ -216,7 +217,7 @@ done: herr_t H5F_flush_tagged_metadata(H5F_t * f, haddr_t tag, hid_t dxpl_id) { - H5F_io_info_t fio_info; /* I/O info for operation */ + H5F_io_info2_t fio_info; /* I/O info for operation */ herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI(FAIL) @@ -227,10 +228,10 @@ H5F_flush_tagged_metadata(H5F_t * f, haddr_t tag, hid_t dxpl_id) /* Set up I/O info for operation */ fio_info.f = f; - - if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") - /* Flush and reset the accumulator */ if(H5F__accum_reset(&fio_info, TRUE) < 0) diff --git a/src/H5Fmount.c b/src/H5Fmount.c index e3d4952..859b9d6 100644 --- a/src/H5Fmount.c +++ b/src/H5Fmount.c @@ -614,7 +614,7 @@ H5F_mount_count_ids(H5F_t *f, unsigned *nopen_files, unsigned *nopen_objs) *------------------------------------------------------------------------- */ static herr_t -H5F_flush_mounts_recurse(H5F_t *f, hid_t dxpl_id) +H5F_flush_mounts_recurse(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id) { unsigned nerrors = 0; /* Errors from recursive flushes */ unsigned u; /* Index variable */ @@ -627,11 +627,11 @@ H5F_flush_mounts_recurse(H5F_t *f, hid_t dxpl_id) /* Flush all child files, not stopping for errors */ for(u = 0; u < f->shared->mtab.nmounts; u++) - if(H5F_flush_mounts_recurse(f->shared->mtab.child[u].file, dxpl_id) < 0) + if(H5F_flush_mounts_recurse(f->shared->mtab.child[u].file, meta_dxpl_id, raw_dxpl_id) < 0) nerrors++; /* Call the "real" flush routine, for this file */ - if(H5F_flush(f, dxpl_id, FALSE) < 0) + if(H5F__flush(f, meta_dxpl_id, raw_dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file's cached information") /* Check flush errors for children - errors are already on the stack */ @@ -656,7 +656,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F_flush_mounts(H5F_t *f, hid_t dxpl_id) +H5F_flush_mounts(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id) { herr_t ret_value = SUCCEED; /* Return value */ @@ -670,7 +670,7 @@ H5F_flush_mounts(H5F_t *f, hid_t dxpl_id) f = f->parent; /* Flush the mounted file hierarchy */ - if(H5F_flush_mounts_recurse(f, dxpl_id) < 0) + if(H5F_flush_mounts_recurse(f, meta_dxpl_id, raw_dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush mounted file hierarchy") done: diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 93a3978..4dd9e20 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -42,6 +42,7 @@ #include "H5FSprivate.h" /* File free space */ #include "H5Gprivate.h" /* Groups */ #include "H5Oprivate.h" /* Object header messages */ +#include "H5PBprivate.h" /* Page buffer */ #include "H5UCprivate.h" /* Reference counted object functions */ @@ -68,8 +69,8 @@ /* Macro to abstract checking whether file is using a free space manager */ #define H5F_HAVE_FREE_SPACE_MANAGER(F) \ - ((F)->shared->fs_strategy == H5F_FILE_SPACE_ALL || \ - (F)->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) + ((F)->shared->fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR || \ + (F)->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE) /* Macros for encoding/decoding superblock */ #define H5F_MAX_DRVINFOBLOCK_SIZE 1024 /* Maximum size of superblock driver info buffer */ @@ -200,13 +201,6 @@ typedef struct H5F_meta_accum_t { hbool_t dirty; /* Flag to indicate that the accumulated metadata is dirty */ } H5F_meta_accum_t; -/* Enum for free space manager state */ -typedef enum H5F_fs_state_t { - H5F_FS_STATE_CLOSED, /* Free space manager is closed */ - H5F_FS_STATE_OPEN, /* Free space manager has been opened */ - H5F_FS_STATE_DELETING /* Free space manager is being deleted */ -} H5F_fs_state_t; - /* A record of the mount table */ typedef struct H5F_mount_t { struct H5G_t *group; /* Mount point group held open */ @@ -281,6 +275,7 @@ struct H5F_file_t { unsigned long feature_flags; /* VFL Driver feature Flags */ haddr_t maxaddr; /* Maximum address for file */ + H5PB_t *page_buf; /* The page buffer cache */ H5AC_t *cache; /* The object cache */ H5AC_cache_config_t mdc_initCacheCfg; /* initial configuration for the */ @@ -316,19 +311,38 @@ struct H5F_file_t { H5UC_t *grp_btree_shared; /* Ref-counted group B-tree node info */ /* File space allocation information */ - H5F_file_space_type_t fs_strategy; /* File space handling strategy */ + H5F_fspace_strategy_t fs_strategy; /* File space handling strategy */ hsize_t fs_threshold; /* Free space section threshold */ + hbool_t fs_persist; /* Free-space persist or not */ hbool_t use_tmp_space; /* Whether temp. file space allocation is allowed */ haddr_t tmp_addr; /* Next address to use for temp. space in the file */ + hbool_t point_of_no_return; /* flag to indicate that we can't go back and delete a freespace header when it's used up */ + + H5F_fs_state_t fs_state[H5F_MEM_PAGE_NTYPES]; /* State of free space manager for each type */ + haddr_t fs_addr[H5F_MEM_PAGE_NTYPES]; /* Address of free space manager info for each type */ + H5FS_t *fs_man[H5F_MEM_PAGE_NTYPES]; /* Free space manager for each file space type */ + hbool_t first_alloc_dealloc; /* TRUE iff free space managers */ + /* are persistant and have not */ + /* been used accessed for either */ + /* allocation or deallocation */ + /* since file open. */ + haddr_t eoa_pre_fsm_fsalloc; /* eoa pre file space allocation */ + /* for self referential FSMs */ + haddr_t eoa_post_fsm_fsalloc; /* eoa post file space allocation */ + /* for self referential FSMs */ + haddr_t eoa_post_mdci_fsalloc; /* eoa past file space allocation */ + /* for metadata cache image, or */ + /* HADDR_UNDEF if no cache image. */ + + /* Free-space aggregation info */ unsigned fs_aggr_merge[H5FD_MEM_NTYPES]; /* Flags for whether free space can merge with aggregator(s) */ - H5F_fs_state_t fs_state[H5FD_MEM_NTYPES]; /* State of free space manager for each type */ - haddr_t fs_addr[H5FD_MEM_NTYPES]; /* Address of free space manager info for each type */ - H5FS_t *fs_man[H5FD_MEM_NTYPES]; /* Free space manager for each file space type */ - H5FD_mem_t fs_type_map[H5FD_MEM_NTYPES]; /* Mapping of "real" file space type into tracked type */ - H5F_blk_aggr_t meta_aggr; /* Metadata aggregation info */ - /* (if aggregating metadata allocations) */ - H5F_blk_aggr_t sdata_aggr; /* "Small data" aggregation info */ - /* (if aggregating "small data" allocations) */ + H5FD_mem_t fs_type_map[H5FD_MEM_NTYPES]; /* Mapping of "real" file space type into tracked type */ + H5F_blk_aggr_t meta_aggr; /* Metadata aggregation info (if aggregating metadata allocations) */ + H5F_blk_aggr_t sdata_aggr; /* "Small data" aggregation info (if aggregating "small data" allocations) */ + + /* Paged aggregation info */ + hsize_t fs_page_size; /* File space page size */ + size_t pgend_meta_thres; /* Do not track page end meta section <= this threshold */ /* Metadata accumulator information */ H5F_meta_accum_t accum; /* Metadata accumulator info */ @@ -383,11 +397,12 @@ H5FL_EXTERN(H5F_file_t); /* General routines */ H5F_t *H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t *lf); -herr_t H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush); -H5_DLL herr_t H5F_flush(H5F_t *f, hid_t dxpl_id, hbool_t closing); -H5_DLL htri_t H5F_is_hdf5(const char *name, hid_t dxpl_id); +H5_DLL herr_t H5F__dest(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t flush); +H5_DLL herr_t H5F__flush(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t closing); +H5_DLL htri_t H5F__is_hdf5(const char *name, hid_t meta_dxpl_id, hid_t raw_dxpl_id); H5_DLL herr_t H5F_get_objects(const H5F_t *f, unsigned types, size_t max_index, hid_t *obj_id_list, hbool_t app_ref, size_t *obj_id_count_ptr); -H5_DLL ssize_t H5F_get_file_image(H5F_t *f, void *buf_ptr, size_t buf_len, hid_t dxpl_id); +H5_DLL ssize_t H5F_get_file_image(H5F_t *f, void *buf_ptr, size_t buf_len, + hid_t meta_dxpl_id, hid_t raw_dxpl_id); H5_DLL herr_t H5F_close(H5F_t *f); /* File mount related routines */ @@ -397,7 +412,8 @@ H5_DLL herr_t H5F_mount_count_ids(H5F_t *f, unsigned *nopen_files, unsigned *nop /* Superblock related routines */ H5_DLL herr_t H5F__super_init(H5F_t *f, hid_t dxpl_id); -H5_DLL herr_t H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read); +H5_DLL herr_t H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, + hbool_t initial_read); H5_DLL herr_t H5F__super_size(H5F_t *f, hid_t dxpl_id, hsize_t *super_size, hsize_t *super_ext_size); H5_DLL herr_t H5F__super_free(H5F_super_t *sblock); @@ -410,14 +426,14 @@ H5_DLL herr_t H5F_super_ext_close(H5F_t *f, H5O_loc_t *ext_ptr, hid_t dxpl_id, hbool_t was_created); /* Metadata accumulator routines */ -H5_DLL herr_t H5F__accum_read(const H5F_io_info_t *fio_info, H5FD_mem_t type, +H5_DLL herr_t H5F__accum_read(const H5F_io_info2_t *fio_info, H5FD_mem_t type, haddr_t addr, size_t size, void *buf); -H5_DLL herr_t H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t type, +H5_DLL herr_t H5F__accum_write(const H5F_io_info2_t *fio_info, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf); -H5_DLL herr_t H5F__accum_free(const H5F_io_info_t *fio_info, H5FD_mem_t type, +H5_DLL herr_t H5F__accum_free(const H5F_io_info2_t *fio_info, H5FD_mem_t type, haddr_t addr, hsize_t size); -H5_DLL herr_t H5F__accum_flush(const H5F_io_info_t *fio_info); -H5_DLL herr_t H5F__accum_reset(const H5F_io_info_t *fio_info, hbool_t flush); +H5_DLL herr_t H5F__accum_flush(const H5F_io_info2_t *fio_info); +H5_DLL herr_t H5F__accum_reset(const H5F_io_info2_t *fio_info, hbool_t flush); /* Shared file list related routines */ H5_DLL herr_t H5F_sfile_add(H5F_file_t *shared); @@ -431,9 +447,16 @@ H5_DLL herr_t H5F_efc_release(H5F_efc_t *efc); H5_DLL herr_t H5F_efc_destroy(H5F_efc_t *efc); H5_DLL herr_t H5F_efc_try_close(H5F_t *f); +/* Space allocation routines */ +H5_DLL haddr_t H5F_alloc(H5F_t *f, hid_t dxpl_id, H5F_mem_t type, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size); +H5_DLL herr_t H5F_free(H5F_t *f, hid_t dxpl_id, H5F_mem_t type, haddr_t addr, hsize_t size); +H5_DLL htri_t H5F_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, + haddr_t blk_end, hsize_t extra_requested); + /* Functions that get/retrieve values from VFD layer */ H5_DLL herr_t H5F__set_eoa(const H5F_t *f, H5F_mem_t type, haddr_t addr); H5_DLL herr_t H5F__set_base_addr(const H5F_t *f, haddr_t addr); +H5_DLL herr_t H5F__set_paged_aggr(const H5F_t *f, hbool_t paged); /* Functions that flush or evict */ H5_DLL herr_t H5F__evict_cache_entries(H5F_t *f, hid_t dxpl_id); diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 7d288fa..886063a 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -315,8 +315,8 @@ #define H5F_SET_STORE_MSG_CRT_IDX(F, FL) ((F)->shared->store_msg_crt_idx = (FL)) #define H5F_GRP_BTREE_SHARED(F) ((F)->shared->grp_btree_shared) #define H5F_SET_GRP_BTREE_SHARED(F, RC) (((F)->shared->grp_btree_shared = (RC)) ? SUCCEED : FAIL) -#define H5F_USE_TMP_SPACE(F) ((F)->shared->use_tmp_space) -#define H5F_IS_TMP_ADDR(F, ADDR) (H5F_addr_le((F)->shared->tmp_addr, (ADDR))) +#define H5F_USE_TMP_SPACE(F) ((F)->shared->fs.use_tmp_space) +#define H5F_IS_TMP_ADDR(F, ADDR) (H5F_addr_le((F)->shared->fs.tmp_addr, (ADDR))) #define H5F_SET_LATEST_FLAGS(F, FL) ((F)->shared->latest_flags = (FL)) #ifdef H5_HAVE_PARALLEL #define H5F_COLL_MD_READ(F) ((F)->coll_md_read) @@ -324,6 +324,12 @@ #define H5F_USE_MDC_LOGGING(F) ((F)->shared->use_mdc_logging) #define H5F_START_MDC_LOG_ON_ACCESS(F) ((F)->shared->start_mdc_log_on_access) #define H5F_MDC_LOG_LOCATION(F) ((F)->shared->mdc_log_location) +#define H5F_ALIGNMENT(F) ((F)->shared->alignment) +#define H5F_THRESHOLD(F) ((F)->shared->threshold) +#define H5F_PGEND_META_THRES(F) ((F)->shared->fs.pgend_meta_thres) +#define H5F_POINT_OF_NO_RETURN(F) ((F)->shared->fs.point_of_no_return) +#define H5F_FIRST_ALLOC_DEALLOC(F) ((F)->shared->first_alloc_dealloc) +#define H5F_EOA_PRE_FSM_FSALLOC(F) ((F)->shared->eoa_pre_fsm_fsalloc) #else /* H5F_MODULE */ #define H5F_INTENT(F) (H5F_get_intent(F)) #define H5F_OPEN_NAME(F) (H5F_get_open_name(F)) @@ -375,6 +381,12 @@ #define H5F_USE_MDC_LOGGING(F) (H5F_use_mdc_logging(F)) #define H5F_START_MDC_LOG_ON_ACCESS(F) (H5F_start_mdc_log_on_access(F)) #define H5F_MDC_LOG_LOCATION(F) (H5F_mdc_log_location(F)) +#define H5F_ALIGNMENT(F) (H5F_get_alignment(F)) +#define H5F_THRESHOLD(F) (H5F_get_threshold(F)) +#define H5F_PGEND_META_THRES(F) (H5F_get_pgend_meta_thres(F)) +#define H5F_POINT_OF_NO_RETURN(F) (H5F_get_point_of_no_return(F)) +#define H5F_FIRST_ALLOC_DEALLOC(F) (H5F_get_first_alloc_dealloc(F)) +#define H5F_EOA_PRE_FSM_FSALLOC(F) (H5F_get_eoa_pre_fsm_fsalloc(F)) #endif /* H5F_MODULE */ @@ -448,7 +460,9 @@ #define H5F_CRT_SHMSG_LIST_MAX_NAME "shmsg_list_max" /* Shared message list maximum size */ #define H5F_CRT_SHMSG_BTREE_MIN_NAME "shmsg_btree_min" /* Shared message B-tree minimum size */ #define H5F_CRT_FILE_SPACE_STRATEGY_NAME "file_space_strategy" /* File space handling strategy */ +#define H5F_CRT_FREE_SPACE_PERSIST_NAME "free_space_persist" /* Free-space persisting status */ #define H5F_CRT_FREE_SPACE_THRESHOLD_NAME "free_space_threshold" /* Free space section threshold */ +#define H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME "file_space_page_size" /* File space page size */ @@ -484,6 +498,9 @@ #define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME "core_write_tracking_page_size" /* The page size in kiB when core VFD write tracking is enabled */ #define H5F_ACS_COLL_MD_WRITE_FLAG_NAME "collective_metadata_write" /* property indicating whether metadata writes are done collectively or not */ #define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_NAME "mdc_initCacheImageCfg" /* Initial metadata cache image creation configuration */ +#define H5F_ACS_PAGE_BUFFER_SIZE_NAME "page_buffer_size" /* the maximum size for the page buffer cache */ +#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME "page_buffer_min_meta_perc" /* the min metadata percentage for the page buffer cache */ +#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME "page_buffer_min_raw_perc" /* the min raw data percentage for the page buffer cache */ /* ======================== File Mount properties ====================*/ #define H5F_MNT_SYM_LOCAL_NAME "local" /* Whether absolute symlinks local to file. */ @@ -522,10 +539,34 @@ /* See format specification on version 1 B-trees */ /* Default file space handling strategy */ -#define H5F_FILE_SPACE_STRATEGY_DEF H5F_FILE_SPACE_ALL +#define H5F_FILE_SPACE_STRATEGY_DEF H5F_FSPACE_STRATEGY_FSM_AGGR + +/* Default free space section threshold used by free-space managers */ +#define H5F_FREE_SPACE_PERSIST_DEF FALSE + /* Default free space section threshold used by free-space managers */ #define H5F_FREE_SPACE_THRESHOLD_DEF 1 +/* For paged aggregation: default file space page size when not set */ +#define H5F_FILE_SPACE_PAGE_SIZE_DEF 4096 +/* For paged aggregation: minimum value for file space page size */ +#define H5F_FILE_SPACE_PAGE_SIZE_MIN 512 + +/* For paged aggregation: drop free-space with size <= this threshold for small meta section */ +#define H5F_FILE_SPACE_PGEND_META_THRES 10 + +/* Default for threshold for alignment (can be set via H5Pset_alignment()) */ +#define H5F_ALIGN_DEF 1 +/* Default for alignment (can be set via H5Pset_alignment()) */ +#define H5F_ALIGN_THRHD_DEF 1 +/* Default size for meta data aggregation block (can be set via H5Pset_meta_block_size()) */ +#define H5F_META_BLOCK_SIZE_DEF 2048 +/* Default size for small data aggregation block (can be set via H5Pset_small_data_block_size()) */ +#define H5F_SDATA_BLOCK_SIZE_DEF 2048 + +/* Check for file using paged aggregation */ +#define H5F_PAGED_AGGR(F) (F->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE && F->shared->fs_page_size) + /* Metadata read attempt values */ #define H5F_METADATA_READ_ATTEMPTS 1 /* Default # of read attempts for non-SWMR access */ #define H5F_SWMR_METADATA_READ_ATTEMPTS 100 /* Default # of read attempts for SWMR access */ @@ -628,18 +669,54 @@ typedef struct H5F_object_flush_t { void *udata; /* User data */ } H5F_object_flush_t; -/* I/O Info for an operation */ +/* I/O Info for an operation (old) */ typedef struct H5F_io_info_t { const H5F_t *f; /* File object */ const struct H5P_genplist_t *dxpl; /* DXPL object */ } H5F_io_info_t; +/* I/O Info for an operation */ +/* (Migrate toward this one, so that both raw data & metadata DXPLs are available) */ +typedef struct H5F_io_info2_t { + const H5F_t *f; /* File object */ + const struct H5P_genplist_t *meta_dxpl; /* Metadata DXPL object */ + const struct H5P_genplist_t *raw_dxpl; /* Raw data DXPL object */ +} H5F_io_info2_t; + /* Concise info about a block of bytes in a file */ typedef struct H5F_block_t { haddr_t offset; /* Offset of the block in the file */ hsize_t length; /* Length of the block in the file */ } H5F_block_t; +/* Enum for free space manager state */ +typedef enum H5F_fs_state_t { + H5F_FS_STATE_CLOSED = 0, /* Free space manager is closed */ + H5F_FS_STATE_OPEN = 1, /* Free space manager has been opened */ + H5F_FS_STATE_DELETING = 2 /* Free space manager is being deleted */ +} H5F_fs_state_t; + +/* For paged aggregation */ +/* The values 0 to 6 is the same as H5F_mem_t */ +typedef enum H5F_mem_page_t { + H5F_MEM_PAGE_DEFAULT = 0, /* Not used */ + H5F_MEM_PAGE_SUPER = 1, + H5F_MEM_PAGE_BTREE = 2, + H5F_MEM_PAGE_DRAW = 3, + H5F_MEM_PAGE_GHEAP = 4, + H5F_MEM_PAGE_LHEAP = 5, + H5F_MEM_PAGE_OHDR = 6, + H5F_MEM_PAGE_LARGE_SUPER = 7, + H5F_MEM_PAGE_LARGE_BTREE = 8, + H5F_MEM_PAGE_LARGE_DRAW = 9, + H5F_MEM_PAGE_LARGE_GHEAP = 10, + H5F_MEM_PAGE_LARGE_LHEAP = 11, + H5F_MEM_PAGE_LARGE_OHDR = 12, + H5F_MEM_PAGE_NTYPES = 13 /* Sentinel value - must be last */ +} H5F_mem_page_t; + +#define H5F_MEM_PAGE_META H5F_MEM_PAGE_SUPER /* Small-sized meta data */ +#define H5F_MEM_PAGE_GENERIC H5F_MEM_PAGE_LARGE_SUPER /* Large-sized generic: meta and raw */ /*****************************/ /* Library-private Variables */ @@ -674,6 +751,10 @@ H5_DLL hid_t H5F_get_access_plist(H5F_t *f, hbool_t app_ref); H5_DLL hid_t H5F_get_id(H5F_t *file, hbool_t app_ref); H5_DLL herr_t H5F_get_obj_count(const H5F_t *f, unsigned types, hbool_t app_ref, size_t *obj_id_count_ptr); H5_DLL herr_t H5F_get_obj_ids(const H5F_t *f, unsigned types, size_t max_objs, hid_t *oid_list, hbool_t app_ref, size_t *obj_id_count_ptr); +H5_DLL hsize_t H5F_get_pgend_meta_thres(const H5F_t *f); +H5_DLL hbool_t H5F_get_point_of_no_return(const H5F_t *f); +H5_DLL hbool_t H5F_get_first_alloc_dealloc(const H5F_t *f); +H5_DLL hbool_t H5F_get_eoa_pre_fsm_fsalloc(const H5F_t *f); /* Functions than retrieve values set/cached from the superblock/FCPL */ H5_DLL haddr_t H5F_get_base_addr(const H5F_t *f); @@ -704,6 +785,8 @@ H5_DLL herr_t H5F_set_grp_btree_shared(H5F_t *f, struct H5UC_t *rc); H5_DLL hbool_t H5F_use_tmp_space(const H5F_t *f); H5_DLL hbool_t H5F_is_tmp_addr(const H5F_t *f, haddr_t addr); H5_DLL herr_t H5F_set_latest_flags(H5F_t *f, unsigned flags); +H5_DLL hsize_t H5F_get_alignment(const H5F_t *f); +H5_DLL hsize_t H5F_get_threshold(const H5F_t *f); #ifdef H5_HAVE_PARALLEL H5_DLL H5P_coll_md_read_flag_t H5F_coll_md_read(const H5F_t *f); H5_DLL void H5F_set_coll_md_read(H5F_t *f, H5P_coll_md_read_flag_t flag); @@ -723,7 +806,7 @@ H5_DLL herr_t H5F_get_vfd_handle(const H5F_t *file, hid_t fapl, void **file_hand H5_DLL hbool_t H5F_is_mount(const H5F_t *file); H5_DLL hbool_t H5F_has_mount(const H5F_t *file); H5_DLL herr_t H5F_traverse_mount(struct H5O_loc_t *oloc/*in,out*/); -H5_DLL herr_t H5F_flush_mounts(H5F_t *f, hid_t dxpl_id); +H5_DLL herr_t H5F_flush_mounts(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id); /* Functions that operate on blocks of bytes wrt super block */ H5_DLL herr_t H5F_block_read(const H5F_t *f, H5FD_mem_t type, haddr_t addr, diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index c57a821..f87aaad 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -182,6 +182,17 @@ typedef enum H5F_libver_t { } H5F_libver_t; /* File space handling strategy */ +typedef enum H5F_fspace_strategy_t { + H5F_FSPACE_STRATEGY_FSM_AGGR = 0, /* Mechanisms: free-space managers, aggregators, and virtual file drivers */ + /* This is the library default when not set */ + H5F_FSPACE_STRATEGY_PAGE = 1, /* Mechanisms: free-space managers with embedded paged aggregation and virtual file drivers */ + H5F_FSPACE_STRATEGY_AGGR = 2, /* Mechanisms: aggregators and virtual file drivers */ + H5F_FSPACE_STRATEGY_NONE = 3, /* Mechanisms: virtual file drivers */ + H5F_FSPACE_STRATEGY_NTYPES /* must be last */ +} H5F_fspace_strategy_t; + +/* Deprecated: File space handling strategy for release 1.10.0 */ +/* They are mapped to H5F_fspace_strategy_t as defined above from release 1.10.1 onwards */ typedef enum H5F_file_space_type_t { H5F_FILE_SPACE_DEFAULT = 0, /* Default (or current) free space strategy setting */ H5F_FILE_SPACE_ALL_PERSIST = 1, /* Persistent free space managers, aggregators, virtual file driver */ @@ -253,6 +264,10 @@ H5_DLL herr_t H5Fget_mdc_logging_status(hid_t file_id, /*OUT*/ hbool_t *is_enabled, /*OUT*/ hbool_t *is_currently_logging); H5_DLL herr_t H5Fformat_convert(hid_t fid); +H5_DLL herr_t H5Freset_page_buffering_stats(hid_t file_id); +H5_DLL herr_t H5Fget_page_buffering_stats(hid_t file_id, unsigned accesses[2], + unsigned hits[2], unsigned misses[2], unsigned evictions[2], unsigned bypasses[2]); + #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5Fset_mpi_atomicity(hid_t file_id, hbool_t flag); H5_DLL herr_t H5Fget_mpi_atomicity(hid_t file_id, hbool_t *flag); diff --git a/src/H5Fquery.c b/src/H5Fquery.c index 14dd655..34fe8f9 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -1235,3 +1235,150 @@ H5F_mdc_log_location(const H5F_t *f) FUNC_LEAVE_NOAPI(f->shared->mdc_log_location) } /* end H5F_mdc_log_location() */ + +/*------------------------------------------------------------------------- + * Function: H5F_get_alignment + * + * Purpose: Retrieve the 'alignment' for the file. + * + * Return: Success: Non-negative, the 'alignment' + * + * Failure: (can't happen) + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +hsize_t +H5F_get_alignment(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->alignment) +} /* end H5F_get_alignment() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_get_threshold + * + * Purpose: Retrieve the 'threshold' for alignment in the file. + * + * Return: Success: Non-negative, the 'threshold' + * + * Failure: (can't happen) + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +hsize_t +H5F_get_threshold(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->threshold) +} /* end H5F_get_threshold() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_get_pgend_meta_thres + * + * Purpose: Retrieve the 'page end meta threshold size' for the file. + * + * Return: Success: Non-negative, the 'pgend_meta_thres' + * + * Failure: (can't happen) + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +hsize_t +H5F_get_pgend_meta_thres(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->pgend_meta_thres) +} /* end H5F_get_pgend_meta_thres() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_get_point_of_no_return + * + * Purpose: Retrieve the 'point of no return' value for the file. + * + * Return: Success: Non-negative, the 'point_of_no_return' + * Failure: (can't happen) + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_get_point_of_no_return(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->point_of_no_return) +} /* end H5F_get_point_of_no_return() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_get_first_alloc_dealloc + * + * Purpose: Retrieve the 'first alloc / dealloc' value for the file. + * + * Return: Success: Non-negative, the 'first_alloc_dealloc' + * Failure: (can't happen) + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_get_first_alloc_dealloc(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->first_alloc_dealloc) +} /* end H5F_get_first_alloc_dealloc() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_get_eoa_pre_fsm_fsalloc + * + * Purpose: Retrieve the 'EOA pre-FSM fsalloc' value for the file. + * + * Return: Success: Non-negative, the 'EOA pre-FSM fsalloc' + * Failure: (can't happen) + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_get_eoa_pre_fsm_fsalloc(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->eoa_pre_fsm_fsalloc) +} /* end H5F_get_eoa_pre_fsm_fsalloc() */ + diff --git a/src/H5Fspace.c b/src/H5Fspace.c new file mode 100644 index 0000000..53570f6 --- /dev/null +++ b/src/H5Fspace.c @@ -0,0 +1,226 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5Fspace.c + * Dec 30 2013 + * Quincey Koziol <koziol@hdfgroup.org> + * + * Purpose: Space allocation routines for the file. + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5Fmodule.h" /* This source code file is part of the H5F module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* File access */ +#include "H5FDprivate.h" /* File drivers */ + + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + + + +/*------------------------------------------------------------------------- + * Function: H5F_alloc + * + * Purpose: Wrapper for H5FD_alloc, to make certain EOA changes are + * reflected in superblock. + * + * Note: When the metadata cache routines are updated to allow + * marking an entry dirty without a H5F_t*, this routine should + * be changed to take a H5F_super_t* directly. + * + * Return: Success: The format address of the new file memory. + * Failure: The undefined address HADDR_UNDEF + * + * Programmer: Quincey Koziol + * Monday, December 30, 2013 + * + *------------------------------------------------------------------------- + */ +haddr_t +H5F_alloc(H5F_t *f, hid_t dxpl_id, H5F_mem_t type, hsize_t size, haddr_t *frag_addr, hsize_t *frag_size) +{ + haddr_t ret_value = 0; /* Return value */ + + FUNC_ENTER_NOAPI(HADDR_UNDEF) + + /* check args */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); + HDassert(type >= H5FD_MEM_DEFAULT && type < H5FD_MEM_NTYPES); + HDassert(size > 0); + + /* Check whether the file can use temporary addresses */ + if(f->shared->use_tmp_space) { + haddr_t eoa; /* Current EOA for the file */ + + /* Get the EOA for the file */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, type))) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, HADDR_UNDEF, "Unable to get eoa") + + /* Check for overlapping into file's temporary allocation space */ + if(H5F_addr_gt((eoa + size), f->shared->tmp_addr)) + HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space") + } /* end if */ + + /* Call the file driver 'alloc' routine */ + ret_value = H5FD_alloc(f->shared->lf, dxpl_id, type, f, size, frag_addr, frag_size); + if(!H5F_addr_defined(ret_value)) + HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, HADDR_UNDEF, "file driver 'alloc' request failed") + + /* Mark EOA dirty */ + if(H5F_eoa_dirty(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, HADDR_UNDEF, "unable to mark EOA as dirty") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_alloc() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_free + * + * Purpose: Wrapper for H5FD_free, to make certain EOA changes are + * reflected in superblock. + * + * Note: When the metadata cache routines are updated to allow + * marking an entry dirty without a H5F_t*, this routine should + * be changed to take a H5F_super_t* directly. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Quincey Koziol + * Monday, December 30, 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, hsize_t size) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Check args */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); + HDassert(type >= H5FD_MEM_DEFAULT && type < H5FD_MEM_NTYPES); + HDassert(size > 0); + + /* Call the file driver 'free' routine */ + if(H5FD_free(f->shared->lf, dxpl_id, type, f, addr, size) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "file driver 'free' request failed") + + /* Mark EOA dirty */ + if(H5F_eoa_dirty(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark EOA as dirty") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_free() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_try_extend + * + * Purpose: Extend a block at the end of the file, if possible. + * + * Note: When the metadata cache routines are updated to allow + * marking an entry dirty without a H5F_t*, this routine should + * be changed to take a H5F_super_t* directly. + * + * Return: Success: TRUE(1) - Block was extended + * FALSE(0) - Block could not be extended + * Failure: FAIL + * + * Programmer: Quincey Koziol + * Monday, 30 December, 2013 + * + *------------------------------------------------------------------------- + */ +htri_t +H5F_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t blk_end, hsize_t extra_requested) +{ + htri_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* check args */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); + HDassert(type >= H5FD_MEM_DEFAULT && type < H5FD_MEM_NTYPES); + HDassert(extra_requested > 0); + + /* Extend the object by extending the underlying file */ + if((ret_value = H5FD_try_extend(f->shared->lf, type, f, dxpl_id, blk_end, extra_requested)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTEXTEND, FAIL, "driver try extend request failed") + + /* H5FD_try_extend() updates driver message and marks the superblock + * dirty, so no need to do it again here. + */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_try_extend() */ + diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c index 3b86dae..58ef9bb 100644 --- a/src/H5Fsuper.c +++ b/src/H5Fsuper.c @@ -29,6 +29,7 @@ #include "H5Fpkg.h" /* File access */ #include "H5FDprivate.h" /* File drivers */ #include "H5Iprivate.h" /* IDs */ +#include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ #include "H5Pprivate.h" /* Property lists */ #include "H5SMprivate.h" /* Shared Object Header Messages */ @@ -320,13 +321,14 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) +H5F__super_read(H5F_t *f, hid_t meta_dxpl_id, hid_t raw_dxpl_id, hbool_t initial_read) { H5P_genplist_t *dxpl = NULL; /* DXPL object */ H5AC_ring_t ring, orig_ring = H5AC_RING_INV; H5F_super_t * sblock = NULL; /* Superblock structure */ H5F_superblock_cache_ud_t udata; /* User data for cache callbacks */ H5P_genplist_t *c_plist; /* File creation property list */ + H5FD_io_info_t fdio_info; /* File driver I/O info */ unsigned sblock_flags = H5AC__NO_FLAGS_SET; /* flags used in superblock unprotect call */ haddr_t super_addr; /* Absolute address of superblock */ haddr_t eof; /* End of file address */ @@ -334,7 +336,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) hbool_t skip_eof_check = FALSE; /* Whether to skip checking the EOF value */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_PACKAGE_TAG(dxpl_id, H5AC__SUPERBLOCK_TAG, FAIL) + FUNC_ENTER_PACKAGE_TAG(meta_dxpl_id, H5AC__SUPERBLOCK_TAG, FAIL) /* initialize the drvinfo to NULL -- we will overwrite this if there * is a driver information block @@ -342,13 +344,19 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) f->shared->drvinfo = NULL; /* Get the DXPL plist object for DXPL ID */ - if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(meta_dxpl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") if((H5P_get(dxpl, H5AC_RING_NAME, &orig_ring)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get property value"); + /* Set up file driver I/O info */ + fdio_info.file = f->shared->lf; + fdio_info.meta_dxpl = dxpl; + if(NULL == (fdio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(raw_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + /* Find the superblock */ - if(H5FD_locate_signature(f->shared->lf, dxpl, &super_addr) < 0) + if(H5FD_locate_signature(&fdio_info, &super_addr) < 0) HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable to locate file signature") if(HADDR_UNDEF == super_addr) HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "file signature not found") @@ -396,7 +404,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set property value"); /* Look up the superblock */ - if(NULL == (sblock = (H5F_super_t *)H5AC_protect(f, dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, &udata, rw_flags))) + if(NULL == (sblock = (H5F_super_t *)H5AC_protect(f, meta_dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, &udata, rw_flags))) HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "unable to load superblock") if(H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) @@ -556,7 +564,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed") /* Look up the driver info block */ - if(NULL == (drvinfo = (H5O_drvinfo_t *)H5AC_protect(f, dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, &drvrinfo_udata, rw_flags))) + if(NULL == (drvinfo = (H5O_drvinfo_t *)H5AC_protect(f, meta_dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, &drvrinfo_udata, rw_flags))) HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "unable to load driver info block") /* Loading the driver info block is enough to set up the right info */ @@ -571,7 +579,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) drvinfo_flags |= H5AC__PIN_ENTRY_FLAG; /* Release the driver info block */ - if(H5AC_unprotect(f, dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, drvinfo, drvinfo_flags) < 0) + if(H5AC_unprotect(f, meta_dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, drvinfo, drvinfo_flags) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTUNPROTECT, FAIL, "unable to release driver info block") /* save a pointer to the driver information cache entry */ @@ -612,14 +620,14 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, FAIL, "unable to open file's superblock extension") /* Check for the extension having a 'driver info' message */ - if((status = H5O_msg_exists(&ext_loc, H5O_DRVINFO_ID, dxpl_id)) < 0) + if((status = H5O_msg_exists(&ext_loc, H5O_DRVINFO_ID, meta_dxpl_id)) < 0) HGOTO_ERROR(H5E_FILE, H5E_EXISTS, FAIL, "unable to read object header") if(status) { /* Check for ignoring the driver info for this file */ if(!udata.ignore_drvrinfo) { /* Retrieve the 'driver info' structure */ - if(NULL == H5O_msg_read(&ext_loc, H5O_DRVINFO_ID, &drvinfo, dxpl_id)) + if(NULL == H5O_msg_read(&ext_loc, H5O_DRVINFO_ID, &drvinfo, meta_dxpl_id)) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "driver info message not present") /* Validate and decode driver information */ @@ -637,15 +645,15 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) } /* end if */ /* Read in the shared OH message information if there is any */ - if(H5SM_get_info(&ext_loc, c_plist, dxpl_id) < 0) + if(H5SM_get_info(&ext_loc, c_plist, meta_dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to read SOHM table information") /* Check for the extension having a 'v1 B-tree "K"' message */ - if((status = H5O_msg_exists(&ext_loc, H5O_BTREEK_ID, dxpl_id)) < 0) + if((status = H5O_msg_exists(&ext_loc, H5O_BTREEK_ID, meta_dxpl_id)) < 0) HGOTO_ERROR(H5E_FILE, H5E_EXISTS, FAIL, "unable to read object header") if(status) { /* Retrieve the 'v1 B-tree "K"' structure */ - if(NULL == H5O_msg_read(&ext_loc, H5O_BTREEK_ID, &btreek, dxpl_id)) + if(NULL == H5O_msg_read(&ext_loc, H5O_BTREEK_ID, &btreek, meta_dxpl_id)) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "v1 B-tree 'K' info message not present") /* Set non-default v1 B-tree 'K' value info from file */ @@ -661,39 +669,105 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) } /* end if */ /* Check for the extension having a 'free-space manager info' message */ - if((status = H5O_msg_exists(&ext_loc, H5O_FSINFO_ID, dxpl_id)) < 0) + if((status = H5O_msg_exists(&ext_loc, H5O_FSINFO_ID, meta_dxpl_id)) < 0) HGOTO_ERROR(H5E_FILE, H5E_EXISTS, FAIL, "unable to read object header") if(status) { - H5O_fsinfo_t fsinfo; /* Free-space manager info message from superblock extension */ + H5O_fsinfo_t fsinfo; /* File space info message from superblock extension */ + uint8_t flags; /* Message flags */ - /* Retrieve the 'free-space manager info' structure */ - if(NULL == H5O_msg_read(&ext_loc, H5O_FSINFO_ID, &fsinfo, dxpl_id)) - HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get free-space manager info message") + /* Get message flags */ + if(H5O_msg_get_flags(&ext_loc, H5O_FSINFO_ID, meta_dxpl_id, &flags) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to message flags for free-space manager info message") - /* Check for non-default info */ - if(f->shared->fs_strategy != fsinfo.strategy) { - f->shared->fs_strategy = fsinfo.strategy; + /* If message is NOT marked "unknown"--set up file space info */ + if(!(flags & H5O_MSG_FLAG_WAS_UNKNOWN)) { - /* Set non-default strategy in the property list */ - if(H5P_set(c_plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &fsinfo.strategy) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy") - } /* end if */ - if(f->shared->fs_threshold != fsinfo.threshold) { - f->shared->fs_threshold = fsinfo.threshold; + /* Retrieve the 'file space info' structure */ + if(NULL == H5O_msg_read(&ext_loc, H5O_FSINFO_ID, &fsinfo, meta_dxpl_id)) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get free-space manager info message") - /* Set non-default threshold in the property list */ - if(H5P_set(c_plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &fsinfo.threshold) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy") - } /* end if */ + /* Update changed values */ + if(f->shared->fs_strategy != fsinfo.strategy) { + f->shared->fs_strategy = fsinfo.strategy; + + /* Set non-default strategy in the property list */ + if(H5P_set(c_plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &fsinfo.strategy) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy") + } /* end if */ + if(f->shared->fs_persist != fsinfo.persist) { + f->shared->fs_persist = fsinfo.persist; + + /* Set non-default strategy in the property list */ + if(H5P_set(c_plist, H5F_CRT_FREE_SPACE_PERSIST_NAME, &fsinfo.persist) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy") + } /* end if */ + if(f->shared->fs_threshold != fsinfo.threshold) { + f->shared->fs_threshold = fsinfo.threshold; + + /* Set non-default threshold in the property list */ + if(H5P_set(c_plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &fsinfo.threshold) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space strategy") + } /* end if */ + + HDassert(f->shared->fs_page_size >= H5F_FILE_SPACE_PAGE_SIZE_MIN); + HDassert(fsinfo.page_size >= H5F_FILE_SPACE_PAGE_SIZE_MIN); + if(f->shared->fs_page_size != fsinfo.page_size) { + f->shared->fs_page_size = fsinfo.page_size; + + /* Set file space page size in the property list */ + if(H5P_set(c_plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, &fsinfo.page_size) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set file space page size") + } /* end if */ + if(f->shared->pgend_meta_thres != fsinfo.pgend_meta_thres) + /* Initialize page end meta threshold */ + f->shared->pgend_meta_thres = fsinfo.pgend_meta_thres; + + if(f->shared->eoa_pre_fsm_fsalloc != fsinfo.eoa_pre_fsm_fsalloc) + f->shared->eoa_pre_fsm_fsalloc = fsinfo.eoa_pre_fsm_fsalloc; + + /* f->shared->eoa_pre_fsm_fsalloc must always be HADDR_UNDEF + * in the absence of persistant free space managers. + */ + HDassert((!f->shared->fs_persist) || (f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF)); + HDassert(!f->shared->first_alloc_dealloc); + + if((f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF) && + (H5F_INTENT(f) & H5F_ACC_RDWR)) + f->shared->first_alloc_dealloc = TRUE; + + f->shared->fs_addr[0] = HADDR_UNDEF; + for(u = 1; u < NELMTS(f->shared->fs_addr); u++) + f->shared->fs_addr[u] = fsinfo.fs_addr[u - 1]; - /* Set free-space manager addresses */ - f->shared->fs_addr[0] = HADDR_UNDEF; - for(u = 1; u < NELMTS(f->shared->fs_addr); u++) - f->shared->fs_addr[u] = fsinfo.fs_addr[u-1]; + if(fsinfo.mapped && (rw_flags & H5AC__READ_ONLY_FLAG) == 0) { + + /* Do the same kluge until we know for sure. VC */ +#if 1 /* bug fix test code -- tidy this up if all goes well */ /* JRM */ + /* KLUGE ALERT!! + * + * H5F_super_ext_write_msg() expects f->shared->sblock to + * be set -- verify that it is NULL, and then set it. + * Set it back to NULL when we are done. + */ + HDassert(f->shared->sblock == NULL); + f->shared->sblock = sblock; +#endif /* JRM */ + + if(H5F_super_ext_remove_msg(f, meta_dxpl_id, H5O_FSINFO_ID) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDELETE, FAIL, "error in removing message from superblock extension") + + if(H5F_super_ext_write_msg(f, meta_dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing fsinfo message to superblock extension") +#if 1 /* bug fix test code -- tidy this up if all goes well */ /* JRM */ + f->shared->sblock = NULL; +#endif /* JRM */ + + } + } /* end if not marked "unknown" */ } /* end if */ /* Check for the extension having a 'metadata cache image' message */ - if((status = H5O_msg_exists(&ext_loc, H5O_MDCI_MSG_ID, dxpl_id)) < 0) + if((status = H5O_msg_exists(&ext_loc, H5O_MDCI_MSG_ID, meta_dxpl_id)) < 0) HGOTO_ERROR(H5E_FILE, H5E_EXISTS, FAIL, "unable to read object header") if(status) { hbool_t rw = ((rw_flags & H5AC__READ_ONLY_FLAG) == 0); @@ -712,7 +786,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) */ /* Retrieve the 'metadata cache image message' structure */ - if(NULL == H5O_msg_read(&ext_loc, H5O_MDCI_MSG_ID, &mdci_msg, dxpl_id)) + if(NULL == H5O_msg_read(&ext_loc, H5O_MDCI_MSG_ID, &mdci_msg, meta_dxpl_id)) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get metadata cache image message") /* Indicate to the cache that there's an image to load on first protect call */ @@ -721,7 +795,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) } /* end if */ /* Close superblock extension */ - if(H5F_super_ext_close(f, &ext_loc, dxpl_id, FALSE) < 0) + if(H5F_super_ext_close(f, &ext_loc, meta_dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension") } /* end if */ @@ -765,7 +839,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) HDassert(f->shared->sblock == NULL); f->shared->sblock = sblock; #endif /* JRM */ - if(H5F_super_ext_write_msg(f, dxpl_id, H5O_DRVINFO_ID, &drvinfo, FALSE, H5O_MSG_NO_FLAGS_SET) < 0) + if(H5F_super_ext_write_msg(f, meta_dxpl_id, H5O_DRVINFO_ID, &drvinfo, FALSE, H5O_MSG_NO_FLAGS_SET) < 0) HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension") #if 1 /* bug fix test code -- tidy this up if all goes well */ /* JRM */ @@ -777,7 +851,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) /* Check for eliminating the driver info block */ else if(H5F_HAS_FEATURE(f, H5FD_FEAT_IGNORE_DRVRINFO)) { /* Remove the driver info message from the superblock extension */ - if(H5F_super_ext_remove_msg(f, dxpl_id, H5O_DRVINFO_ID) < 0) + if(H5F_super_ext_remove_msg(f, meta_dxpl_id, H5O_DRVINFO_ID) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "error in removing message from superblock extension") /* Check if the superblock extension was removed */ @@ -789,13 +863,17 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) /* Set the pointer to the pinned superblock */ f->shared->sblock = sblock; + /* Set the page aggregation mode */ + if(H5F__set_paged_aggr(f, (hbool_t)H5F_PAGED_AGGR(f)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "failed to set paged_aggr status for file driver") + done: /* Reset the ring in the DXPL */ if(H5AC_reset_ring(dxpl, orig_ring) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set property value") /* Release the superblock */ - if(sblock && H5AC_unprotect(f, dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, sblock, sblock_flags) < 0) + if(sblock && H5AC_unprotect(f, meta_dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, sblock, sblock_flags) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTUNPROTECT, FAIL, "unable to close superblock") /* If we have failed, make sure no entries are left in the @@ -808,7 +886,7 @@ done: HDONE_ERROR(H5E_FILE, H5E_CANTUNPIN, FAIL, "unable to unpin driver info") /* Evict the driver info block from the cache */ - if(H5AC_expunge_entry(f, dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, H5AC__NO_FLAGS_SET) < 0) + if(H5AC_expunge_entry(f, meta_dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, H5AC__NO_FLAGS_SET) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTEXPUNGE, FAIL, "unable to expunge driver info block") } /* end if */ @@ -819,7 +897,7 @@ done: HDONE_ERROR(H5E_FILE, H5E_CANTUNPIN, FAIL, "unable to unpin superblock") /* Evict the superblock from the cache */ - if(H5AC_expunge_entry(f, dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, H5AC__NO_FLAGS_SET) < 0) + if(H5AC_expunge_entry(f, meta_dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, H5AC__NO_FLAGS_SET) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTEXPUNGE, FAIL, "unable to expunge superblock") } /* end if */ } /* end if */ @@ -890,7 +968,9 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) /* Check for non-default free-space settings */ if(!(f->shared->fs_strategy == H5F_FILE_SPACE_STRATEGY_DEF && - f->shared->fs_threshold == H5F_FREE_SPACE_THRESHOLD_DEF)) + f->shared->fs_persist == H5F_FREE_SPACE_PERSIST_DEF && + f->shared->fs_threshold == H5F_FREE_SPACE_THRESHOLD_DEF && + f->shared->fs_page_size == H5F_FILE_SPACE_PAGE_SIZE_DEF)) non_default_fs_settings = TRUE; /* Bump superblock version if latest superblock version support is enabled */ @@ -899,8 +979,12 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) /* Bump superblock version to create superblock extension for SOHM info */ else if(f->shared->sohm_nindexes > 0) super_vers = HDF5_SUPERBLOCK_VERSION_2; - /* Bump superblock version to create superblock extension for - * non-default file space strategy or non-default free-space threshold + /* + * Bump superblock version to create superblock extension for: + * -- non-default file space strategy or + * -- non-default persisting free-space or + * -- non-default free-space threshold or + * -- non-default page_size */ else if(non_default_fs_settings) super_vers = HDF5_SUPERBLOCK_VERSION_2; @@ -921,6 +1005,9 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set superblock version") } /* end if */ + if(H5FD_set_paged_aggr(f->shared->lf, (hbool_t)H5F_PAGED_AGGR(f)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "failed to set paged_aggr status for file driver") + /* * The superblock starts immediately after the user-defined * header, which we have already insured is a proper size. The @@ -932,9 +1019,12 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) /* Sanity check the userblock size vs. the file's allocation alignment */ if(userblock_size > 0) { - if(userblock_size < f->shared->alignment) + /* Set up the alignment to use for page or aggr fs */ + hsize_t alignment = H5F_PAGED_AGGR(f) ? f->shared->fs_page_size : f->shared->alignment; + + if(userblock_size < alignment) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "userblock size must be > file object alignment") - if(0 != (userblock_size % f->shared->alignment)) + if(0 != (userblock_size % alignment)) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "userblock size must be an integral multiple of file object alignment") } /* end if */ @@ -990,10 +1080,6 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) if(super_vers < HDF5_SUPERBLOCK_VERSION_2) superblock_size += driver_size; - /* Reserve space in the file for the superblock, instead of allocating it */ - if(H5F__set_eoa(f, H5FD_MEM_SUPER, superblock_size) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to set EOA value for superblock") - /* Set the ring type in the DXPL */ if(H5AC_set_ring(dxpl_id, H5AC_RING_SB, &dxpl, &orig_ring) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set ring value") @@ -1006,6 +1092,10 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) /* Keep a copy of the superblock info */ f->shared->sblock = sblock; + /* Allocate space for the superblock */ + if(HADDR_UNDEF == H5MF_alloc(f, H5FD_MEM_SUPER, dxpl_id, superblock_size)) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "file allocation failed for superblock") + /* set the drvinfo filed to NULL -- will overwrite this later if needed */ f->shared->drvinfo = NULL; @@ -1108,17 +1198,23 @@ H5F__super_init(H5F_t *f, hid_t dxpl_id) } /* end if */ /* Check for non-default free-space info settings */ - if(non_default_fs_settings) { - H5FD_mem_t type; /* Memory type for iteration */ - H5O_fsinfo_t fsinfo; /* Free space manager info message */ - - /* Write free-space manager info message to superblock extension object header if needed */ - fsinfo.strategy = f->shared->fs_strategy; - fsinfo.threshold = f->shared->fs_threshold; - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - fsinfo.fs_addr[type-1] = HADDR_UNDEF; - - if(H5O_msg_create(&ext_loc, H5O_FSINFO_ID, H5O_MSG_FLAG_DONTSHARE, H5O_UPDATE_TIME, &fsinfo, dxpl_id) < 0) + if(non_default_fs_settings) { + H5F_mem_page_t ptype; + H5O_fsinfo_t fsinfo; /* Free space manager info message */ + + /* Write free-space manager info message to superblock extension object header if needed */ + fsinfo.strategy = f->shared->fs_strategy; + fsinfo.persist = f->shared->fs_persist; + fsinfo.threshold = f->shared->fs_threshold; + fsinfo.page_size = f->shared->fs_page_size; + fsinfo.pgend_meta_thres = f->shared->pgend_meta_thres; + fsinfo.eoa_pre_fsm_fsalloc = HADDR_UNDEF; + fsinfo.mapped = FALSE; + + for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + fsinfo.fs_addr[ptype - 1] = HADDR_UNDEF; + + if(H5O_msg_create(&ext_loc, H5O_FSINFO_ID, H5O_MSG_FLAG_DONTSHARE | H5O_MSG_FLAG_MARK_IF_UNKNOWN, H5O_UPDATE_TIME, &fsinfo, dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to update free-space info header message") } /* end if */ } /* end if */ diff --git a/src/H5HFsection.c b/src/H5HFsection.c index 37ff8f4..b113ca1 100644 --- a/src/H5HFsection.c +++ b/src/H5HFsection.c @@ -85,14 +85,14 @@ static herr_t H5HF_sect_single_full_dblock(H5HF_hdr_t *hdr, hid_t dxpl_id, H5HF_free_section_t *sect); /* 'single' section callbacks */ -static herr_t H5HF_sect_single_add(H5FS_section_info_t *sect, unsigned *flags, +static herr_t H5HF_sect_single_add(H5FS_section_info_t **sect, unsigned *flags, void *udata); static H5FS_section_info_t *H5HF_sect_single_deserialize(const H5FS_section_class_t *cls, hid_t dxpl_id, const uint8_t *buf, haddr_t sect_addr, hsize_t sect_size, unsigned *des_flags); static htri_t H5HF_sect_single_can_merge(const H5FS_section_info_t *sect1, const H5FS_section_info_t *sect2, void *udata); -static herr_t H5HF_sect_single_merge(H5FS_section_info_t *sect1, +static herr_t H5HF_sect_single_merge(H5FS_section_info_t **sect1, H5FS_section_info_t *sect2, void *udata); static htri_t H5HF_sect_single_can_shrink(const H5FS_section_info_t *sect, void *udata); @@ -121,7 +121,7 @@ static H5FS_section_info_t *H5HF_sect_row_deserialize(const H5FS_section_class_t unsigned *des_flags); static htri_t H5HF_sect_row_can_merge(const H5FS_section_info_t *sect1, const H5FS_section_info_t *sect2, void *udata); -static herr_t H5HF_sect_row_merge(H5FS_section_info_t *sect1, +static herr_t H5HF_sect_row_merge(H5FS_section_info_t **sect1, H5FS_section_info_t *sect2, void *udata); static htri_t H5HF_sect_row_can_shrink(const H5FS_section_info_t *sect, void *udata); @@ -811,7 +811,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF_sect_single_add(H5FS_section_info_t *_sect, unsigned *flags, void *_udata) +H5HF_sect_single_add(H5FS_section_info_t **_sect, unsigned *flags, void *_udata) { herr_t ret_value = SUCCEED; /* Return value */ @@ -821,7 +821,7 @@ H5HF_sect_single_add(H5FS_section_info_t *_sect, unsigned *flags, void *_udata) * have already been checked when it was first added */ if(!(*flags & H5FS_ADD_DESERIALIZING)) { - H5HF_free_section_t *sect = (H5HF_free_section_t *)_sect; /* Fractal heap free section */ + H5HF_free_section_t **sect = (H5HF_free_section_t **)_sect; /* Fractal heap free section */ H5HF_sect_add_ud_t *udata = (H5HF_sect_add_ud_t *)_udata; /* User callback data */ H5HF_hdr_t *hdr = udata->hdr; /* Fractal heap header */ hid_t dxpl_id = udata->dxpl_id; /* DXPL ID for operation */ @@ -832,14 +832,14 @@ H5HF_sect_single_add(H5FS_section_info_t *_sect, unsigned *flags, void *_udata) /* Check if single section covers entire direct block it's in */ /* (converts to row section possibly) */ - if(H5HF_sect_single_full_dblock(hdr, dxpl_id, sect) < 0) + if(H5HF_sect_single_full_dblock(hdr, dxpl_id, (*sect)) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTCONVERT, FAIL, "can't check/convert single section") /* Set the "returned space" flag if the single section was changed * into a row section, so the "merging & shrinking" algorithm * gets executed in the free space manager */ - if(sect->sect_info.type != H5HF_FSPACE_SECT_SINGLE) + if((*sect)->sect_info.type != H5HF_FSPACE_SECT_SINGLE) *flags |= H5FS_ADD_RETURNED_SPACE; } /* end if */ @@ -949,10 +949,10 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF_sect_single_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, +H5HF_sect_single_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2, void *_udata) { - H5HF_free_section_t *sect1 = (H5HF_free_section_t *)_sect1; /* Fractal heap free section */ + H5HF_free_section_t **sect1 = (H5HF_free_section_t **)_sect1; /* Fractal heap free section */ H5HF_free_section_t *sect2 = (H5HF_free_section_t *)_sect2; /* Fractal heap free section */ H5HF_sect_add_ud_t *udata = (H5HF_sect_add_ud_t *)_udata; /* User callback data */ H5HF_hdr_t *hdr = udata->hdr; /* Fractal heap header */ @@ -963,26 +963,26 @@ H5HF_sect_single_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, /* Check arguments. */ HDassert(sect1); - HDassert(sect1->sect_info.type == H5HF_FSPACE_SECT_SINGLE); + HDassert((*sect1)->sect_info.type == H5HF_FSPACE_SECT_SINGLE); HDassert(sect2); HDassert(sect2->sect_info.type == H5HF_FSPACE_SECT_SINGLE); - HDassert(H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr)); + HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr)); /* Add second section's size to first section */ - sect1->sect_info.size += sect2->sect_info.size; + (*sect1)->sect_info.size += sect2->sect_info.size; /* Get rid of second section */ if(H5HF_sect_single_free((H5FS_section_info_t *)sect2) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTRELEASE, FAIL, "can't free section node") /* Check to see if we should revive first section */ - if(sect1->sect_info.state != H5FS_SECT_LIVE) - if(H5HF_sect_single_revive(hdr, dxpl_id, sect1) < 0) + if((*sect1)->sect_info.state != H5FS_SECT_LIVE) + if(H5HF_sect_single_revive(hdr, dxpl_id, (*sect1)) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, FAIL, "can't revive single free section") /* Check if single section covers entire direct block it's in */ /* (converts to row section possibly) */ - if(H5HF_sect_single_full_dblock(hdr, dxpl_id, sect1) < 0) + if(H5HF_sect_single_full_dblock(hdr, dxpl_id, (*sect1)) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTCONVERT, FAIL, "can't check/convert single section") done: @@ -1771,10 +1771,10 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF_sect_row_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, +H5HF_sect_row_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2, void *_udata) { - H5HF_free_section_t *sect1 = (H5HF_free_section_t *)_sect1; /* Fractal heap free section */ + H5HF_free_section_t **sect1 = (H5HF_free_section_t **)_sect1; /* Fractal heap free section */ H5HF_free_section_t *sect2 = (H5HF_free_section_t *)_sect2; /* Fractal heap free section */ H5HF_sect_add_ud_t *udata = (H5HF_sect_add_ud_t *)_udata; /* User callback data */ H5HF_hdr_t *hdr = udata->hdr; /* Fractal heap header */ @@ -1785,7 +1785,7 @@ H5HF_sect_row_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, /* Check arguments. */ HDassert(sect1); - HDassert(sect1->sect_info.type == H5HF_FSPACE_SECT_FIRST_ROW); + HDassert((*sect1)->sect_info.type == H5HF_FSPACE_SECT_FIRST_ROW); HDassert(sect2); HDassert(sect2->sect_info.type == H5HF_FSPACE_SECT_FIRST_ROW); @@ -1802,8 +1802,8 @@ H5HF_sect_row_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, } /* end if */ else { /* Check to see if we should revive first section */ - if(sect1->sect_info.state != H5FS_SECT_LIVE) - if(H5HF_sect_row_revive(hdr, dxpl_id, sect1) < 0) + if((*sect1)->sect_info.state != H5FS_SECT_LIVE) + if(H5HF_sect_row_revive(hdr, dxpl_id, (*sect1)) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, FAIL, "can't revive single free section") /* Check to see if we should revive second section */ @@ -1812,7 +1812,7 @@ H5HF_sect_row_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, FAIL, "can't revive single free section") /* Merge rows' underlying indirect sections together */ - if(H5HF_sect_indirect_merge_row(hdr, dxpl_id, sect1, sect2) < 0) + if(H5HF_sect_indirect_merge_row(hdr, dxpl_id, (*sect1), sect2) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTMERGE, FAIL, "can't merge underlying indirect sections") } /* end else */ @@ -29,6 +29,7 @@ /****************/ #define H5F_FRIEND /*suppress error about including H5Fpkg */ +#define H5FS_FRIEND /*suppress error about including H5Fpkg */ #include "H5MFmodule.h" /* This source code file is part of the H5MF module */ @@ -38,6 +39,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* File access */ +#include "H5FSpkg.h" /* File access */ #include "H5Iprivate.h" /* IDs */ #include "H5MFpkg.h" /* File memory management */ #include "H5VMprivate.h" /* Vectors and arrays */ @@ -50,11 +52,6 @@ #define H5MF_FSPACE_SHRINK 80 /* Percent of "normal" size to shrink serialized free space size */ #define H5MF_FSPACE_EXPAND 120 /* Percent of "normal" size to expand serialized free space size */ -/* Map an allocation request type to a free list */ -#define H5MF_ALLOC_TO_FS_TYPE(F, T) ((H5FD_MEM_DEFAULT == (F)->shared->fs_type_map[T]) \ - ? (T) : (F)->shared->fs_type_map[T]) - - /******************/ /* Local Typedefs */ /******************/ @@ -84,11 +81,24 @@ typedef struct { /********************/ /* Allocator routines */ -static herr_t H5MF_alloc_create(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type); -static herr_t H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type); -static herr_t H5MF__close_delete(H5F_t *f, hid_t dxpl_id, H5P_genplist_t **dxpl); +static haddr_t H5MF__alloc_pagefs(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size); + +/* "File closing" routines */ +static herr_t H5MF__close_aggrfs(H5F_t *f, hid_t dxpl_id); +static herr_t H5MF__close_pagefs(H5F_t *f, hid_t dxpl_id); static herr_t H5MF__close_shrink_eoa(H5F_t *f, hid_t dxpl_id); +/* General routines */ +static herr_t H5MF__get_free_sects(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5MF_sect_iter_ud_t *sect_udata, size_t *nums); +static hbool_t H5MF__fsm_type_is_self_referential(H5F_t *f, H5F_mem_page_t fsm_type); +static hbool_t H5MF__fsm_is_self_referential(H5F_t *f, H5FS_t *fspace); + +/* Free-space type manager routines */ +static herr_t H5MF__create_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type); +static herr_t H5MF__close_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type); +static herr_t H5MF__delete_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type); +static herr_t H5MF__close_delete_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type); + /*********************/ /* Package Variables */ @@ -223,10 +233,54 @@ done: /*------------------------------------------------------------------------- - * Function: H5MF__alloc_open + * Function: H5MF_alloc_to_fs_type + * + * Purpose: Map "alloc_type" to the free-space manager type + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi; Nov 2016 + * + *------------------------------------------------------------------------- + */ +void +H5MF_alloc_to_fs_type(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size, H5F_mem_page_t *fs_type) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(fs_type); + + if(H5F_PAGED_AGGR(f)) { /* paged aggregation */ + if(size >= f->shared->fs_page_size) { + if(H5F_HAS_FEATURE(f, H5FD_FEAT_PAGED_AGGR)) { /* multi or split driver */ + /* For non-contiguous address space, map to large size free-space manager for each alloc_type */ + if(H5FD_MEM_DEFAULT == f->shared->fs_type_map[alloc_type]) + *fs_type = (H5F_mem_page_t) (alloc_type + (H5FD_MEM_NTYPES - 1)); + else + *fs_type = (H5F_mem_page_t) (f->shared->fs_type_map[alloc_type] + (H5FD_MEM_NTYPES - 1)); + } /* end if */ + else + /* For contiguous address space, map to generic large size free-space manager */ + *fs_type = H5F_MEM_PAGE_GENERIC; /* H5F_MEM_PAGE_SUPER */ + } /* end if */ + else + *fs_type = (H5F_mem_page_t)H5MF_ALLOC_TO_FS_AGGR_TYPE(f, alloc_type); + } /* end if */ + else /* non-paged aggregation */ + *fs_type = (H5F_mem_page_t)H5MF_ALLOC_TO_FS_AGGR_TYPE(f, alloc_type); + + FUNC_LEAVE_NOAPI_VOID +} /* end H5MF_alloc_to_fs_type() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_open_fstype * * Purpose: Open an existing free space manager of TYPE for file by - * creating a free-space structure + * creating a free-space structure. + * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types. * * Return: Success: non-negative * Failure: negative @@ -238,31 +292,48 @@ done: *------------------------------------------------------------------------- */ herr_t -H5MF__alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) +H5MF_open_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type) { const H5FS_section_class_t *classes[] = { /* Free space section classes implemented for file */ - H5MF_FSPACE_SECT_CLS_SIMPLE}; - H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t fsm_ring; /* Free space manager ring */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - hbool_t reset_ring = FALSE; /* Whether the ring was set */ - herr_t ret_value = SUCCEED; /* Return value */ + H5MF_FSPACE_SECT_CLS_SIMPLE, + H5MF_FSPACE_SECT_CLS_SMALL, + H5MF_FSPACE_SECT_CLS_LARGE }; + hsize_t alignment; /* Alignment to use */ + hsize_t threshold; /* Threshold to use */ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) /* * Check arguments. */ HDassert(f); + if(H5F_PAGED_AGGR(f)) + HDassert(type < H5F_MEM_PAGE_NTYPES); + else { + HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); + HDassert((H5FD_mem_t)type != H5FD_MEM_NOLIST); + } /* end else */ HDassert(f->shared); - HDassert(type != H5FD_MEM_NOLIST); HDassert(H5F_addr_defined(f->shared->fs_addr[type])); HDassert(f->shared->fs_state[type] == H5F_FS_STATE_CLOSED); - HDassert(type == H5MF_ALLOC_TO_FS_TYPE(f, type)); + + /* Set up the aligment and threshold to use depending on the manager type */ + if(H5F_PAGED_AGGR(f)) { + alignment = (type == H5F_MEM_PAGE_GENERIC) ? f->shared->fs_page_size : (hsize_t)H5F_ALIGN_DEF; + threshold = H5F_ALIGN_THRHD_DEF; + } /* end if */ + else { + alignment = f->shared->alignment; + threshold = f->shared->threshold; + } /* end else */ /* Set the ring type in the DXPL */ - if((type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) + if(H5MF__fsm_type_is_self_referential(f, type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -272,7 +343,7 @@ H5MF__alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) /* Open an existing free space structure for the file */ if(NULL == (f->shared->fs_man[type] = H5FS_open(f, dxpl_id, f->shared->fs_addr[type], - NELMTS(classes), classes, f, f->shared->alignment, f->shared->threshold))) + NELMTS(classes), classes, f, alignment, threshold))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space info") /* Set the state for the free space manager to "open", if it is now */ @@ -286,14 +357,15 @@ done: HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* end H5MF__alloc_open() */ +} /* end H5MF_open_fstype() */ /*------------------------------------------------------------------------- - * Function: H5MF_alloc_create + * Function: H5MF__create_fstype * * Purpose: Create free space manager of TYPE for the file by creating - * a free-space structure + * a free-space structure + * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types. * * Return: Success: non-negative * Failure: negative @@ -305,21 +377,34 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5MF_alloc_create(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) +H5MF__create_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type) { const H5FS_section_class_t *classes[] = { /* Free space section classes implemented for file */ - H5MF_FSPACE_SECT_CLS_SIMPLE}; - herr_t ret_value = SUCCEED; /* Return value */ - H5FS_create_t fs_create; /* Free space creation parameters */ + H5MF_FSPACE_SECT_CLS_SIMPLE, + H5MF_FSPACE_SECT_CLS_SMALL, + H5MF_FSPACE_SECT_CLS_LARGE }; + H5FS_create_t fs_create; /* Free space creation parameters */ + hsize_t alignment; /* Alignment to use */ + hsize_t threshold; /* Threshold to use */ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) /* * Check arguments. */ HDassert(f); + if(H5F_PAGED_AGGR(f)) + HDassert(type < H5F_MEM_PAGE_NTYPES); + else { + HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); + HDassert((H5FD_mem_t)type != H5FD_MEM_NOLIST); + } /* end else */ HDassert(f->shared); - HDassert(type != H5FD_MEM_NOLIST); HDassert(!H5F_addr_defined(f->shared->fs_addr[type])); HDassert(f->shared->fs_state[type] == H5F_FS_STATE_CLOSED); @@ -330,24 +415,48 @@ H5MF_alloc_create(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) fs_create.max_sect_addr = 1 + H5VM_log2_gen((uint64_t)f->shared->maxaddr); fs_create.max_sect_size = f->shared->maxaddr; - if(NULL == (f->shared->fs_man[type] = H5FS_create(f, dxpl_id, NULL, - &fs_create, NELMTS(classes), classes, f, f->shared->alignment, f->shared->threshold))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space info") + /* Set up alignment and threshold to use depending on TYPE */ + if(H5F_PAGED_AGGR(f)) { + alignment = (type == H5F_MEM_PAGE_GENERIC) ? f->shared->fs_page_size : (hsize_t)H5F_ALIGN_DEF; + threshold = H5F_ALIGN_THRHD_DEF; + } /* end if */ + else { + alignment = f->shared->alignment; + threshold = f->shared->threshold; + } /* end else */ + /* Set the ring type in the DXPL */ + if(H5MF__fsm_type_is_self_referential(f, type)) + fsm_ring = H5AC_RING_MDFSM; + else + fsm_ring = H5AC_RING_RDFSM; + if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + + if(NULL == (f->shared->fs_man[type] = H5FS_create(f, dxpl_id, NULL, + &fs_create, NELMTS(classes), classes, f, alignment, threshold))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space info") /* Set the state for the free space manager to "open", if it is now */ if(f->shared->fs_man[type]) f->shared->fs_state[type] = H5F_FS_STATE_OPEN; done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* end H5MF_alloc_create() */ +} /* end H5MF__create_fstype() */ /*------------------------------------------------------------------------- - * Function: H5MF__alloc_start + * Function: H5MF_start_fstype * - * Purpose: Open or create a free space manager of a given type + * Purpose: Open or create a free space manager of a given TYPE. + * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types. * * Return: Success: non-negative * Failure: negative @@ -359,40 +468,123 @@ done: *------------------------------------------------------------------------- */ herr_t -H5MF__alloc_start(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) +H5MF_start_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) /* * Check arguments. */ HDassert(f); HDassert(f->shared); - HDassert(type != H5FD_MEM_NOLIST); + if(H5F_PAGED_AGGR(f)) + HDassert(type < H5F_MEM_PAGE_NTYPES); + else { + HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); + HDassert((H5FD_mem_t)type != H5FD_MEM_NOLIST); + } /* end else */ /* Check if the free space manager exists already */ if(H5F_addr_defined(f->shared->fs_addr[type])) { /* Open existing free space manager */ - if(H5MF__alloc_open(f, dxpl_id, type) < 0) + if(H5MF_open_fstype(f, dxpl_id, type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTOPENOBJ, FAIL, "can't initialize file free space") } /* end if */ else { /* Create new free space manager */ - if(H5MF_alloc_create(f, dxpl_id, type) < 0) + if(H5MF__create_fstype(f, dxpl_id, type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCREATE, FAIL, "can't initialize file free space") } /* end else */ done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5MF__alloc_start() */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF_start_fstype() */ /*------------------------------------------------------------------------- - * Function: H5MF__alloc_close + * Function: H5MF__delete_fstype * - * Purpose: Close an existing free space manager of TYPE for file + * Purpose: Delete the free-space manager as specified by TYPE. + * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi; April 2013 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF__delete_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + haddr_t tmp_fs_addr; /* Temporary holder for free space manager address */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + + /* check args */ + HDassert(f); + if(H5F_PAGED_AGGR(f)) + HDassert(type < H5F_MEM_PAGE_NTYPES); + else + HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); + HDassert(H5F_addr_defined(f->shared->fs_addr[type])); + + /* Put address into temporary variable and reset it */ + /* (Avoids loopback in file space freeing routine) */ + tmp_fs_addr = f->shared->fs_addr[type]; + f->shared->fs_addr[type] = HADDR_UNDEF; + + /* Shift to "deleting" state, to make certain we don't track any + * file space freed as a result of deleting the free space manager. + */ + f->shared->fs_state[type] = H5F_FS_STATE_DELETING; + + /* Set the ring type in the DXPL */ + if(H5MF__fsm_type_is_self_referential(f, type)) + fsm_ring = H5AC_RING_MDFSM; + else + fsm_ring = H5AC_RING_RDFSM; + if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Before deleting free space manager\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Delete free space manager for this type */ + if(H5FS_delete(f, dxpl_id, tmp_fs_addr) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't delete free space manager") + + /* Shift [back] to closed state */ + HDassert(f->shared->fs_state[type] == H5F_FS_STATE_DELETING); + f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; + + /* Sanity check that the free space manager for this type wasn't started up again */ + HDassert(!H5F_addr_defined(f->shared->fs_addr[type])); + +done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF__delete_fstype() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF__close_fstype + * + * Purpose: Close the free space manager of TYPE for file + * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types. * * Return: Success: non-negative * Failure: negative @@ -402,7 +594,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) +H5MF__close_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type) { herr_t ret_value = SUCCEED; /* Return value */ @@ -412,11 +604,18 @@ H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) * Check arguments. */ HDassert(f); + if(H5F_PAGED_AGGR(f)) + HDassert(type < H5F_MEM_PAGE_NTYPES); + else + HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); HDassert(f->shared); - HDassert(type != H5FD_MEM_NOLIST); HDassert(f->shared->fs_man[type]); HDassert(f->shared->fs_state[type] != H5F_FS_STATE_CLOSED); +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Before closing free space manager\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + /* Close an existing free space structure for the file */ if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free space info") @@ -425,7 +624,158 @@ H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) done: FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* end H5MF__alloc_close() */ +} /* end H5MF__close_fstype() */ + + + +/*------------------------------------------------------------------------- + * Function: H5MF_add_sect + * + * Purpose: To add a section to the specified free-space manager. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi; April 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5MF_add_sect(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, H5FS_t *fspace, H5MF_free_section_t *node) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */ + H5MF_sect_ud_t udata; /* User data for callback */ + H5F_mem_page_t fs_type; /* Free space type (mapped from allocation type) */ + + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + + HDassert(f); + HDassert(fspace); + HDassert(node); + + H5MF_alloc_to_fs_type(f, alloc_type, node->sect_info.size, &fs_type); + + /* Construct user data for callbacks */ + udata.f = f; + udata.dxpl_id = dxpl_id; + udata.alloc_type = alloc_type; + udata.allow_sect_absorb = TRUE; + udata.allow_eoa_shrink_only = FALSE; + + /* Set the ring type in the DXPL */ + if(H5MF__fsm_is_self_referential(f, fspace)) + fsm_ring = H5AC_RING_MDFSM; + else + fsm_ring = H5AC_RING_RDFSM; + if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: adding node, node->sect_info.addr = %a, node->sect_info.size = %Hu\n", FUNC, node->sect_info.addr, node->sect_info.size); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + /* Add the section */ + if(H5FS_sect_add(f, dxpl_id, fspace, (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, &udata) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't re-add section to file free space") + +done: + /* Reset the ring in the DXPL */ + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF_add_sect() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_find_sect + * + * Purpose: To find a section from the specified free-space manager to fulfill the request. + * If found, re-add the left-over space back to the manager. + * + * Return: TRUE if a section is found to fulfill the request + * FALSE if not + * + * Programmer: Vailin Choi; April 2013 + * + *------------------------------------------------------------------------- + */ +htri_t +H5MF_find_sect(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size, H5FS_t *fspace, haddr_t *addr) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */ + H5MF_free_section_t *node; /* Free space section pointer */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + htri_t ret_value = FAIL; /* Whether an existing free list node was found */ + + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + + HDassert(f); + HDassert(fspace); + + /* Set the ring type in the DXPL */ + if(H5MF__fsm_is_self_referential(f, fspace)) + fsm_ring = H5AC_RING_MDFSM; + else + fsm_ring = H5AC_RING_RDFSM; + if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + + /* Try to get a section from the free space manager */ + if((ret_value = H5FS_sect_find(f, dxpl_id, fspace, size, (H5FS_section_info_t **)&node)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "error locating free space in file") + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: section found = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Check for actually finding section */ + if(ret_value) { + /* Sanity check */ + HDassert(node); + + /* Retrieve return value */ + if(addr) + *addr = node->sect_info.addr; + + /* Check for eliminating the section */ + if(node->sect_info.size == size) { +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: freeing node\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Free section node */ + if(H5MF_sect_free((H5FS_section_info_t *)node) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") + } /* end if */ + else { + /* Adjust information for section */ + node->sect_info.addr += size; + node->sect_info.size -= size; + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: re-adding node, node->sect_info.size = %Hu\n", FUNC, node->sect_info.size); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Re-add the section to the free-space manager */ + if(H5MF_add_sect(f, alloc_type, dxpl_id, fspace, node) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't re-add section to file free space") + } /* end else */ + } /* end if */ + +done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF_find_sect() */ /*------------------------------------------------------------------------- @@ -451,7 +801,7 @@ H5MF_alloc(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size) H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ H5AC_ring_t fsm_ring = H5AC_RING_INV; /* free space manager ring */ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - H5FD_mem_t fs_type; /* Free space type (mapped from allocation type) */ + H5F_mem_page_t fs_type; /* Free space type (mapped from allocation type) */ hbool_t reset_ring = FALSE; /* Whether the ring was set */ haddr_t ret_value = HADDR_UNDEF; /* Return value */ @@ -466,12 +816,20 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ HDassert(f->shared->lf); HDassert(size > 0); - /* Get free space type from allocation type */ - fs_type = H5MF_ALLOC_TO_FS_TYPE(f, alloc_type); + if(f->shared->first_alloc_dealloc) { + HDassert(! H5AC_cache_image_pending(f)); + if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "tidy of self referential fsm hack failed") + } /* end if */ + + H5MF_alloc_to_fs_type(f, alloc_type, size, &fs_type); + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Check 1.0\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG_MORE */ /* Set the ring type in the DXPL */ - if((fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) + if(H5MF__fsm_type_is_self_referential(f, fs_type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -481,74 +839,46 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ /* Check if we are using the free space manager for this file */ if(H5F_HAVE_FREE_SPACE_MANAGER(f)) { + /* We are about to change the contents of the free space manager -- + * notify metadata cache that the associated fsm ring is + * unsettled + */ + if(H5AC_unsettle_ring(f, fsm_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_SYSTEM, HADDR_UNDEF, "attempt to notify cache that ring is unsettled failed") + /* Check if the free space manager for the file has been initialized */ - if(!f->shared->fs_man[fs_type] && H5F_addr_defined(f->shared->fs_addr[fs_type])) - if(H5MF__alloc_open(f, dxpl_id, fs_type) < 0) + if(!f->shared->fs_man[fs_type] && H5F_addr_defined(f->shared->fs_addr[fs_type])) { + /* Open the free-space manager */ + if(H5MF_open_fstype(f, dxpl_id, fs_type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTOPENOBJ, HADDR_UNDEF, "can't initialize file free space") + HDassert(f->shared->fs_man[fs_type]); + } /* end if */ /* Search for large enough space in the free space manager */ - if(f->shared->fs_man[fs_type]) { - H5MF_free_section_t *node; /* Free space section pointer */ - htri_t node_found = FALSE; /* Whether an existing free list node was found */ - - /* Try to get a section from the free space manager */ - if((node_found = H5FS_sect_find(f, dxpl_id, f->shared->fs_man[fs_type], size, (H5FS_section_info_t **)&node)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "error locating free space in file") -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 1.5, node_found = %t\n", FUNC, node_found); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - - /* Check for actually finding section */ - if(node_found) { - /* Sanity check */ - HDassert(node); - - /* Retrieve return value */ - ret_value = node->sect_info.addr; - - /* Check for eliminating the section */ - if(node->sect_info.size == size) { -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 1.6, freeing node\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - /* Free section node */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, HADDR_UNDEF, "can't free simple section node") - } /* end if */ - else { - H5MF_sect_ud_t udata; /* User data for callback */ - - /* Adjust information for section */ - node->sect_info.addr += size; - node->sect_info.size -= size; - - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = dxpl_id; - udata.alloc_type = alloc_type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; + if(f->shared->fs_man[fs_type]) + if(H5MF_find_sect(f, alloc_type, dxpl_id, size, f->shared->fs_man[fs_type], &ret_value) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "error locating a node") + } /* end if */ + /* If no space is found from the free-space manager, continue further action */ + if(!H5F_addr_defined(ret_value)) { #ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 1.7, re-adding node, node->sect_info.size = %Hu\n", FUNC, node->sect_info.size); +HDfprintf(stderr, "%s: Check 2.0\n", FUNC); #endif /* H5MF_ALLOC_DEBUG_MORE */ - /* Re-insert section node into file's free space */ - if(H5FS_sect_add(f, dxpl_id, f->shared->fs_man[fs_type], (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, &udata) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, HADDR_UNDEF, "can't re-add section to file free space") - } /* end else */ - - /* Leave now */ - HGOTO_DONE(ret_value) - } /* end if */ + if(f->shared->fs_strategy == H5F_FSPACE_STRATEGY_PAGE) { + HDassert(f->shared->fs_page_size >= H5F_FILE_SPACE_PAGE_SIZE_MIN); + if(HADDR_UNDEF == (ret_value = H5MF__alloc_pagefs(f, alloc_type, dxpl_id, size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "allocation failed from paged aggregation") } /* end if */ + else { /* For non-paged aggregation, continue further action */ + if(HADDR_UNDEF == (ret_value = H5MF_aggr_vfd_alloc(f, alloc_type, dxpl_id, size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "allocation failed from aggr/vfd") + } /* end else */ + } /* end if */ + HDassert(H5F_addr_defined(ret_value)); #ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 2.0\n", FUNC); +HDfprintf(stderr, "%s: Check 3.0\n", FUNC); #endif /* H5MF_ALLOC_DEBUG_MORE */ - } /* end if */ - - /* Allocate from the metadata aggregator (or the VFD) */ - if(HADDR_UNDEF == (ret_value = H5MF_aggr_vfd_alloc(f, alloc_type, dxpl_id, size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "allocation failed from aggr/vfd") done: /* Reset the ring in the DXPL */ @@ -568,6 +898,146 @@ H5MF_sects_dump(f, dxpl_id, stderr); /*------------------------------------------------------------------------- + * Function: H5MF__alloc_pagefs + * + * Purpose: Allocate space from either the large or small free-space manager. + * For "large" request: + * Allocate request from VFD + * Determine mis-aligned fragment and return the fragment to the + * appropriate manager + * For "small" request: + * Allocate a page from the large manager + * Determine whether space is available from a mis-aligned fragment + * being returned to the manager + * Return left-over space to the manager after fulfilling request + * + * Return: Success: The file address of new chunk. + * Failure: HADDR_UNDEF + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static haddr_t +H5MF__alloc_pagefs(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size) +{ + H5F_mem_page_t ptype; /* Free-space mananger type */ + H5MF_free_section_t *node = NULL; /* Free space section pointer */ + haddr_t ret_value = HADDR_UNDEF; /* Return value */ + + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, HADDR_UNDEF) + +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_type, size); +#endif /* H5MF_ALLOC_DEBUG */ + + H5MF_alloc_to_fs_type(f, alloc_type, size, &ptype); + + switch(ptype) { + case H5F_MEM_PAGE_GENERIC: + case H5F_MEM_PAGE_LARGE_BTREE: + case H5F_MEM_PAGE_LARGE_DRAW: + case H5F_MEM_PAGE_LARGE_GHEAP: + case H5F_MEM_PAGE_LARGE_LHEAP: + case H5F_MEM_PAGE_LARGE_OHDR: + { + haddr_t eoa; /* EOA for the file */ + hsize_t frag_size = 0; /* Fragment size */ + + /* Get the EOA for the file */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, alloc_type))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, HADDR_UNDEF, "Unable to get eoa") + HDassert(!(eoa % f->shared->fs_page_size)); + + H5MF_EOA_MISALIGN(f, (eoa+size), f->shared->fs_page_size, frag_size); + + /* Allocate from VFD */ + if(HADDR_UNDEF == (ret_value = H5F_alloc(f, dxpl_id, alloc_type, size + frag_size, NULL, NULL))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space") + + /* If there is a mis-aligned fragment at EOA */ + if(frag_size) { + + /* Start up the free-space manager */ + if(!(f->shared->fs_man[ptype])) + if(H5MF_start_fstype(f, dxpl_id, ptype) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, HADDR_UNDEF, "can't initialize file free space") + + /* Create free space section for the fragment */ + if(NULL == (node = H5MF_sect_new(H5MF_FSPACE_SECT_LARGE, ret_value + size, frag_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, HADDR_UNDEF, "can't initialize free space section") + + /* Add the fragment to the large free-space manager */ + if(H5MF_add_sect(f, alloc_type, dxpl_id, f->shared->fs_man[ptype], node) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, HADDR_UNDEF, "can't re-add section to file free space") + + node = NULL; + } /* end if */ + } + break; + + case H5F_MEM_PAGE_META: + case H5F_MEM_PAGE_DRAW: + case H5F_MEM_PAGE_BTREE: + case H5F_MEM_PAGE_GHEAP: + case H5F_MEM_PAGE_LHEAP: + case H5F_MEM_PAGE_OHDR: + { + haddr_t new_page; /* The address for the new file size page */ + + /* Allocate one file space page */ + if(HADDR_UNDEF == (new_page = H5MF_alloc(f, alloc_type, dxpl_id, f->shared->fs_page_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space") + + /* Start up the free-space manager */ + if(!(f->shared->fs_man[ptype])) + if(H5MF_start_fstype(f, dxpl_id, ptype) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, HADDR_UNDEF, "can't initialize file free space") + HDassert(f->shared->fs_man[ptype]); + + if(NULL == (node = H5MF_sect_new(H5MF_FSPACE_SECT_SMALL, (new_page + size), (f->shared->fs_page_size - size)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, HADDR_UNDEF, "can't initialize free space section") + + /* Add the remaining space in the page to the manager */ + if(H5MF_add_sect(f, alloc_type, dxpl_id, f->shared->fs_man[ptype], node) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, HADDR_UNDEF, "can't re-add section to file free space") + + node = NULL; + + /* Insert the new page into the Page Buffer list of new pages so + we don't read an empty page from disk */ + if(f->shared->page_buf != NULL && H5PB_add_new_page(f, alloc_type, new_page) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, HADDR_UNDEF, "can't add new page to Page Buffer new page list") + + ret_value = new_page; + } + break; + + case H5F_MEM_PAGE_NTYPES: + case H5F_MEM_PAGE_DEFAULT: + default: + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space: unrecognized type") + break; + } /* end switch */ + +done: +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", FUNC, ret_value, size); +#endif /* H5MF_ALLOC_DEBUG */ +#ifdef H5MF_ALLOC_DEBUG_DUMP +H5MF_sects_dump(f, dxpl_id, stderr); +#endif /* H5MF_ALLOC_DEBUG_DUMP */ + + /* Release section node, if allocated and not added to section list or merged */ + if(node) + if(H5MF_sect_free((H5FS_section_info_t *)node) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, HADDR_UNDEF, "can't free section node") + + FUNC_LEAVE_NOAPI_TAG(ret_value, HADDR_UNDEF) +} /* end H5MF__alloc_pagefs() */ + + +/*------------------------------------------------------------------------- * Function: H5MF_alloc_tmp * * Purpose: Allocate temporary space in the file @@ -645,14 +1115,14 @@ herr_t H5MF_xfree(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr, hsize_t size) { - H5F_io_info_t fio_info; /* I/O info for operation */ + H5F_io_info2_t fio_info; /* I/O info for operation */ + H5F_mem_page_t fs_type; /* Free space type (mapped from allocation type) */ H5MF_free_section_t *node = NULL; /* Free space section pointer */ - H5MF_sect_ud_t udata; /* User data for callback */ + unsigned ctype; /* section class type */ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t fsm_ring; /* Free space manager ring */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* Ring of fsm */ hbool_t reset_ring = FALSE; /* Whether the ring was set */ - H5FD_mem_t fs_type; /* Free space type (mapped from allocation type) */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) @@ -666,19 +1136,16 @@ HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUN HGOTO_DONE(SUCCEED) HDassert(addr != 0); /* Can't deallocate the superblock :-) */ - /* Check for attempting to free space that's a 'temporary' file address */ - if(H5F_addr_le(f->shared->tmp_addr, addr)) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, FAIL, "attempting to free temporary file space") + if(f->shared->first_alloc_dealloc) { + HDassert(!H5AC_cache_image_pending(f)); + if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed") + } /* end if */ - /* Get free space type from allocation type */ - fs_type = H5MF_ALLOC_TO_FS_TYPE(f, alloc_type); -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: fs_type = %u\n", FUNC, (unsigned)fs_type); -#endif /* H5MF_ALLOC_DEBUG_MORE */ + H5MF_alloc_to_fs_type(f, alloc_type, size, &fs_type); /* Set the ring type in the DXPL */ - if((fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) + if(H5MF__fsm_type_is_self_referential(f, fs_type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -686,10 +1153,33 @@ HDfprintf(stderr, "%s: fs_type = %u\n", FUNC, (unsigned)fs_type); HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") reset_ring = TRUE; + /* we are about to change the contents of the free space manager -- + * notify metadata cache that the associated fsm ring is + * unsettled + */ + /* Only do so for strategies that use free-space managers */ + if(H5F_HAVE_FREE_SPACE_MANAGER(f)) + if(H5AC_unsettle_ring(f, fsm_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_SYSTEM, FAIL, "attempt to notify cache that ring is unsettled failed") + + /* Check for attempting to free space that's a 'temporary' file address */ + if(H5F_addr_le(f->shared->tmp_addr, addr)) + HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, FAIL, "attempting to free temporary file space") + /* Set up I/O info for operation */ fio_info.f = f; - if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(H5FD_MEM_DRAW == alloc_type) { + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end if */ + else { + if(NULL == (fio_info.meta_dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + if(NULL == (fio_info.raw_dxpl = (H5P_genplist_t *)H5I_object(H5AC_rawdata_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + } /* end else */ /* Check if the space to free intersects with the file's metadata accumulator */ if(H5F__accum_free(&fio_info, alloc_type, addr, size) < 0) @@ -702,7 +1192,7 @@ HDfprintf(stderr, "%s: fs_type = %u\n", FUNC, (unsigned)fs_type); * space is at the end of the file */ #ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)fs_type, f->shared->fs_addr[fs_type]); +HDfprintf(stderr, "%s: fs_addr = %a\n", FUNC, f->shared->fs_addr[fs_type]); #endif /* H5MF_ALLOC_DEBUG_MORE */ if(!H5F_addr_defined(f->shared->fs_addr[fs_type])) { htri_t status; /* "can absorb" status for section into */ @@ -726,8 +1216,8 @@ HDfprintf(stderr, "%s: dropping addr = %a, size = %Hu, on the floor!\n", FUNC, a /* If we are deleting the free space manager, leave now, to avoid * [re-]starting it. - * or if file space strategy type is not using a free space manager - * (H5F_FILE_SPACE_AGGR_VFD or H5F_FILE_SPACE_VFD), drop free space + * or if file space strategy type is not using a free space manager + * (H5F_FSPACE_STRATEGY_AGGR or H5F_FSPACE_STRATEGY_NONE), drop free space * section on the floor. * * Note: this drops the space to free on the floor... @@ -745,38 +1235,42 @@ HDfprintf(stderr, "%s: dropping addr = %a, size = %Hu, on the floor!\n", FUNC, a * space isn't at the end of the file, so start up (or create) * the file space manager */ - if(H5MF__alloc_start(f, dxpl_id, fs_type) < 0) + if(H5MF_start_fstype(f, dxpl_id, fs_type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") } /* end if */ - /* Create free space section for block */ - if(NULL == (node = H5MF_sect_simple_new(addr, size))) + /* Create the free-space section for the freed section */ + ctype = H5MF_SECT_CLASS_TYPE(f, size); + if(NULL == (node = H5MF_sect_new(ctype, addr, size))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section") - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = dxpl_id; - udata.alloc_type = alloc_type; - udata.allow_sect_absorb = TRUE; - udata.allow_eoa_shrink_only = FALSE; - - /* If size of section freed is larger than threshold, add it to the free space manager */ + /* If size of the freed section is larger than threshold, add it to the free space manager */ if(size >= f->shared->fs_threshold) { HDassert(f->shared->fs_man[fs_type]); #ifdef H5MF_ALLOC_DEBUG_MORE HDfprintf(stderr, "%s: Before H5FS_sect_add()\n", FUNC); #endif /* H5MF_ALLOC_DEBUG_MORE */ + /* Add to the free space for the file */ - if(H5FS_sect_add(f, dxpl_id, f->shared->fs_man[fs_type], (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, &udata) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't add section to file free space") - node = NULL; + if(H5MF_add_sect(f, alloc_type, dxpl_id, f->shared->fs_man[fs_type], node) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't add section to file free space") + node = NULL; + #ifdef H5MF_ALLOC_DEBUG_MORE HDfprintf(stderr, "%s: After H5FS_sect_add()\n", FUNC); #endif /* H5MF_ALLOC_DEBUG_MORE */ } /* end if */ else { htri_t merged; /* Whether node was merged */ + H5MF_sect_ud_t udata; /* User data for callback */ + + /* Construct user data for callbacks */ + udata.f = f; + udata.dxpl_id = dxpl_id; + udata.alloc_type = alloc_type; + udata.allow_sect_absorb = TRUE; + udata.allow_eoa_shrink_only = FALSE; /* Try to merge the section that is smaller than threshold */ if((merged = H5FS_sect_try_merge(f, dxpl_id, f->shared->fs_man[fs_type], (H5FS_section_info_t *)node, H5FS_ADD_RETURNED_SPACE, &udata)) < 0) @@ -794,7 +1288,7 @@ done: /* Release section node, if allocated and not added to section list or merged */ if(node) - if(H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) + if(H5MF_sect_free((H5FS_section_info_t *)node) < 0) HDONE_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") #ifdef H5MF_ALLOC_DEBUG @@ -811,6 +1305,14 @@ H5MF_sects_dump(f, dxpl_id, stderr); * Function: H5MF_try_extend * * Purpose: Extend a block in the file if possible. + * For non-paged aggregation: + * --try to extend at EOA + * --try to extend into the aggregators + * --try to extend into a free-space section if adjoined + * For paged aggregation: + * --try to extend at EOA + * --try to extend into a free-space section if adjoined + * --try to extend into the page end threshold if a metadata block * * Return: Success: TRUE(1) - Block was extended * FALSE(0) - Block could not be extended @@ -825,14 +1327,16 @@ htri_t H5MF_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type, haddr_t addr, hsize_t size, hsize_t extra_requested) { - H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t fsm_ring; /* Free space manager ring */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - haddr_t end; /* End of block to extend */ - H5FD_mem_t fs_type; /* Memory type of the free space manager */ - H5FD_mem_t map_type; /* Mapped type */ - hbool_t reset_ring = FALSE; /* Whether the ring was set */ - htri_t ret_value = FAIL; /* Return value */ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* ring of fsm */ + haddr_t end; /* End of block to extend */ + H5FD_mem_t map_type; /* Mapped type */ + H5F_mem_page_t fs_type; /* free space type */ + htri_t allow_extend = TRUE; /* Possible to extend the block */ + hsize_t frag_size = 0; /* Size of mis-aligned fragment */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + htri_t ret_value = FALSE; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG @@ -849,12 +1353,33 @@ HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_r /* Compute end of block to extend */ end = addr + size; + /* For paged aggregation: + * To extend a small block: can only extend if not crossing page boundary + * To extend a large block at EOA: calculate in advance mis-aligned fragment so EOA will still end at page boundary + */ + if(H5F_PAGED_AGGR(f)) { + if(size < f->shared->fs_page_size) { + /* To extend a small block: cannot cross page boundary */ + if((addr / f->shared->fs_page_size) != (((end + extra_requested) - 1) / f->shared->fs_page_size)) + allow_extend = FALSE; + } /* end if */ + else { + haddr_t eoa; /* EOA for the file */ + + /* To extend a large block: calculate in advance the mis-aligned fragment so EOA will end at page boundary if extended */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, alloc_type))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "Unable to get eoa") + HDassert(!(eoa % f->shared->fs_page_size)); + + H5MF_EOA_MISALIGN(f, (eoa+extra_requested), f->shared->fs_page_size, frag_size); + } /* end else */ + } /* end if */ + /* Get free space type from allocation type */ - fs_type = H5MF_ALLOC_TO_FS_TYPE(f, alloc_type); + H5MF_alloc_to_fs_type(f, alloc_type, size, &fs_type); /* Set the ring type in the DXPL */ - if((fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (fs_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) + if(H5MF__fsm_type_is_self_referential(f, fs_type)) fsm_ring = H5AC_RING_MDFSM; else fsm_ring = H5AC_RING_RDFSM; @@ -862,29 +1387,90 @@ HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_r HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") reset_ring = TRUE; - /* Check if the block is exactly at the end of the file */ - if((ret_value = H5FD_try_extend(f->shared->lf, map_type, f, dxpl_id, end, extra_requested)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file") - else if(ret_value == FALSE) { - H5F_blk_aggr_t *aggr; /* Aggregator to use */ + if(allow_extend) { + /* Try extending the block at EOA */ + if((ret_value = H5F_try_extend(f, dxpl_id, map_type, end, extra_requested + frag_size)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file") +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: extended = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* If extending at EOA succeeds: */ + /* for paged aggregation, put the fragment into the large-sized free-space manager */ + if(ret_value == TRUE && H5F_PAGED_AGGR(f) && frag_size) { + H5MF_free_section_t *node = NULL; /* Free space section pointer */ - /* Check for test block able to extend aggregation block */ - aggr = (map_type == H5FD_MEM_DRAW) ? &(f->shared->sdata_aggr) : &(f->shared->meta_aggr); - if((ret_value = H5MF_aggr_try_extend(f, dxpl_id, aggr, map_type, end, extra_requested)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending aggregation block") - else if(ret_value == FALSE) { + /* Should be large-sized block */ + HDassert(size >= f->shared->fs_page_size); + + /* Start up the free-space manager */ + if(!(f->shared->fs_man[fs_type])) + if(H5MF_start_fstype(f, dxpl_id, fs_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") + + /* Create free space section for the fragment */ + if(NULL == (node = H5MF_sect_new(H5MF_FSPACE_SECT_LARGE, end + extra_requested, frag_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section") + + /* Add the fragment to the large-sized free-space manager */ + if(H5MF_add_sect(f, alloc_type, dxpl_id, f->shared->fs_man[fs_type], node) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINSERT, FAIL, "can't re-add section to file free space") + + node = NULL; + } /* end if */ + + /* For non-paged aggregation: try to extend into the aggregators */ + if(ret_value == FALSE && (f->shared->fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR || + f->shared->fs_strategy == H5F_FSPACE_STRATEGY_AGGR) ) { + H5F_blk_aggr_t *aggr; /* Aggregator to use */ + + /* Check if the block is able to extend into aggregation block */ + aggr = (map_type == H5FD_MEM_DRAW) ? &(f->shared->sdata_aggr) : &(f->shared->meta_aggr); + if((ret_value = H5MF_aggr_try_extend(f, dxpl_id, aggr, map_type, end, extra_requested)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending aggregation block") + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: H5MF_aggr_try_extend = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + } /* end if */ + + /* If no extension so far, try to extend into a free-space section */ + if(ret_value == FALSE && ((f->shared->fs_strategy == H5F_FSPACE_STRATEGY_FSM_AGGR) || + (H5F_PAGED_AGGR(f))) ) { + H5MF_sect_ud_t udata; /* User data */ + + /* Construct user data for callbacks */ + udata.f = f; + udata.dxpl_id = dxpl_id; + udata.alloc_type = alloc_type; /* Check if the free space for the file has been initialized */ if(!f->shared->fs_man[fs_type] && H5F_addr_defined(f->shared->fs_addr[fs_type])) - if(H5MF__alloc_open(f, dxpl_id, fs_type) < 0) + /* Open the free-space manager */ + if(H5MF_open_fstype(f, dxpl_id, fs_type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") - /* Check for test block able to block in free space manager */ - if(f->shared->fs_man[fs_type]) - if((ret_value = H5FS_sect_try_extend(f, dxpl_id, f->shared->fs_man[fs_type], addr, size, extra_requested)) < 0) + /* Try to extend the block into a free-space section */ + if(f->shared->fs_man[fs_type]) { + if((ret_value = H5FS_sect_try_extend(f, dxpl_id, f->shared->fs_man[fs_type], addr, size, extra_requested, H5FS_ADD_RETURNED_SPACE, &udata)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending block in free space manager") +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Try to H5FS_sect_try_extend = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + } /* end if */ + + /* For paged aggregation and a metadata block: try to extend into page end threshold */ + if(ret_value == FALSE && H5F_PAGED_AGGR(f) && map_type != H5FD_MEM_DRAW) { + H5MF_EOA_MISALIGN(f, end, f->shared->fs_page_size, frag_size); + + if(frag_size <= H5F_PGEND_META_THRES(f) && extra_requested <= frag_size) + ret_value = TRUE; +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Try to extend into the page end threshold = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + } /* end if */ } /* end if */ - } /* end else-if */ + } /* allow_extend */ done: /* Reset the ring in the DXPL */ @@ -904,174 +1490,316 @@ H5MF_sects_dump(f, dxpl_id, stderr); /*------------------------------------------------------------------------- - * Function: H5MF_get_freespace + * Function: H5MF_try_shrink * - * Purpose: Retrieve the amount of free space in a file. + * Purpose: Try to shrink the size of a file with a block or absorb it + * into a block aggregator. * - * Return: Success: Amount of free space in file - * Failure: Negative + * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * Monday, October 6, 2003 + * koziol@hdfgroup.org + * Feb 14 2008 * - * Modifications: - * Vailin Choi; July 2012 - * As the default free-list mapping is changed to H5FD_FLMAP_DICHOTOMY, - * checks are added to account for the last section of each free-space manager - * and the remaining space in the two aggregators are at EOF. *------------------------------------------------------------------------- */ -herr_t -H5MF_get_freespace(H5F_t *f, hid_t dxpl_id, hsize_t *tot_space, hsize_t *meta_size) +htri_t +H5MF_try_shrink(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr, + hsize_t size) { - H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t curr_ring; /* Current ring value */ - H5AC_ring_t needed_ring; /* Ring value needed for loop iteration */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - haddr_t eoa; /* End of allocated space in the file */ - haddr_t ma_addr = HADDR_UNDEF; /* Base "metadata aggregator" address */ - hsize_t ma_size = 0; /* Size of "metadata aggregator" */ - haddr_t sda_addr = HADDR_UNDEF; /* Base "small data aggregator" address */ - hsize_t sda_size = 0; /* Size of "small data aggregator" */ - hsize_t tot_fs_size = 0; /* Amount of all free space managed */ - hsize_t tot_meta_size = 0; /* Amount of metadata for free space managers */ - H5FD_mem_t type; /* Memory type for iteration */ - hbool_t fs_started[H5FD_MEM_NTYPES]; /* Indicate whether the free-space manager has been started */ - hbool_t eoa_shrank; /* Whether an EOA shrink occurs */ - hbool_t reset_ring = FALSE; /* Whether the ring was set */ - herr_t ret_value = SUCCEED; /* Return value */ + H5MF_free_section_t *node = NULL; /* Free space section pointer */ + H5MF_sect_ud_t udata; /* User data for callback */ + H5FS_section_class_t *sect_cls; /* Section class */ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t fsm_ring = H5AC_RING_INV; /* Ring of fsm */ + H5F_mem_page_t fs_type; /* Free space type */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + htri_t ret_value = FAIL; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUNC, (unsigned)alloc_type, addr, size); +#endif /* H5MF_ALLOC_DEBUG */ - /* check args */ + /* check arguments */ HDassert(f); HDassert(f->shared); HDassert(f->shared->lf); + HDassert(H5F_addr_defined(addr)); + HDassert(size > 0); - /* Retrieve the 'eoa' for the file */ - if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, H5FD_MEM_DEFAULT))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + /* Set up free-space section class information */ + sect_cls = H5MF_SECT_CLS_TYPE(f, size); + HDassert(sect_cls); + + /* Get free space type from allocation type */ + H5MF_alloc_to_fs_type(f, alloc_type, size, &fs_type); /* Set the ring type in the DXPL */ - if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + if(H5MF__fsm_type_is_self_referential(f, fs_type)) + fsm_ring = H5AC_RING_MDFSM; + else + fsm_ring = H5AC_RING_RDFSM; + if(H5AC_set_ring(dxpl_id, fsm_ring, &dxpl, &orig_ring) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") reset_ring = TRUE; - curr_ring = H5AC_RING_RDFSM; - /* Retrieve metadata aggregator info, if available */ - if(H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query metadata aggregator stats") + /* Create free-space section for block */ + if(NULL == (node = H5MF_sect_new(sect_cls->type, addr, size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section") - /* Retrieve 'small data' aggregator info, if available */ - if(H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sda_addr, &sda_size) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query small data aggregator stats") + /* Construct user data for callbacks */ + udata.f = f; + udata.dxpl_id = dxpl_id; + udata.alloc_type = alloc_type; + udata.allow_sect_absorb = FALSE; /* Force section to be absorbed into aggregator */ + udata.allow_eoa_shrink_only = FALSE; - /* Iterate over all the free space types that have managers and get each free list's space */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { + /* Check if the block can shrink the container */ + if(sect_cls->can_shrink) { + if((ret_value = (*sect_cls->can_shrink)((const H5FS_section_info_t *)node, &udata)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTMERGE, FAIL, "can't check if section can shrink container") + if(ret_value > 0) { + HDassert(sect_cls->shrink); - fs_started[type] = FALSE; + if((*sect_cls->shrink)((H5FS_section_info_t **)&node, &udata) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink container") + } /* end if */ + } /* end if */ - /* test to see if we need to switch rings -- do so if required */ - if((type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; +done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (2)") - curr_ring = needed_ring; - } /* end if */ + /* Free section node allocated */ + if(node && H5MF_sect_free((H5FS_section_info_t *)node) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") - /* Check if the free space for the file has been initialized */ - if(!f->shared->fs_man[type] && H5F_addr_defined(f->shared->fs_addr[type])) { - if(H5MF__alloc_open(f, dxpl_id, type) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") - HDassert(f->shared->fs_man[type]); - fs_started[type] = TRUE; - } /* end if */ +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF_try_shrink() */ - /* Check if there's free space of this type */ - if(f->shared->fs_man[type]) { - hsize_t type_fs_size = 0; /* Amount of free space managed for each type */ - hsize_t type_meta_size = 0; /* Amount of free space metadata for each type */ + +/*------------------------------------------------------------------------- + * Function: H5MF_close + * + * Purpose: Close the free space tracker(s) for a file: + * paged or non-paged aggregation + * + * Return: SUCCEED/FAIL + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +herr_t +H5MF_close(H5F_t *f, hid_t dxpl_id) +{ + herr_t ret_value = SUCCEED; /* Return value */ - /* Retrieve free space size from free space manager */ - if(H5FS_sect_stats(f->shared->fs_man[type], &type_fs_size, NULL) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space stats") - if(H5FS_size(f, f->shared->fs_man[type], &type_meta_size) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space metadata stats") + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Entering\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ - /* Increment total free space for types */ - tot_fs_size += type_fs_size; - tot_meta_size += type_meta_size; - } /* end if */ - } /* end for */ + /* check args */ + HDassert(f); + HDassert(f->shared); - /* Iterate until no more EOA shrink occurs */ - do { - eoa_shrank = FALSE; + if(H5F_PAGED_AGGR(f)) { + if((ret_value = H5MF__close_pagefs(f, dxpl_id)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't close free-space managers for 'page' file space") + } /* end if */ + else { + if((ret_value = H5MF__close_aggrfs(f, dxpl_id)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't close free-space managers for 'aggr' file space") + } /* end else */ - /* Check the last section of each free-space manager */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - haddr_t sect_addr = HADDR_UNDEF; - hsize_t sect_size = 0; - - if(f->shared->fs_man[type]) { - if(H5FS_sect_query_last_sect(f->shared->fs_man[type], §_addr, §_size) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query last section on merge list") - - /* Deduct space from previous accumulation if the section is at EOA */ - if(H5F_addr_eq(sect_addr + sect_size, eoa)) { - eoa = sect_addr; - eoa_shrank = TRUE; - tot_fs_size -= sect_size; - } /* end if */ - } /* end if */ - } /* end for */ - - /* Check the metadata and raw data aggregators */ - if(ma_size > 0 && H5F_addr_eq(ma_addr + ma_size, eoa)) { - eoa = ma_addr; - eoa_shrank = TRUE; - ma_size = 0; - } /* end if */ - if(sda_size > 0 && H5F_addr_eq(sda_addr + sda_size, eoa)) { - eoa = sda_addr; - eoa_shrank = TRUE; - sda_size = 0; - } /* end if */ - } while(eoa_shrank); +done: - /* Close the free-space managers if they were opened earlier in this routine */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - if(fs_started[type]) { +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF_close() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF__close_delete_fstype + * + * Purpose: Common code for closing and deleting the freespace manager + * of TYPE for file. + * Note that TYPE can be H5F_mem_page_t or H5FD_mem_t enum types. + * + * Return: SUCCEED/FAIL + * + * Programmer: Vailin Choi + * Jan 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF__close_delete_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Entering\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + + /* check args */ + HDassert(f); + HDassert(f->shared); + if(H5F_PAGED_AGGR(f)) + HDassert(type < H5F_MEM_PAGE_NTYPES); + else + HDassert((H5FD_mem_t)type < H5FD_MEM_NTYPES); + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Check 1.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* If the free space manager for this type is open, close it */ + if(f->shared->fs_man[type]) + if(H5MF__close_fstype(f, dxpl_id, type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close the free space manager") + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Check 2.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* If there is free space manager info for this type, delete it */ + if(H5F_addr_defined(f->shared->fs_addr[type])) + if(H5MF__delete_fstype(f, dxpl_id, type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't delete the free space manager") + +done: +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* H5MF__close_delete() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_try_close + * + * Purpose: This is called by H5Fformat_convert() to close and delete + * free-space managers when downgrading persistent free-space + * to non-persistent. + * + * Return: SUCCEED/FAIL + * + * Programmer: Vailin Choi + * Jan 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5MF_try_close(H5F_t *f, hid_t dxpl_id) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Entering\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + + /* check args */ + HDassert(f); + + /* If there have been no file space allocations / deallocation so + * far, must call H5MF_tidy_self_referential_fsm_hack() to float + * all self referential FSMs and release file space allocated to + * them. Otherwise, the function will be called after the format + * conversion, and will become very confused. + * + * The situation is further complicated if a cache image exists + * and had not yet been loaded into the metadata cache. In this + * case, call H5AC_force_cache_image_load() instead of + * H5MF_tidy_self_referential_fsm_hack(). H5AC_force_cache_image_load() + * will load the cache image, and then call + * H5MF_tidy_self_referential_fsm_hack() to discard the cache image + * block. + */ + if(f->shared->first_alloc_dealloc) { + if(H5AC_cache_image_pending(f)) { + if(H5AC_force_cache_image_load(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "forced cache image load failed") + } /* end if */ + else { + if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed") + } /* end else */ + } /* end if */ + + /* Set the ring type in the DXPL. In most cases, we will + * need H5AC_RING_RDFSM, so initialy set the ring in + * the DXPL to that value. We will alter this later if + * needed. + */ + if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + curr_ring = H5AC_RING_RDFSM; + + if(H5F_PAGED_AGGR(f)) { + H5F_mem_page_t ptype; /* Memory type for iteration */ + /* Iterate over all the free space types that have managers and + * get each free list's space + */ + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) { + /* Test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, ptype)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring ) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)") + curr_ring = needed_ring; + } /* end if */ + + if(H5MF__close_delete_fstype(f, dxpl_id, ptype) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close the free space manager") + } /* end for */ + } /* end if */ + else { + H5FD_mem_t type; /* Memory type for iteration */ + + /* Iterate over all the free space types that have managers and + * get each free list's space + */ + for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { /* test to see if we need to switch rings -- do so if required */ - if((type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) + if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) needed_ring = H5AC_RING_MDFSM; else needed_ring = H5AC_RING_RDFSM; if(needed_ring != curr_ring) { if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (3)") + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") curr_ring = needed_ring; } /* end if */ - if(H5MF__alloc_close(f, dxpl_id, type) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't close file free space") - } /* end if */ - } /* end for */ - - /* Set the value(s) to return */ - /* (The metadata & small data aggregators count as free space now, since they aren't at EOA) */ - if(tot_space) - *tot_space = tot_fs_size + ma_size + sda_size; - if(meta_size) - *meta_size = tot_meta_size; + if(H5MF__close_delete_fstype(f, dxpl_id, (H5F_mem_page_t)type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close the free space manager") + } /* end for */ + } /* end else */ done: /* Reset the ring in the DXPL */ @@ -1079,75 +1807,370 @@ done: if(H5AC_reset_ring(dxpl, orig_ring) < 0) HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* end H5MF_get_freespace() */ +} /* H5MF_try_close() */ /*------------------------------------------------------------------------- - * Function: H5MF_try_shrink + * Function: H5MF__close_aggrfs * - * Purpose: Try to shrink the size of a file with a block or absorb it - * into a block aggregator. + * Purpose: Close the free space tracker(s) for a file: non-paged aggregation * - * Return: Non-negative on success/Negative on failure + * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org - * Feb 14 2008 + * Tuesday, January 22, 2008 * *------------------------------------------------------------------------- */ -htri_t -H5MF_try_shrink(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr, - hsize_t size) +static herr_t +H5MF__close_aggrfs(H5F_t *f, hid_t dxpl_id) { - H5MF_free_section_t *node = NULL; /* Free space section pointer */ - H5MF_sect_ud_t udata; /* User data for callback */ - htri_t ret_value = FAIL; /* Return value */ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration. */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5FD_mem_t type; /* Memory type for iteration */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUNC, (unsigned)alloc_type, addr, size); +HDfprintf(stderr, "%s: Entering\n", FUNC); #endif /* H5MF_ALLOC_DEBUG */ - /* check arguments */ + /* check args */ HDassert(f); HDassert(f->shared); HDassert(f->shared->lf); - HDassert(H5F_addr_defined(addr)); - HDassert(size > 0); + HDassert(f->shared->sblock); - /* Create free space section for block */ - if(NULL == (node = H5MF_sect_simple_new(addr, size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize free space section") + /* Set the ring type in the DXPL. In most cases, we will + * need H5AC_RING_RDFSM, so initialy set the ring in + * the DXPL to that value. We will alter this later if + * needed. + */ + if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + curr_ring = H5AC_RING_RDFSM; - /* Construct user data for callbacks */ - udata.f = f; - udata.dxpl_id = dxpl_id; - udata.alloc_type = alloc_type; - udata.allow_sect_absorb = FALSE; /* Force section to be absorbed into aggregator */ - udata.allow_eoa_shrink_only = FALSE; - - /* Call the "can shrink" callback for the section */ - if((ret_value = H5MF_sect_simple_can_shrink((const H5FS_section_info_t *)node, &udata)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTMERGE, FAIL, "can't check if section can shrink container") - else if(ret_value > 0) { - /* Shrink or absorb the section */ - if(H5MF_sect_simple_shrink((H5FS_section_info_t **)&node, &udata) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink container") + /* Free the space in aggregators */ + /* (for space not at EOA, it may be put into free space managers) */ + if(H5MF_free_aggrs(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators") + + /* Trying shrinking the EOA for the file */ + if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") + + /* Making free-space managers persistent for superblock version >= 2 */ + if(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2 + && f->shared->fs_persist) { + H5O_fsinfo_t fsinfo; /* File space info message */ + haddr_t final_eoa; /* Final eoa -- for sanity check */ + H5F_mem_page_t ptype; /* Memory type for iteration */ + + /* superblock extension and free space manager message should + * exist at this point -- verify at least the former. + */ + HDassert(H5F_addr_defined(f->shared->sblock->ext_addr)); + + /* file space for all non-empty free space managers should be + * allocated at this point, and these free space managers should + * be written to file and thus their headers and section info + * entries in the metadata cache should be clean. + */ + + /* gather data for the free space manager superblock extension message. + * + * In passing, verify that all the free space managers are closed. + */ + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + fsinfo.fs_addr[ptype - 1] = HADDR_UNDEF; + for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) + fsinfo.fs_addr[type-1] = f->shared->fs_addr[type]; + fsinfo.strategy = f->shared->fs_strategy; + fsinfo.persist = f->shared->fs_persist; + fsinfo.threshold = f->shared->fs_threshold; + fsinfo.page_size = f->shared->fs_page_size; + fsinfo.pgend_meta_thres = f->shared->pgend_meta_thres; + fsinfo.eoa_pre_fsm_fsalloc = f->shared->eoa_pre_fsm_fsalloc; + + /* Write the free space manager message -- message must already exist */ + if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension") + + /* Close the free space managers */ + for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { + if(f->shared->fs_man[type]) { + /* test to see if we need to switch rings -- do + * so if required + */ + if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)") + curr_ring = needed_ring; + } /* end if */ + + HDassert(f->shared->fs_state[type] == H5F_FS_STATE_OPEN); + + if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close free space manager") + f->shared->fs_man[type] = NULL; + f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; + } /* end if */ + f->shared->fs_addr[type] = HADDR_UNDEF; + } /* end for */ + + /* verify that we haven't dirtied any metadata cache entries + * from the metadata free space manager ring out. + */ + HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM)); + + /* verify that the aggregators are still shutdown. */ + HDassert(f->shared->sdata_aggr.tot_size == 0); + HDassert(f->shared->sdata_aggr.addr == 0); + HDassert(f->shared->sdata_aggr.size == 0); + + HDassert(f->shared->meta_aggr.tot_size == 0); + HDassert(f->shared->meta_aggr.addr == 0); + HDassert(f->shared->meta_aggr.size == 0); + + /* Trying shrinking the EOA for the file */ + /* (in case any free space is now at the EOA) */ + if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") + + /* get the eoa, and verify that it has the expected value */ + if(HADDR_UNDEF == (final_eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)) ) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file size") + + /* f->shared->eoa_post_fsm_fsalloc is undefined if there has + * been no file space allocation or deallocation since file + * open. + */ + HDassert((f->shared->first_alloc_dealloc) || (final_eoa == f->shared->eoa_post_fsm_fsalloc)); } /* end if */ + else { /* super_vers can be 0, 1, 2 */ + for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) + if(H5MF__close_delete_fstype(f, dxpl_id, (H5F_mem_page_t)type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") + } /* end else */ + + /* Free the space in aggregators (again) */ + /* (in case any free space information re-started them) */ + if(H5MF_free_aggrs(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators") + + /* Trying shrinking the EOA for the file */ + /* (in case any free space is now at the EOA) */ + if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") - /* Free section node allocated */ - if(node && H5MF_sect_simple_free((H5FS_section_info_t *)node) < 0) - HDONE_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF__close_aggrfs() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF__close_pagefs + * + * Purpose: Close the free space tracker(s) for a file: paged aggregation + * + * Return: SUCCEED/FAIL + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF__close_pagefs(H5F_t *f, hid_t dxpl_id) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* ring value needed for this + * iteration. + */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5F_mem_page_t ptype; /* Memory type for iteration */ + H5O_fsinfo_t fsinfo; /* File space info message */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + herr_t ret_value = SUCCEED; /* Return value */ + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value); +HDfprintf(stderr, "%s: Entering\n", FUNC); #endif /* H5MF_ALLOC_DEBUG */ - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5MF_try_shrink() */ + + /* check args */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); + HDassert(f->shared->sblock); + HDassert(f->shared->fs_page_size); + HDassert(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2); + + /* Set the ring type in the DXPL. In most cases, we will + * need H5AC_RING_RDFSM, so initialy set the ring in + * the DXPL to that value. We will alter this later if + * needed. + */ + if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + curr_ring = H5AC_RING_RDFSM; + + /* Trying shrinking the EOA for the file */ + if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") + + /* Set up file space info message */ + fsinfo.strategy = f->shared->fs_strategy; + fsinfo.persist = f->shared->fs_persist; + fsinfo.threshold = f->shared->fs_threshold; + fsinfo.page_size = f->shared->fs_page_size; + fsinfo.pgend_meta_thres = f->shared->pgend_meta_thres; + fsinfo.eoa_pre_fsm_fsalloc = HADDR_UNDEF; + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + fsinfo.fs_addr[ptype - 1] = HADDR_UNDEF; + + if(f->shared->fs_persist) { + haddr_t final_eoa; /* final eoa -- for sanity check */ + + /* superblock extension and free space manager message should + * exist at this point -- verify at least the former. + */ + HDassert(H5F_addr_defined(f->shared->sblock->ext_addr)); + + /* file space for all non-empty free space managers should be + * allocated at this point, and these free space managers should + * be written to file and thus their headers and section info + * entries in the metadata cache should be clean. + */ + + /* gather data for the free space manager superblock extension message. + * Only need addresses of FSMs and eoa prior to allocation of + * file space for the self referential free space managers. Other + * data was gathered above. + */ + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + fsinfo.fs_addr[ptype-1] = f->shared->fs_addr[ptype]; + fsinfo.eoa_pre_fsm_fsalloc = f->shared->eoa_pre_fsm_fsalloc; + + /* Write the free space manager message -- message must already exist */ + if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension") + + /* Close the free space managers */ + /* use H5MF__close_fstype() for this? */ + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) { + if(f->shared->fs_man[ptype]) { + /* test to see if we need to switch rings -- do + * so if required + */ + if(H5MF__fsm_type_is_self_referential(f, ptype)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)") + curr_ring = needed_ring; + } /* end if */ + + HDassert(f->shared->fs_state[ptype] == H5F_FS_STATE_OPEN); + + if(H5FS_close(f, dxpl_id, f->shared->fs_man[ptype]) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close free space manager") + f->shared->fs_man[ptype] = NULL; + f->shared->fs_state[ptype] = H5F_FS_STATE_CLOSED; + } /* end if */ + f->shared->fs_addr[ptype] = HADDR_UNDEF; + } /* end for */ + + /* verify that we haven't dirtied any metadata cache entries + * from the metadata free space manager ring out. + */ + HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM)); + + /* Trying shrinking the EOA for the file */ + /* (in case any free space is now at the EOA) */ + if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") + + /* get the eoa, and verify that it has the expected value */ + if(HADDR_UNDEF == (final_eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT)) ) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file size") + + /* f->shared->eoa_post_fsm_fsalloc is undefined if there has + * been no file space allocation or deallocation since file + * open. + * + * If there is a cache image in the file at file open, + * f->shared->first_alloc_dealloc will always be FALSE unless + * the file is opened R/O, as otherwise, the image will have been + * read and discarded by this point. + * + * If a cache image was created on file close, the actual EOA + * should be in f->shared->eoa_post_mdci_fsalloc. Note that in + * this case, it is conceivable that f->shared->first_alloc_dealloc + * will still be TRUE, as the cache image is allocated directly from + * the file driver layer. However, as this possibility seems remote, + * it is ignored in the following assert. + */ + HDassert((f->shared->first_alloc_dealloc) || + (final_eoa == f->shared->eoa_post_fsm_fsalloc) || + ((H5F_addr_defined(f->shared->eoa_post_mdci_fsalloc)) && + (final_eoa == f->shared->eoa_post_mdci_fsalloc))); + } /* end if */ + else { + /* Iterate over all the free space types that have managers + * and get each free list's space + */ + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + if(H5MF__close_delete_fstype(f, dxpl_id, ptype) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close the free space manager") + + /* Write file space info message to superblock extension object header */ + /* Create the superblock extension object header in advance if needed */ + if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension") + } /* end else */ + + /* Trying shrinking the EOA for the file */ + /* (in case any free space is now at the EOA) */ + if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") + +done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: Leaving\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF__close_pagefs() */ /*------------------------------------------------------------------------- @@ -1165,17 +2188,16 @@ HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value); static herr_t H5MF__close_shrink_eoa(H5F_t *f, hid_t dxpl_id) { - H5FD_mem_t type; /* Memory type for iteration */ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration. */ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - H5AC_ring_t curr_ring = H5AC_RING_INV; /* current ring value */ - H5AC_ring_t needed_ring = H5AC_RING_INV; /* ring value needed for this - * iteration. - */ + H5F_mem_t type; + H5F_mem_page_t ptype; /* Memory type for iteration */ hbool_t eoa_shrank; /* Whether an EOA shrink occurs */ - hbool_t reset_ring = FALSE; /* Whether the ring was set */ htri_t status; /* Status value */ H5MF_sect_ud_t udata; /* User data for callback */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) @@ -1187,13 +2209,10 @@ H5MF__close_shrink_eoa(H5F_t *f, hid_t dxpl_id) /* Construct user data for callbacks */ udata.f = f; udata.dxpl_id = dxpl_id; - udata.allow_sect_absorb = FALSE; - udata.allow_eoa_shrink_only = TRUE; + udata.allow_sect_absorb = FALSE; + udata.allow_eoa_shrink_only = TRUE; - /* Set the ring type in the DXPL. In most cases, we will - * need H5AC_RING_RDFSM, so initialy set the ring type in - * the DXPL to that value. We will alter this later if needed. - */ + /* Set the ring type in the DXPL */ if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value(1)") reset_ring = TRUE; @@ -1203,36 +2222,62 @@ H5MF__close_shrink_eoa(H5F_t *f, hid_t dxpl_id) do { eoa_shrank = FALSE; - /* Check the last section of each free-space manager */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { + if(H5F_PAGED_AGGR(f)) { + /* Check the last section of each free-space manager */ + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) { + if(f->shared->fs_man[ptype]) { + /* Test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, ptype)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; - /* test to see if we need to switch rings -- do so if required */ - if((H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)") + curr_ring = needed_ring; + } /* end if */ - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (2)") - curr_ring = needed_ring; - } /* end if */ + udata.alloc_type = (H5FD_mem_t)((H5FD_mem_t)ptype < H5FD_MEM_NTYPES ? ptype : ((ptype % H5FD_MEM_NTYPES) + 1)); - if(f->shared->fs_man[type]) { - udata.alloc_type = type; - if((status = H5FS_sect_try_shrink_eoa(f, dxpl_id, f->shared->fs_man[type], &udata)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") - else if(status > 0) - eoa_shrank = TRUE; - } /* end if */ - } /* end for */ + if((status = H5FS_sect_try_shrink_eoa(f, dxpl_id, f->shared->fs_man[ptype], &udata)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") + else if(status > 0) + eoa_shrank = TRUE; + } /* end if */ + } /* end for */ + } /* end if */ + else { + /* Check the last section of each free-space manager */ + for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { + if(f->shared->fs_man[type]) { + /* Test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)") + curr_ring = needed_ring; + } /* end if */ - /* check the two aggregators */ - if((status = H5MF_aggrs_try_shrink_eoa(f, dxpl_id)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") - else if(status > 0) - eoa_shrank = TRUE; + udata.alloc_type = type; + + if((status = H5FS_sect_try_shrink_eoa(f, dxpl_id, f->shared->fs_man[type], &udata)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") + else if(status > 0) + eoa_shrank = TRUE; + } /* end if */ + } /* end for */ + + /* check the two aggregators */ + if((status = H5MF_aggrs_try_shrink_eoa(f, dxpl_id)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") + else if(status > 0) + eoa_shrank = TRUE; + } /* end else */ } while(eoa_shrank); done: @@ -1246,6 +2291,370 @@ done: /*------------------------------------------------------------------------- + * Function: H5MF_get_freespace + * + * Purpose: Retrieve the amount of free space in the file + * + * Return: Success: Amount of free space in file + * Failure: Negative + * + * Programmer: Quincey Koziol + * Monday, October 6, 2003 + * + *------------------------------------------------------------------------- + */ +herr_t +H5MF_get_freespace(H5F_t *f, hid_t dxpl_id, hsize_t *tot_space, hsize_t *meta_size) +{ + haddr_t ma_addr = HADDR_UNDEF; /* Base "metadata aggregator" address */ + hsize_t ma_size = 0; /* Size of "metadata aggregator" */ + haddr_t sda_addr = HADDR_UNDEF; /* Base "small data aggregator" address */ + hsize_t sda_size = 0; /* Size of "small data aggregator" */ + hsize_t tot_fs_size = 0; /* Amount of all free space managed */ + hsize_t tot_meta_size = 0; /* Amount of metadata for free space managers */ + H5FD_mem_t tt; /* Memory type for iteration */ + H5F_mem_page_t type; /* Memory type for iteration */ + H5F_mem_page_t start_type; /* Memory type for iteration */ + H5F_mem_page_t end_type; /* Memory type for iteration */ + htri_t fs_started[H5F_MEM_PAGE_NTYPES]; /* Indicate whether the free-space manager has been started */ + haddr_t fs_eoa[H5FD_MEM_NTYPES]; /* EAO for each free-space manager */ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration. */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + + /* check args */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); + + /* Set the ring type in the DXPL. In most cases, we will + * need H5AC_RING_RDFSM, so initialy set the ring in + * the DXPL to that value. We will alter this later if + * needed. + */ + if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + reset_ring = TRUE; + curr_ring = H5AC_RING_RDFSM; + + /* Determine start/end points for loop */ + if(H5F_PAGED_AGGR(f)) { + start_type = H5F_MEM_PAGE_META; + end_type = H5F_MEM_PAGE_NTYPES; + } /* end if */ + else { + start_type = (H5F_mem_page_t)H5FD_MEM_SUPER; + end_type = (H5F_mem_page_t)H5FD_MEM_NTYPES; + } /* end else */ + + for(tt = H5FD_MEM_SUPER; tt < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, tt)) + if(HADDR_UNDEF == (fs_eoa[tt] = H5F_get_eoa(f, tt))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + if(!H5F_PAGED_AGGR(f)) { + /* Retrieve metadata aggregator info, if available */ + if(H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query metadata aggregator stats") + + /* Retrieve 'small data' aggregator info, if available */ + if(H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sda_addr, &sda_size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query small data aggregator stats") + } /* end if */ + + /* Iterate over all the free space types that have managers and get each free list's space */ + for(type = start_type; type < end_type; H5_INC_ENUM(H5F_mem_page_t, type)) { + fs_started[type] = FALSE; + + /* Check if the free space for the file has been initialized */ + if(!f->shared->fs_man[type] && H5F_addr_defined(f->shared->fs_addr[type])) { + if(H5MF_open_fstype(f, dxpl_id, type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") + HDassert(f->shared->fs_man[type]); + fs_started[type] = TRUE; + } /* end if */ + + /* test to see if we need to switch rings -- do + * so if required + */ + if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + curr_ring = needed_ring; + } /* end if */ + + /* Check if there's free space of this type */ + if(f->shared->fs_man[type]) { + hsize_t type_fs_size = 0; /* Amount of free space managed for each type */ + hsize_t type_meta_size = 0; /* Amount of free space metadata for each type */ + + /* Retrieve free space size from free space manager */ + if(H5FS_sect_stats(f->shared->fs_man[type], &type_fs_size, NULL) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space stats") + if(H5FS_size(f, f->shared->fs_man[type], &type_meta_size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space metadata stats") + + /* Increment total free space for types */ + tot_fs_size += type_fs_size; + tot_meta_size += type_meta_size; + } /* end if */ + } /* end for */ + + /* Close the free-space managers if they were opened earlier in this routine */ + for(type = start_type; type < end_type; H5_INC_ENUM(H5F_mem_page_t, type)) { + /* Test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, (H5F_mem_page_t)type)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + curr_ring = needed_ring; + } /* end if */ + + if(fs_started[type]) + if(H5MF__close_fstype(f, dxpl_id, type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't close file free space") + } /* end for */ + + /* Set the value(s) to return */ + /* (The metadata & small data aggregators count as free space now, since they aren't at EOA) */ + if(tot_space) + *tot_space = tot_fs_size + ma_size + sda_size; + if(meta_size) + *meta_size = tot_meta_size; + +done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF_get_freespace() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_get_free_sections() + * + * Purpose: To retrieve free-space section information for + * paged or non-paged aggregation + * + * Return: SUCCEED/FAIL + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +ssize_t +H5MF_get_free_sections(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, size_t nsects, H5F_sect_info_t *sect_info) +{ + H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration. */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + size_t total_sects = 0; /* Total number of sections */ + H5MF_sect_iter_ud_t sect_udata; /* User data for callback */ + H5F_mem_page_t start_type, end_type; /* Memory types to iterate over */ + H5F_mem_page_t ty; /* Memory type for iteration */ + hbool_t reset_ring = FALSE; /* Whether the ring was set */ + ssize_t ret_value = -1; /* Return value */ + + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, -1) + + /* check args */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->lf); + + /* H5MF_tidy_self_referential_fsm_hack() will fail if any self + * referential FSM is opened prior to the call to it. Thus call + * it here if necessary and if it hasn't been called already. + * + * The situation is further complicated if a cache image exists + * and had not yet been loaded into the metadata cache. In this + * case, call H5AC_force_cache_image_load() instead of + * H5MF_tidy_self_referential_fsm_hack(). H5AC_force_cache_image_load() + * will load the cache image, and then call + * H5MF_tidy_self_referential_fsm_hack() to discard the cache image + * block. + */ + if(f->shared->first_alloc_dealloc) { + if(H5AC_cache_image_pending(f)) { + if(H5AC_force_cache_image_load(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "forced cache image load failed") + } /* end if */ + else { + if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed") + } /* end else */ + } /* end if */ + + if(type == H5FD_MEM_DEFAULT) { + start_type = H5F_MEM_PAGE_SUPER; + end_type = H5F_MEM_PAGE_NTYPES; + } /* end if */ + else { + start_type = end_type = (H5F_mem_page_t)type; + if(H5F_PAGED_AGGR(f)) /* set to the corresponding LARGE free-space manager */ + end_type = (H5F_mem_page_t)(end_type + H5FD_MEM_NTYPES); + else + H5_INC_ENUM(H5F_mem_page_t, end_type); + } /* end else */ + + /* Set up user data for section iteration */ + sect_udata.sects = sect_info; + sect_udata.sect_count = nsects; + sect_udata.sect_idx = 0; + + /* Set the ring type in the DXPL. In most cases, we will + * need H5AC_RING_RDFSM, so initialy set the ring in + * the DXPL to that value. We will alter this later if + * needed. + */ + if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value(0)") + reset_ring = TRUE; + curr_ring = H5AC_RING_RDFSM; + + /* Iterate over memory types, retrieving the number of sections of each type */ + for(ty = start_type; ty < end_type; H5_INC_ENUM(H5F_mem_page_t, ty)) { + hbool_t fs_started = FALSE; /* The free-space manager is opened or not */ + size_t nums = 0; /* The number of free-space sections */ + + /* Test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, ty)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (1)") + curr_ring = needed_ring; + } /* end if */ + + if(!f->shared->fs_man[ty] && H5F_addr_defined(f->shared->fs_addr[ty])) { + if(H5MF_open_fstype(f, dxpl_id, ty) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't open the free space manager") + HDassert(f->shared->fs_man[ty]); + fs_started = TRUE; + } /* end if */ + + /* Check if there's free space sections of this type */ + if(f->shared->fs_man[ty]) + if(H5MF__get_free_sects(f, dxpl_id, f->shared->fs_man[ty], §_udata, &nums) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get section info for the free space manager") + + /* Increment total # of sections */ + total_sects += nums; + + /* Close the free space manager of this type, if we started it here */ + if(fs_started) + if(H5MF__close_fstype(f, dxpl_id, ty) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCLOSEOBJ, FAIL, "can't close file free space") + if((H5F_PAGED_AGGR(f)) && (type != H5FD_MEM_DEFAULT)) + ty = (H5F_mem_page_t)(ty + H5FD_MEM_NTYPES - 2); + } /* end for */ + + /* Set return value */ + ret_value = (ssize_t)total_sects; + +done: + /* Reset the ring in the DXPL */ + if(reset_ring) + if(H5AC_reset_ring(dxpl, orig_ring) < 0) + HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + + FUNC_LEAVE_NOAPI_TAG(ret_value, -1) +} /* H5MF_get_free_sections() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sects_cb() + * + * Purpose: Iterator callback for each free-space section + * Retrieve address and size into user data + * + * Return: Always succeed + * + * Programmer: Vailin Choi + * July 1st, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF_sects_cb(H5FS_section_info_t *_sect, void *_udata) +{ + H5MF_free_section_t *sect = (H5MF_free_section_t *)_sect; + H5MF_sect_iter_ud_t *udata = (H5MF_sect_iter_ud_t *)_udata; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + if(udata->sect_idx < udata->sect_count) { + udata->sects[udata->sect_idx].addr = sect->sect_info.addr; + udata->sects[udata->sect_idx].size = sect->sect_info.size; + udata->sect_idx++; + } /* end if */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5MF_sects_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF__get_free_sects + * + * Purpose: Retrieve section information for the specified free-space manager. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF__get_free_sects(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5MF_sect_iter_ud_t *sect_udata, size_t *nums) +{ + hsize_t hnums = 0; /* # of sections */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* check args */ + HDassert(f); + HDassert(sect_udata); + HDassert(nums); + HDassert(fspace); + + /* Query how many sections of this type */ + if(H5FS_sect_stats(fspace, NULL, &hnums) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space stats") + H5_CHECKED_ASSIGN(*nums, size_t, hnums, hsize_t); + + /* Check if we should retrieve the section info */ + if(sect_udata->sects && *nums > 0) + /* Iterate over all the free space sections of this type, adding them to the user's section info */ + if(H5FS_sect_iterate(f, dxpl_id, fspace, H5MF_sects_cb, sect_udata) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_BADITER, FAIL, "can't iterate over sections") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF__get_free_sects() */ + + +/*------------------------------------------------------------------------- * Function: H5MF_settle_raw_data_fsm() * * Purpose: Handle any tasks required before the metadata cache @@ -1253,19 +2662,48 @@ done: * and any metadata free space managers that reside in the * raw data free space manager ring. * - * Specifically, any metadata managers that DON'T handle - * space allocation for free space manager header or section - * info will reside in the raw data free space manager ring. - * As of this writing, the plan is to move to only two free space - * managers, one for raw data and one for metadata -- which - * means that only the raw data free space manager will reside - * in the free space manager ring. However, this has not been - * fully implemented yet, so this code must support the - * possibilty of multiple metadata free space managers, at most - * two of which handle free space manager header or section info, - * and thus reside in the metadata free space manager ring. - * - * At present, the task list is: + * Specifically, this means any metadata managers that DON'T + * handle space allocation for free space manager header or + * section info will reside in the raw data free space manager + * ring. + * + * In the absence of page allocation, there is at most one + * free space manager per memory type defined in H5F_mem_t. + * Of these, the one that allocates H5FD_MEM_DRAW will + * always reside in the raw data free space manager ring. + * If there is more than one metadata free space manager, + * all that don't handle H5FD_MEM_FSPACE_HDR or + * H5FD_MEM_FSPACE_SINFO (which map to H5FD_MEM_OHDR and + * H5FD_MEM_LHEAP respectively) will reside in the raw + * data free space manager ring as well + * + * With page allocation, the situation is conceptually + * identical, but more complex in practice. + * + * In the worst case (multi file driver) page allocation + * can result in two free space managers for each memory + * type -- one for small (less than on equal to one page) + * allocations, and one for large (greater than one page) + * allocations. + * + * In the more common one file case, page allocation will + * result in a total of three free space managers -- one for + * small (<= one page) raw data allocations, one for small + * metadata allocations (i.e, all memory types other than + * H5FD_MEM_DRAW), and one for all large (> one page) + * allocations. + * + * Despite these complications, the solution is the same in + * the page allocation case -- free space managers (be they + * small data or large) are assigned to the raw data free + * space manager ring if they don't allocate file space for + * free space managers. Note that in the one file case, the + * large free space manager must be assigned to the metadata + * free space manager ring, as it both allocates pages for + * the metadata free space manager, and allocates space for + * large (> 1 page) metadata cache entries. + * + * At present, the task list for this routine is: * * 1) Reduce the EOA to the extent possible. To do this: * @@ -1320,26 +2758,43 @@ done: herr_t H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) { + int pass_count; + hsize_t alloc_size; + H5F_mem_t mem_type; /* Memory type for iteration */ + H5F_mem_page_t fsm_type; /* FSM type for iteration */ + H5O_fsinfo_t fsinfo; /* Free space manager info message */ + H5FS_stat_t fs_stat; /* Information for free-space manager */ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ + H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration */ hbool_t reset_ring = FALSE; /* Whether the ring was set */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) - /* check args */ + /* Check args */ HDassert(f); HDassert(f->shared); HDassert(fsm_settled); - /* Only need to settle things if we are persisting the free space info */ - if(f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) { - H5O_fsinfo_t fsinfo; /* Free space manager info message */ - H5FD_mem_t type; /* Memory type for iteration */ - H5AC_ring_t curr_ring = H5AC_RING_INV; /* Current ring value */ - H5AC_ring_t needed_ring = H5AC_RING_INV; /* Ring value needed for this iteration */ - hbool_t fsm_opened[H5FD_MEM_NTYPES]; /* State of FSM */ - hbool_t fsm_visited[H5FD_MEM_NTYPES]; /* State of FSM */ + /* Only need to settle things if we are persisting the free space info + * and allocation/deallocation has occurred. + */ + if(f->shared->fs_persist && !f->shared->first_alloc_dealloc) { + hbool_t fsm_opened[H5F_MEM_PAGE_NTYPES]; /* State of FSM */ + hbool_t fsm_visited[H5F_MEM_PAGE_NTYPES]; /* State of FSM */ + + /* Sanity check */ + HDassert(f->shared->sblock); + + /* should only be called if file is opened R/W */ + HDassert(H5F_INTENT(f) & H5F_ACC_RDWR); + + /* shouldn't be called unless we have a superblock supporting the + * superblock extension. + */ + HDassert(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2); /* Initialize fsm_opened and fsm_visited */ HDmemset(fsm_opened, 0, sizeof(fsm_opened)); @@ -1356,8 +2811,12 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) * * Note that while the raw data aggregator should not be restarted during * the close process, this need not be the case for the metadata aggregator. + * + * Note also that the aggregators will not exist if page aggregation + * is enabled -- skip this if so. */ - if(H5MF_free_aggrs(f, dxpl_id) < 0) + /* Vailin -- is this correct? */ + if(!H5F_PAGED_AGGR(f) && (H5MF_free_aggrs(f, dxpl_id) < 0)) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free aggregators") /* Set the ring type in the DXPL. In most cases, we will @@ -1379,7 +2838,7 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) * In the case of the raw data free space manager, and any other free * space manager that does not allocate space for free space managers, * allocations should be complete at this point, as all raw data should - * have space allocated and be flushed to file at this point. Thus we + * have space allocated and be flushed to file by now. Thus we * can examine such free space managers and only re-allocate space for * them if they contain free space. Do this later in this function after * the EOA has been reduced to the extent possible. @@ -1390,79 +2849,100 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) * are possible), the matter is more ticklish due to the self- * referential nature of the problem. These FSMs are dealt with in * H5MF_settle_meta_data_fsm(). + * + * Since paged allocation may be enabled, there may be up to two + * free space managers per memory type -- one for small and one for + * large allocation. Hence we must loop over the memory types twice + * setting the allocation size accordingly if paged allocation is + * enabled. */ - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - H5FS_stat_t fs_stat; /* Information for free-space manager */ - H5FD_mem_t fsm_type; /* File memory type for FSM */ - - /* There is potentially a many-to-one mapping from memory types to - * free space managers. Use the fsm_visited[] array to avoid visiting - * a given FSM more than once. Use fsm_opened[] to track which FSMs - * must be closed at the end of this function. - */ - fsm_type = H5MF_ALLOC_TO_FS_TYPE(f, type); - if(!fsm_visited[fsm_type]) { - fsm_visited[fsm_type] = TRUE; - - /* If there is no active FSM for this type, but such a FSM has - * space allocated in file, open it so that we can free its file - * space. - */ - if(NULL == f->shared->fs_man[fsm_type]) { - if(H5F_addr_defined(f->shared->fs_addr[fsm_type])) { - /* Sanity check */ - HDassert(fsm_opened[fsm_type] == FALSE); - - /* Start up FSM for the file memory type */ - if(H5MF__alloc_open(f, dxpl_id, fsm_type) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") - fsm_opened[fsm_type] = TRUE; - } /* end if */ + for(pass_count = 0; pass_count <= 1; pass_count++) { + if(pass_count == 0) + alloc_size = 1; + else if ( H5F_PAGED_AGGR(f) ) + alloc_size = f->shared->fs_page_size + 1; + else /* no need for a second pass */ + break; + + for(mem_type = H5FD_MEM_SUPER; mem_type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5F_mem_t, mem_type)) { + H5MF_alloc_to_fs_type(f, mem_type, alloc_size, &fsm_type); + + if(pass_count == 0) { /* this is the first pass */ + HDassert(fsm_type > H5F_MEM_PAGE_DEFAULT); + HDassert(fsm_type < H5F_MEM_PAGE_LARGE_SUPER); } /* end if */ + else if(H5F_PAGED_AGGR(f)) { /* page alloc active */ + HDassert(fsm_type >= H5F_MEM_PAGE_LARGE_SUPER); + HDassert(fsm_type < H5F_MEM_PAGE_NTYPES); + } /* end else-if */ + else /* paged allocation disabled -- should be unreachable */ + HDassert(FALSE); + + if(!fsm_visited[fsm_type]) { + fsm_visited[fsm_type] = TRUE; + + /* If there is no active FSM for this type, but such a FSM has + * space allocated in file, open it so that we can free its file + * space. + */ + if(NULL == f->shared->fs_man[fsm_type]) { + if(H5F_addr_defined(f->shared->fs_addr[fsm_type])) { + /* Sanity check */ + HDassert(fsm_opened[fsm_type] == FALSE); - /* Check for an actual FSM for this type now */ - /* (Possibly opened in previous step) */ - if(f->shared->fs_man[fsm_type]) { - /* Test to see if we need to switch rings -- do so if required */ - if((fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") - curr_ring = needed_ring; + if(H5MF_open_fstype(f, dxpl_id, fsm_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") + fsm_opened[fsm_type] = TRUE; + } /* end if */ } /* end if */ - /* Query free space manager info for this type */ - if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") + if(f->shared->fs_man[fsm_type]) { + /* Test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, fsm_type)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; + + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + curr_ring = needed_ring; + } /* end if */ - /* Check if the free space manager has space in the file */ - if(H5F_addr_defined(fs_stat.addr) || H5F_addr_defined(fs_stat.sect_addr)) { - /* Delete the free space manager in the file. Will - * reallocate later if the free space manager contains - * any free space. - */ - if(H5FS_free(f, f->shared->fs_man[fsm_type], dxpl_id) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers") - f->shared->fs_addr[fsm_type] = HADDR_UNDEF; + /* Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") + + /* Check if the free space manager has space in the file */ + if(H5F_addr_defined(fs_stat.addr) || H5F_addr_defined(fs_stat.sect_addr)) { + /* Delete the free space manager in the file. Will + * reallocate later if the free space manager contains + * any free space. + */ + if(H5FS_free(f, f->shared->fs_man[fsm_type], dxpl_id, TRUE) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers") + f->shared->fs_addr[fsm_type] = HADDR_UNDEF; + } /* end if */ } /* end if */ - } /* end if */ - /* note that we are tracking opened FSM -- we will close them - * at the end of the function. - */ - } /* end if */ + /* note that we are tracking opened FSM -- we will close them + * at the end of the function. + */ + } /* end if */ + } /* end for */ } /* end for */ + /* c) Delete the free space manager superblock extension message * if allocated. * * Must do this since the routine that writes / creates superblock * extension messages will choke if the target message is * unexpectedly either absent or present. + * + * Update: This is probably unecessary, as I gather that the + * file space manager info message is guaranteed to exist. + * Leave it in for now, but consider removing it. */ if(H5F_addr_defined(f->shared->sblock->ext_addr)) if(H5F_super_ext_remove_msg(f, dxpl_id, H5O_FSINFO_ID) < 0) @@ -1486,11 +2966,16 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) * those addresses are unknown. This is OK -- we will write the correct * values to the message at free space manager shutdown. */ - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - fsinfo.fs_addr[type - 1] = HADDR_UNDEF; + for(fsm_type = H5F_MEM_PAGE_SUPER; fsm_type < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, fsm_type)) + fsinfo.fs_addr[fsm_type - 1] = HADDR_UNDEF; fsinfo.strategy = f->shared->fs_strategy; + fsinfo.persist = f->shared->fs_persist; fsinfo.threshold = f->shared->fs_threshold; - if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE, H5O_MSG_NO_FLAGS_SET) < 0) + fsinfo.page_size = f->shared->fs_page_size; + fsinfo.pgend_meta_thres = f->shared->pgend_meta_thres; + fsinfo.eoa_pre_fsm_fsalloc = HADDR_UNDEF; + + if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, TRUE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing fsinfo message to superblock extension") @@ -1514,102 +2999,117 @@ H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) */ /* Reinitialize fsm_visited */ - HDmemset(fsm_visited, 0, sizeof(fsm_visited)); - - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - H5FD_mem_t fsm_type; /* File memory type for FSM */ - - fsm_type = H5MF_ALLOC_TO_FS_TYPE(f, type); - - /* test to see if we need to switch rings -- do so if required */ - if((fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (fsm_type == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; - - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring)< 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") - curr_ring = needed_ring; - } /* end if */ - - /* Since there can be a many-to-one mapping from memory types - * to free space managers, ensure that we don't visit any FSM - * more than once. - */ - if(!fsm_visited[fsm_type]) { - fsm_visited[fsm_type] = TRUE; - - if(f->shared->fs_man[fsm_type]) { - /* Only allocate file space if the target free space manager - * doesn't allocate file space for free space managers. Note - * that this is also the deciding factor as to whether a FSM - * in in the raw data FSM ring. - */ - if((fsm_type != H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - && (fsm_type != H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) { - H5FS_stat_t fs_stat; /* Information for free-space manager */ + for(fsm_type = H5F_MEM_PAGE_SUPER; fsm_type < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, fsm_type)) + fsm_visited[fsm_type] = FALSE; + + for(pass_count = 0; pass_count <= 1; pass_count++) { + if(pass_count == 0) + alloc_size = 1; + else if(H5F_PAGED_AGGR(f)) + alloc_size = f->shared->fs_page_size + 1; + else /* no need for a second pass */ + break; + + for(mem_type = H5FD_MEM_SUPER; mem_type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5F_mem_t, mem_type)) { + H5MF_alloc_to_fs_type(f, mem_type, alloc_size, &fsm_type); + + if(pass_count == 0) { /* this is the first pass */ + HDassert(fsm_type > H5F_MEM_PAGE_DEFAULT); + HDassert(fsm_type < H5F_MEM_PAGE_LARGE_SUPER); + } /* end if */ + else if(H5F_PAGED_AGGR(f)) { /* page alloc active */ + HDassert(fsm_type >= H5F_MEM_PAGE_LARGE_SUPER); + HDassert(fsm_type < H5F_MEM_PAGE_NTYPES); + } /* end else-if */ + else /* paged allocation disabled -- should be unreachable */ + HDassert(FALSE); - /* The current ring should be H5AC_RING_RDFSM */ - HDassert(curr_ring == H5AC_RING_RDFSM); + /* test to see if we need to switch rings -- do so if required */ + if(H5MF__fsm_type_is_self_referential(f, fsm_type)) + needed_ring = H5AC_RING_MDFSM; + else + needed_ring = H5AC_RING_RDFSM; - /* Query free space manager info for this type */ - if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0 ) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") + if(needed_ring != curr_ring) { + if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring)< 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") + curr_ring = needed_ring; + } /* end if */ - /* If the free space manager contains section info, - * allocate space for the header and sinfo (note that - * space must not be allocated at present -- verify - * verify this with assertions). + /* Since there can be a many-to-one mapping from memory types + * to free space managers, ensure that we don't visit any FSM + * more than once. + */ + if(!fsm_visited[fsm_type]) { + fsm_visited[fsm_type] = TRUE; + + if(f->shared->fs_man[fsm_type]) { + /* Only allocate file space if the target free space manager + * doesn't allocate file space for free space managers. Note + * that this is also the deciding factor as to whether a FSM + * in in the raw data FSM ring. */ - if(fs_stat.serial_sect_count > 0) { - /* Sanity check */ - HDassert(!H5F_addr_defined(fs_stat.addr)); - - /* Allocate FSM header */ - if(H5FS_alloc_hdr(f, f->shared->fs_man[fsm_type], &f->shared->fs_addr[fsm_type], dxpl_id) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocated free-space header") - - /* Allocate FSM section info */ - HDassert(!H5F_addr_defined(fs_stat.sect_addr)); - HDassert(fs_stat.alloc_sect_size == 0); - if(H5FS_alloc_sect(f, f->shared->fs_man[fsm_type], dxpl_id) < 0 ) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate free-space section info") + if(!H5MF__fsm_type_is_self_referential(f, fsm_type)) { + /* The current ring should be H5AC_RING_RDFSM */ + HDassert(curr_ring == H5AC_RING_RDFSM); + + /* Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0 ) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") + + /* If the free space manager contains section info, + * allocate space for the header and sinfo (note that + * space must not be allocated at present -- verify + * verify this with assertions). + */ + if(fs_stat.serial_sect_count > 0) { + /* Sanity check */ + HDassert(!H5F_addr_defined(fs_stat.addr)); + + /* Allocate FSM header */ + if(H5FS_alloc_hdr(f, f->shared->fs_man[fsm_type], &f->shared->fs_addr[fsm_type], dxpl_id) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocated free-space header") + + /* Allocate FSM section info */ + HDassert(!H5F_addr_defined(fs_stat.sect_addr)); + HDassert(fs_stat.alloc_sect_size == 0); + if(H5FS_alloc_sect(f, f->shared->fs_man[fsm_type], dxpl_id) < 0 ) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate free-space section info") #ifndef NDEBUG - /* Re-Query free space manager info for this type */ - if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") - - HDassert(H5F_addr_defined(fs_stat.addr)); - HDassert(H5F_addr_defined(fs_stat.sect_addr)); - HDassert(fs_stat.serial_sect_count > 0); - HDassert(fs_stat.alloc_sect_size > 0); - HDassert(fs_stat.alloc_sect_size == fs_stat.sect_size); + /* Re-Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[fsm_type], &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") + + HDassert(H5F_addr_defined(fs_stat.addr)); + HDassert(H5F_addr_defined(fs_stat.sect_addr)); + HDassert(fs_stat.serial_sect_count > 0); + HDassert(fs_stat.alloc_sect_size > 0); + HDassert(fs_stat.alloc_sect_size == fs_stat.sect_size); #endif /* NDEBUG */ + } /* end if */ + else { + HDassert(!H5F_addr_defined(fs_stat.addr)); + HDassert(!H5F_addr_defined(fs_stat.sect_addr)); + HDassert(fs_stat.serial_sect_count == 0); + HDassert(fs_stat.alloc_sect_size == 0); + } /* end else */ } /* end if */ - else { - HDassert(!H5F_addr_defined(fs_stat.addr)); - HDassert(!H5F_addr_defined(fs_stat.sect_addr)); - HDassert(fs_stat.serial_sect_count == 0); - HDassert(fs_stat.alloc_sect_size == 0); - } /* end else */ } /* end if */ - } /* end if */ - /* Close any opened FSMs */ - if(fsm_opened[fsm_type]) { - if(H5MF__alloc_close(f, dxpl_id, fsm_type) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't close file free space manager") - fsm_opened[fsm_type] = FALSE; + /* Close any opened FSMs */ + if(fsm_opened[fsm_type]) { + if(H5MF__close_fstype(f, dxpl_id, fsm_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't close file free space manager") + fsm_opened[fsm_type] = FALSE; + } /* end if */ } /* end if */ - } /* end if */ + } /* end for */ } /* end for */ /* verify that all opened FSMs were closed */ - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - HDassert(!fsm_opened[type]); + for(fsm_type = H5F_MEM_PAGE_SUPER; fsm_type < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, fsm_type)) + HDassert(!fsm_opened[fsm_type]); /* Indicate that the FSM was settled successfully */ *fsm_settled = TRUE; @@ -1630,11 +3130,11 @@ done: * * Purpose: If the free space manager is persistent, handle any tasks * required before the metadata cache can serialize or flush - * the metadata free space manager(sI) that handle file space + * the metadata free space manager(s) that handle file space * allocation for free space managers. * * In most cases, there will be only one manager assigned - * to this role. However, since for reason or reason unknown, + * to this role. However, since for reasons unknown, * free space manager headers and section info blocks are * different classes of memory, it is possible that two free * space managers will be involved. @@ -1642,7 +3142,7 @@ done: * On entry to this function, the raw data settle routine * (H5MF_settle_raw_data_fsm()) should have: * - * 1) Freed the accumulators. + * 1) Freed the aggregators. * * 2) Freed all file space allocated to the free space managers. * @@ -1678,9 +3178,12 @@ done: * 1) Verify that the free space manager(s) involved in file * space allocation for free space managers are still floating. * - * 2) Free the accumulators. + * 2) Free the aggregators. * - * 3) Reduce the EOA to the extent possible. + * 3) Reduce the EOA to the extent possible, and make note + * of the resulting value. This value will be stored + * in the fsinfo superblock extension message and be used + * in the subsequent file open. * * 4) Re-allocate space for any free space manager(s) that: * @@ -1714,6 +3217,9 @@ done: * severe time pressure at the moment, the above brute * force solution is attractive. * + * 5) Make note of the EOA -- used for sanity checking on + * FSM shutdown. + * * Return: SUCCEED/FAIL * * Programmer: John Mainzer @@ -1724,10 +3230,23 @@ done: herr_t H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) { + H5F_mem_page_t sm_fshdr_fs_type; /* small fs hdr fsm */ + H5F_mem_page_t sm_fssinfo_fs_type; /* small fs sinfo fsm */ + H5F_mem_page_t lg_fshdr_fs_type; /* large fs hdr fsm */ + H5F_mem_page_t lg_fssinfo_fs_type; /* large fs sinfo fsm */ + H5FS_t *sm_hdr_fspace = NULL; /* ptr to sm FSM hdr alloc FSM */ + H5FS_t *sm_sinfo_fspace = NULL; /* ptr to sm FSM sinfo alloc FSM */ + H5FS_t *lg_hdr_fspace = NULL; /* ptr to lg FSM hdr alloc FSM */ + H5FS_t *lg_sinfo_fspace = NULL; /* ptr to lg FSM sinfo alloc FSM */ + haddr_t eoa_pre_fsm_fsalloc; /* eoa pre file space allocation */ + /* for self referential FSMs */ + haddr_t eoa_post_fsm_fsalloc; /* eoa post file space allocation */ + /* for self referential FSMs */ + H5FS_stat_t fs_stat; /* Information for hdr FSM */ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ hbool_t reset_ring = FALSE; /* Whether we set the ring */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) @@ -1736,29 +3255,49 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) HDassert(f->shared); HDassert(fsm_settled); - /* Only need to settle things if we are persisting the free space info */ - if(f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) { - H5FS_t *hdr_fspace; /* Ptr to FSM hdr alloc FSM */ - H5FS_t *sinfo_fspace; /* Ptr to FSM sinfo alloc FSM */ - H5FS_stat_t fs_stat; /* Information for FSM */ - H5FD_mem_t hdr_fsm_alloc_type; /* FSM hdr alloc type */ - H5FD_mem_t sinfo_fsm_alloc_type; /* FSM info alloc type */ - - /* Get the file memory types for the FSM header & section info */ - hdr_fsm_alloc_type = H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR); - sinfo_fsm_alloc_type = H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO); - - /* Sanity checks */ - HDassert(hdr_fsm_alloc_type > H5FD_MEM_NOLIST); - HDassert(hdr_fsm_alloc_type < H5FD_MEM_NTYPES); - HDassert(sinfo_fsm_alloc_type > H5FD_MEM_NOLIST); - HDassert(sinfo_fsm_alloc_type < H5FD_MEM_NTYPES); - HDassert(!H5F_addr_defined(f->shared->fs_addr[hdr_fsm_alloc_type])); - HDassert(!H5F_addr_defined(f->shared->fs_addr[sinfo_fsm_alloc_type])); - - /* Note that in most cases, hdr_fspace will equal sinfo_fspace. */ - hdr_fspace = f->shared->fs_man[hdr_fsm_alloc_type]; - sinfo_fspace = f->shared->fs_man[sinfo_fsm_alloc_type]; + /* Only need to settle things if we are persisting the free space info + * and allocation/deallocation has occurred. + */ + if(f->shared->fs_persist && !f->shared->first_alloc_dealloc) { + /* Sanity check */ + HDassert(f->shared->lf); + + /* should only be called if file is opened R/W */ + HDassert(H5F_INTENT(f) & H5F_ACC_RDWR); + + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fs_type); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fs_type); + + HDassert(sm_fshdr_fs_type > H5F_MEM_PAGE_DEFAULT); + HDassert(sm_fshdr_fs_type < H5F_MEM_PAGE_LARGE_SUPER); + + HDassert(sm_fssinfo_fs_type > H5F_MEM_PAGE_DEFAULT); + HDassert(sm_fssinfo_fs_type < H5F_MEM_PAGE_LARGE_SUPER); + + HDassert(!H5F_addr_defined(f->shared->fs_addr[sm_fshdr_fs_type])); + HDassert(!H5F_addr_defined(f->shared->fs_addr[sm_fssinfo_fs_type])); + + /* Note that in most cases, sm_hdr_fspace will equal sm_sinfo_fspace. */ + sm_hdr_fspace = f->shared->fs_man[sm_fshdr_fs_type]; + sm_sinfo_fspace = f->shared->fs_man[sm_fssinfo_fs_type]; + + if(H5F_PAGED_AGGR(f)) { + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fs_type); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fs_type); + + HDassert(lg_fshdr_fs_type >= H5F_MEM_PAGE_LARGE_SUPER); + HDassert(lg_fshdr_fs_type < H5F_MEM_PAGE_NTYPES); + + HDassert(lg_fssinfo_fs_type >= H5F_MEM_PAGE_LARGE_SUPER); + HDassert(lg_fssinfo_fs_type < H5F_MEM_PAGE_NTYPES); + + HDassert(!H5F_addr_defined(f->shared->fs_addr[lg_fshdr_fs_type])); + HDassert(!H5F_addr_defined(f->shared->fs_addr[lg_fssinfo_fs_type])); + + /* Note that in most cases, lg_hdr_fspace will equal lg_sinfo_fspace. */ + lg_hdr_fspace = f->shared->fs_man[lg_fshdr_fs_type]; + lg_sinfo_fspace = f->shared->fs_man[lg_fssinfo_fs_type]; + } /* end if */ /* Set the ring in the dxpl appropriately for subsequent calls */ if(H5AC_set_ring(dxpl_id, H5AC_RING_MDFSM, &dxpl, &orig_ring) < 0) @@ -1766,41 +3305,109 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) reset_ring = TRUE; #ifndef NDEBUG - /* Verify that hdr_fspace is floating if it exists */ - if(hdr_fspace) { + /* Verify that sm_hdr_fspace is floating if it exists */ + if(sm_hdr_fspace) { /* Query free space manager info for this type */ - if(H5FS_stat_info(f, hdr_fspace, &fs_stat) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") + if(H5FS_stat_info(f, sm_hdr_fspace, &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") HDassert(!H5F_addr_defined(fs_stat.addr)); HDassert(!H5F_addr_defined(fs_stat.sect_addr)); HDassert(fs_stat.alloc_sect_size == 0); } /* end if */ - /* Verify that sinfo_fspace is floating if it exists */ - if((sinfo_fspace) && (hdr_fspace != sinfo_fspace)) { + /* Verify that sm_sinfo_fspace is floating if it exists and is distinct */ + if((sm_sinfo_fspace) && (sm_hdr_fspace != sm_sinfo_fspace)) { /* Query free space manager info for this type */ - if(H5FS_stat_info(f, sinfo_fspace, &fs_stat) < 0) + if(H5FS_stat_info(f, sm_sinfo_fspace, &fs_stat) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") HDassert(!H5F_addr_defined(fs_stat.addr)); HDassert(!H5F_addr_defined(fs_stat.sect_addr)); HDassert(fs_stat.alloc_sect_size == 0); } /* end if */ + + if(H5F_PAGED_AGGR(f)) { + /* Verify that lg_hdr_fspace is floating if it exists */ + if(lg_hdr_fspace) { + /* Query free space manager info for this type */ + if(H5FS_stat_info(f, lg_hdr_fspace, &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info (3)") + + HDassert(!H5F_addr_defined(fs_stat.addr)); + HDassert(!H5F_addr_defined(fs_stat.sect_addr)); + HDassert(fs_stat.alloc_sect_size == 0); + } /* end if */ + + /* Verify that lg_sinfo_fspace is floating if it + * exists and is distinct + */ + if((lg_sinfo_fspace) && (lg_hdr_fspace != lg_sinfo_fspace)) { + /* Query free space manager info for this type */ + if(H5FS_stat_info(f, lg_sinfo_fspace, &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info (4)") + + HDassert(!H5F_addr_defined(fs_stat.addr)); + HDassert(!H5F_addr_defined(fs_stat.sect_addr)); + HDassert(fs_stat.alloc_sect_size == 0); + } /* end if */ + } /* end if */ #endif /* NDEBUG */ /* Free the space in the metadata aggregator. Do this via the * H5MF_free_aggrs() call. Note that the raw data aggregator must * have already been freed. Sanity checks for this? + * + * Note that the aggregators will not exist if paged aggregation + * is enabled -- don't attempt to free if this is the case. */ + /* Vailin -- is this correct? */ /* (for space not at EOF, it may be put into free space managers) */ - if(H5MF_free_aggrs(f, dxpl_id) < 0) + if((!H5F_PAGED_AGGR(f)) && (H5MF_free_aggrs(f, dxpl_id) < 0)) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free aggregators") /* Trying shrinking the EOA for the file */ if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") + /* At this point, the EOA should be set to a value that contains + * the allocation for all user data, all non self referential FSMs, + * the superblock and all superblock extension messages. + * + * Make note of the current EOA. We will store this value in the + * free space manager superblock extension message. Since space for + * everything other than the self referential FSMs (and possibly the + * cache image) has been allocated at this point, this allows us to + * to float the self referential FSMs on the first file space allocation / + * deallocaiton and then set the EOA to this value before we handle + * the allocation / deallocation. (If a cache image exists, the + * first allocation / deallocation will be the deallocation of space + * for the cache image). + * + * WARNING: This approach settling the self referential free space + * managers and allocating space for them in the file will + * not work as currently implemented with the split and + * multi file drivers, as the self referential free space + * manager header and section info can be stored in up to + * two different files -- requiring that up to two EOA's + * be stored in the the free space managers super block + * extension message. + * + * As of this writing, we are solving this problem by + * simply not supporting persistant FSMs with the split + * and multi file drivers. + * + * Current plans are to do away with the multi file + * driver, so this should be a non-issue in this case. + * + * We should be able to support the split file driver + * without a file format change. However, the code to + * do so does not exist at present. + */ + if(HADDR_UNDEF == (eoa_pre_fsm_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT))) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get EOA") + + /* ******************* PROBLEM: ******************** * * If the file has an alignement other than 1, and if @@ -1849,15 +3456,39 @@ H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled) * This is isn't good, but due to schedule pressure, we will just drop * the fragment on the floor for now. */ - if(hdr_fspace) - if(H5FS_alloc_vfd_alloc_hdr_and_section_info(f, dxpl_id, hdr_fspace, - &(f->shared->fs_addr[hdr_fsm_alloc_type])) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate hdr FSM file space") + if(sm_hdr_fspace) + if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, dxpl_id, sm_hdr_fspace, &(f->shared->fs_addr[sm_fshdr_fs_type])) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sm hdr FSM file space") + + if(sm_sinfo_fspace && (sm_sinfo_fspace != sm_hdr_fspace)) + if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, dxpl_id, sm_sinfo_fspace, &(f->shared->fs_addr[sm_fssinfo_fs_type])) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sm sinfo FSM file space") + + if(H5F_PAGED_AGGR(f)) { + if(lg_hdr_fspace) + if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, dxpl_id, lg_hdr_fspace, &(f->shared->fs_addr[lg_fshdr_fs_type])) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg hdr FSM file space") + + if(lg_sinfo_fspace && (lg_sinfo_fspace != lg_hdr_fspace)) + if(H5FS_vfd_alloc_hdr_and_section_info_if_needed(f, dxpl_id, lg_sinfo_fspace, &(f->shared->fs_addr[lg_fssinfo_fs_type])) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate lg sinfo FSM file space") + } /* end if */ - if(sinfo_fspace && (sinfo_fspace != hdr_fspace)) - if(H5FS_alloc_vfd_alloc_hdr_and_section_info(f, dxpl_id, sinfo_fspace, - &(f->shared->fs_addr[sinfo_fsm_alloc_type])) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't vfd allocate sinfo FSM file space") + /* Get the eoa after allocation of file space for the self referential + * free space managers. Assuming no cache image, this should be the + * final EOA of the file. + */ + if(HADDR_UNDEF == (eoa_post_fsm_fsalloc = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT))) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get file size") + + /* All free space managers should have file space allocated for them + * now, and should see no further allocations / deallocations. Store + * the pre and post file space allocaton for self referential FSMs EOA + * for use when we actually write the free space manager superblock + * extension message. + */ + f->shared->eoa_pre_fsm_fsalloc = eoa_pre_fsm_fsalloc; + f->shared->eoa_post_fsm_fsalloc = eoa_post_fsm_fsalloc; /* Indicate that the FSM was settled successfully */ *fsm_settled = TRUE; @@ -1868,479 +3499,410 @@ done: if(H5AC_reset_ring(dxpl, orig_ring) < 0) HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") - FUNC_LEAVE_NOAPI(ret_value) + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) } /* H5MF_settle_meta_data_fsm() */ /*------------------------------------------------------------------------- - * Function: H5MF__close_delete + * Function: H5MF__fsm_type_is_self_referential() * - * Purpose: Common code for closing and deleting freespace managers from - * the file. + * Purpose: Return TRUE if the indicated free space manager allocates + * file space for free space managers. Return FALSE otherwise. * - * Return: SUCCEED/FAIL + * Return: TRUE/FALSE * - * Programmer: Vailin Choi - * Jan 2016 + * Programmer: John Mainzer + * 12/6/16 * *------------------------------------------------------------------------- */ -static herr_t -H5MF__close_delete(H5F_t *f, hid_t dxpl_id, H5P_genplist_t **dxpl) +hbool_t +H5MF__fsm_type_is_self_referential(H5F_t *f, H5F_mem_page_t fsm_type) { - H5FD_mem_t type; /* Memory type for iteration */ - H5AC_ring_t curr_ring = H5AC_RING_INV; /* current ring value */ - herr_t ret_value = SUCCEED; /* Return value */ + H5F_mem_page_t sm_fshdr_fsm; + H5F_mem_page_t sm_fssinfo_fsm; + H5F_mem_page_t lg_fshdr_fsm; + H5F_mem_page_t lg_fssinfo_fsm; + hbool_t result = FALSE; - FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Entering\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG */ + FUNC_ENTER_NOAPI_NOINIT_NOERR - /* check args */ + /* Sanity check */ HDassert(f); HDassert(f->shared); + HDassert(fsm_type >= H5F_MEM_PAGE_DEFAULT); + HDassert(fsm_type < H5F_MEM_PAGE_NTYPES); - /* Iterate over all the free space types that have managers and get each free list's space */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - H5AC_ring_t needed_ring; /* Ring value needed for this iteration */ + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fsm); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fsm); -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 1.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]); -#endif /* H5MF_ALLOC_DEBUG_MORE */ + if(H5F_PAGED_AGGR(f)) { + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fsm); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fsm); - /* test to see if we need to switch rings -- do so if required */ - if((H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; + result = (fsm_type == sm_fshdr_fsm) || (fsm_type == sm_fssinfo_fsm) + || (fsm_type == lg_fshdr_fsm) || (fsm_type == lg_fssinfo_fsm); + } /* end if */ + else { + /* In principle, fsm_type should always be less than + * H5F_MEM_PAGE_LARGE_SUPER whenever paged aggregation + * is not enabled. However, since there is code that does + * not observe this prinicple, force the result to FALSE if + * fsm_type is greater than or equal to H5F_MEM_PAGE_LARGE_SUPER. + */ + if(fsm_type >= H5F_MEM_PAGE_LARGE_SUPER) + result = FALSE; else - needed_ring = H5AC_RING_RDFSM; - - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (4)") - curr_ring = needed_ring; - } /* end if */ - - /* If the free space manager for this type is open, close it */ - if(f->shared->fs_man[type]) { -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Before closing free space manager\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free space info") - f->shared->fs_man[type] = NULL; - f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; - } /* end if */ -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Check 2.0 - f->shared->fs_man[%u] = %p, f->shared->fs_addr[%u] = %a\n", FUNC, (unsigned)type, f->shared->fs_man[type], (unsigned)type, f->shared->fs_addr[type]); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - - /* If there is free space manager info for this type, delete it */ - if(H5F_addr_defined(f->shared->fs_addr[type])) { - haddr_t tmp_fs_addr; /* Temporary holder for free space manager address */ - - /* Put address into temporary variable and reset it */ - /* (Avoids loopback in file space freeing routine) */ - tmp_fs_addr = f->shared->fs_addr[type]; - f->shared->fs_addr[type] = HADDR_UNDEF; - - /* Shift to "deleting" state, to make certain we don't track any - * file space freed as a result of deleting the free space manager. - */ - f->shared->fs_state[type] = H5F_FS_STATE_DELETING; - -#ifdef H5MF_ALLOC_DEBUG_MORE -HDfprintf(stderr, "%s: Before deleting free space manager\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG_MORE */ - - /* Delete free space manager for this type */ - if(H5FS_delete(f, dxpl_id, tmp_fs_addr) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't delete free space manager") - - /* Shift [back] to closed state */ - HDassert(f->shared->fs_state[type] == H5F_FS_STATE_DELETING); - f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; - - /* Sanity check that the free space manager for this type wasn't started up again */ - HDassert(!H5F_addr_defined(f->shared->fs_addr[type])); - } /* end if */ - } /* end for */ + result = (fsm_type == sm_fshdr_fsm) || (fsm_type == sm_fssinfo_fsm); + } /* end else */ -done: -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Leaving\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG */ - FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* H5MF__close_delete() */ + FUNC_LEAVE_NOAPI(result) +} /* H5MF__fsm_type_is_self_referential() */ /*------------------------------------------------------------------------- - * Function: H5MF_try_close + * Function: H5MF__fsm_is_self_referential() * - * Purpose: This is called by H5Fformat_convert() to close and delete - * free-space managers when downgrading persistent free-space - * to non-persistent. + * Purpose: Return TRUE if the indicated free space manager allocates + * file space for free space managers. Return FALSE otherwise. * - * Return: SUCCEED/FAIL + * Return: TRUE/FALSE * - * Programmer: Vailin Choi - * Jan 2016 + * Programmer: John Mainzer + * 12/6/16 * *------------------------------------------------------------------------- */ -herr_t -H5MF_try_close(H5F_t *f, hid_t dxpl_id) +static hbool_t +H5MF__fsm_is_self_referential(H5F_t *f, H5FS_t *fspace) { - H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - hbool_t reset_ring = FALSE; /* Whether the ring was set */ - herr_t ret_value = SUCCEED; /* Return value */ + H5F_mem_page_t sm_fshdr_fsm; + H5F_mem_page_t sm_fssinfo_fsm; + hbool_t result = FALSE; - FUNC_ENTER_NOAPI(FAIL) -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Entering\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG */ + FUNC_ENTER_STATIC_NOERR - /* check args */ + /* Sanity check */ HDassert(f); + HDassert(f->shared); + HDassert(fspace); - /* Set the ring type in the DXPL */ - if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") - reset_ring = TRUE; + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fsm); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fsm); - /* Close and delete freespace managers from the file */ - if(H5MF__close_delete(f, dxpl_id, &dxpl) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to close delete free-space managers") + if(H5F_PAGED_AGGR(f)) { + H5F_mem_page_t lg_fshdr_fsm; + H5F_mem_page_t lg_fssinfo_fsm; -done: - /* Reset the ring in the DXPL */ - if(reset_ring) - if(H5AC_reset_ring(dxpl, orig_ring) < 0) - HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fsm); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fsm); -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Leaving\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG */ - FUNC_LEAVE_NOAPI(ret_value) -} /* H5MF_try_close() */ + result = (fspace == f->shared->fs_man[sm_fshdr_fsm]) || + (fspace == f->shared->fs_man[sm_fssinfo_fsm]) || + (fspace == f->shared->fs_man[lg_fshdr_fsm]) || + (fspace == f->shared->fs_man[lg_fssinfo_fsm]); + } /* end if */ + else + result = (fspace == f->shared->fs_man[sm_fshdr_fsm]) || + (fspace == f->shared->fs_man[sm_fssinfo_fsm]); + + FUNC_LEAVE_NOAPI(result) +} /* H5MF__fsm_is_self_referential() */ /*------------------------------------------------------------------------- - * Function: H5MF_close - * - * Purpose: Close the free space tracker(s) for a file - * - * Return: SUCCEED/FAIL + * Function: H5MF_tidy_self_referential_fsm_hack + * + * Purpose: As discussed in the comments of the settle routines above, + * the existence of self referential free space managers + * as currently implemented creates the possibility of + * infinite loops at file close. + * + * As a hack to avoid this, we have added code to settle + * self referential free space managers, and then allocate + * space for them directly from the file driver. + * + * To avoid dropping ever increasing amounts of file space + * on the floor with each subsequent file close/open cycle, + * we need to clean this up on file open. To avoid this, + * this function is called on the first file space allocation + * or deallocation after file open to float the self referential + * free space managers and reduce the EOA to the value it + * had before the direct allocation of space for the self + * referential free space managers. + * + * The function proceeds as follows: + * + * 1) Verify that f->shared->first_alloc_dealloc is TRUE, + * and then set it to FALSE. + * + * 2) Get the current EOA. Verify that it is greater than + * or equal to f->shared->eoa_pre_fsm_fsalloc. If the + * current eoa is equal to f->shared->eoa_pre_fsm_fsalloc, + * no self referential FSMs were stored, and we are done. + * + * NOTE: This will have to be reworked somewhat for + * cache image. + * + * 3) Load the self referential FSMs. In passing verify that + * the lowest address of a FSM header is equal to + * f->shared->eoa_pre_fsm_fsalloc.' + * + * Note that we don't have to use any special I/O for + * this -- we can use the regular I/O methods even if + * paged aggregation and page buffering is enabled. + * + * 4) Float the FSMs. Ensure that the file space is NOT + * released. + * + * 5) Set EOA equal to f->shared->eoa_pre_fsm_fsalloc, + * and then set f->shared->eoa_pre_fsm_fsalloc to + * HADDR_UNDEF. + * + * If page buffering, verify that the new EOA is + * on a page boundary, and expunge any pages in the + * page buffer after the new EOA. + * + * Note that this function is also called from test code + * when it is necessary to startup a self referential + * free space manager prior to the first file space + * allocation / deallocation. Failure to do so will + * result in assertion failures in this function on + * the first file space allocation / deallocation. + * + * Return: SUCCEED/FAIL * - * Programmer: Quincey Koziol - * Tuesday, January 22, 2008 + * Programmer: John Mainzer + * 12/11/16 * - * Modifications: - * Vailin Choi; July 2012 - * As the default free-list mapping is changed to H5FD_FLMAP_DICHOTOMY, - * modifications are needed to shrink EOA if the last section of each free-space manager - * and the remaining space in the two aggregators are at EOA. - *------------------------------------------------------------------------- */ herr_t -H5MF_close(H5F_t *f, hid_t dxpl_id) +H5MF_tidy_self_referential_fsm_hack(H5F_t *f, hid_t dxpl_id) { + haddr_t eoa; /* EOA of file */ + hsize_t tail_size = 0; /* Size of chunk to free */ H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t curr_ring; /* Current ring value */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - H5FD_mem_t type; /* Memory type for iteration */ + H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ + haddr_t first_srfsm_hdr = HADDR_UNDEF; /* Addr of first self referential */ + /* fsm header in file */ + H5FS_stat_t fs_stat; /* Information for hdr FSM */ + H5F_mem_page_t sm_fshdr_fs_type; /* Small fs hdr fsm */ + H5F_mem_page_t sm_fssinfo_fs_type; /* Small fs sinfo fsm */ + H5F_mem_page_t lg_fshdr_fs_type; /* Large fs hdr fsm */ + H5F_mem_page_t lg_fssinfo_fs_type; /* Large fs sinfo fsm */ hbool_t reset_ring = FALSE; /* Whether the ring was set */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Entering\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG */ /* check args */ HDassert(f); HDassert(f->shared); - HDassert(f->shared->lf); - HDassert(f->shared->sblock); + HDassert(f->shared->fs_persist); + HDassert(f->shared->first_alloc_dealloc); - /* Set the ring type in the DXPL. In most cases, we will - * need H5AC_RING_RDFSM, so initialy set the ring in - * the DXPL to that value. We will alter this later if - * needed. + /* Set the ring type in the DXPL. Since we are only dealing with + * self referential FSMs, we will only need H5AC_RING_MDFSM. */ - if(H5AC_set_ring(dxpl_id, H5AC_RING_RDFSM, &dxpl, &orig_ring) < 0) + if(H5AC_set_ring(dxpl_id, H5AC_RING_MDFSM, &dxpl, &orig_ring) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") reset_ring = TRUE; - curr_ring = H5AC_RING_RDFSM; - /* Free the space in aggregators */ - /* (for space not at EOF, it may be put into free space managers) */ - if(H5MF_free_aggrs(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators") + /* 1) Verify that f->shared->first_alloc_dealloc is TRUE, + * and then set it to FALSE. + */ + HDassert(f->shared->first_alloc_dealloc); + f->shared->first_alloc_dealloc = FALSE; - /* Trying shrinking the EOA for the file */ - if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") - /* Making free-space managers persistent for superblock version >= 2 */ - if(f->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_2 - && f->shared->fs_strategy == H5F_FILE_SPACE_ALL_PERSIST) { - H5O_fsinfo_t fsinfo; /* Free space manager info message */ + /* 2) Get the current EOA. Verify that it is greater than + * or equal to f->shared->eoa_pre_fsm_fsalloc. If the + * current eoa is equal to f->shared->eoa_pre_fsm_fsalloc, + * no self referential FSMs were stored, and we are done. + * + * NOTE: This will have to be reworked somewhat for + * cache image. + */ + if(HADDR_UNDEF == (eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT))) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get EOA") + HDassert(H5F_addr_le(f->shared->eoa_pre_fsm_fsalloc, eoa)); - /* Superblock extension and free space manager message should - * exist at this point -- verify at least the former. - */ - HDassert(H5F_addr_defined(f->shared->sblock->ext_addr)); + if(H5F_addr_eq(f->shared->eoa_pre_fsm_fsalloc, eoa)) + HGOTO_DONE(SUCCEED) - /* Note that unlike the previous version of this code, we do not - * delete free space managers that have no section to store. - * - * Can't do this, as that would involve freeing file space, which would - * dirty the free space manager in question. - * - * Fortunately, the code doesn't seem to care about this. - */ - /* Gather data for the free space manager superblock extension message. - * - * In passing, verify that all the free space managers are closed. - */ - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - fsinfo.fs_addr[type - 1] = f->shared->fs_addr[type]; - fsinfo.strategy = f->shared->fs_strategy; - fsinfo.threshold = f->shared->fs_threshold; + /* 3) Load the self referential FSMs. In passing verify that + * the lowest address of a FSM header is equal to + * f->shared->eoa_pre_fsm_fsalloc.' + * + * Note that we don't have to use any special I/O for + * this -- we can use the regular I/O methods even if + * paged aggregation and page buffering is enabled. + */ + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, (size_t)1, &sm_fshdr_fs_type); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, (size_t)1, &sm_fssinfo_fs_type); + HDassert(sm_fshdr_fs_type > H5F_MEM_PAGE_DEFAULT); + HDassert(sm_fshdr_fs_type < H5F_MEM_PAGE_LARGE_SUPER); - /* Write the free space manager message -- message must already exist */ - if(H5F_super_ext_write_msg(f, dxpl_id, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_NO_FLAGS_SET) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_WRITEERROR, FAIL, "error in writing message to superblock extension") + HDassert(sm_fssinfo_fs_type > H5F_MEM_PAGE_DEFAULT); + HDassert(sm_fssinfo_fs_type < H5F_MEM_PAGE_LARGE_SUPER); - /* Final close of free-space managers */ - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - if(f->shared->fs_man[type]) { - H5AC_ring_t needed_ring; /* Ring value needed for this iteration */ + HDassert(NULL == f->shared->fs_man[sm_fshdr_fs_type]); + HDassert(NULL == f->shared->fs_man[sm_fssinfo_fs_type]); - /* test to see if we need to switch rings -- do so if required */ - if((H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (H5MF_ALLOC_TO_FS_TYPE(f, type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; + if(H5F_addr_defined(f->shared->fs_addr[sm_fshdr_fs_type])) { + first_srfsm_hdr = f->shared->fs_addr[sm_fshdr_fs_type]; - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (3)") - curr_ring = needed_ring; - } /* end if */ + /* open the FSM */ + if(H5MF_open_fstype(f, dxpl_id, sm_fshdr_fs_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") - HDassert(f->shared->fs_state[type] == H5F_FS_STATE_OPEN); - if(H5FS_close(f, dxpl_id, f->shared->fs_man[type]) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't close free space manager") - f->shared->fs_man[type] = NULL; - f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; - } /* end if */ - f->shared->fs_addr[type] = HADDR_UNDEF; - } /* end for */ + HDassert(f->shared->fs_man[sm_fshdr_fs_type]); + } /* end if */ - /* Verify that we haven't dirtied any metadata cache entries - * from the metadata free space manager ring out. - */ - HDassert(H5AC_cache_is_clean(f, H5AC_RING_MDFSM)); + if((sm_fshdr_fs_type != sm_fssinfo_fs_type) && + (H5F_addr_defined(f->shared->fs_addr[sm_fssinfo_fs_type]))) { - /* Verify that the aggregators are still shutdown. */ - HDassert(f->shared->sdata_aggr.tot_size == 0); - HDassert(f->shared->sdata_aggr.addr == 0); - HDassert(f->shared->sdata_aggr.size == 0); - HDassert(f->shared->meta_aggr.tot_size == 0); - HDassert(f->shared->meta_aggr.addr == 0); - HDassert(f->shared->meta_aggr.size == 0); - } /* end if */ - else { /* super_vers can be 0, 1, 2 */ - /* Close and delete freespace managers from the file */ - if(H5MF__close_delete(f, dxpl_id, &dxpl) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") + if(!H5F_addr_defined(first_srfsm_hdr) || + (H5F_addr_defined(first_srfsm_hdr) && + H5F_addr_lt(f->shared->fs_addr[sm_fssinfo_fs_type], first_srfsm_hdr))) + first_srfsm_hdr = f->shared->fs_addr[sm_fssinfo_fs_type]; - /* moved code that was for both the persistant and non persistant free - * space managers to the non-persistant case. In the persistant - * case, the EOA should already be as shrunked as it can be, and the - * aggregators should alread be shut down. - */ + HDassert(NULL == f->shared->fs_man[sm_fssinfo_fs_type]); - /* Free the space in aggregators (again) */ - /* (in case any free space information re-started them) */ - if(H5MF_free_aggrs(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTFREE, FAIL, "can't free aggregators") + /* open the FSM */ + if(H5MF_open_fstype(f, dxpl_id, sm_fssinfo_fs_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") - /* Trying shrinking the EOA for the file */ - /* (in case any free space is now at the EOA) */ - if(H5MF__close_shrink_eoa(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't shrink eoa") - } /* end else */ + HDassert(f->shared->fs_man[sm_fssinfo_fs_type]); + } /* end if */ -done: - /* Reset the ring in the DXPL */ - if(reset_ring) - if(H5AC_reset_ring(dxpl, orig_ring) < 0) - HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") + if(H5F_PAGED_AGGR(f)) { + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_HDR, f->shared->fs_page_size + 1, &lg_fshdr_fs_type); + H5MF_alloc_to_fs_type(f, H5FD_MEM_FSPACE_SINFO, f->shared->fs_page_size + 1, &lg_fssinfo_fs_type); -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Leaving\n", FUNC); -#endif /* H5MF_ALLOC_DEBUG */ - FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* end H5MF_close() */ + HDassert(lg_fshdr_fs_type >= H5F_MEM_PAGE_LARGE_SUPER); + HDassert(lg_fshdr_fs_type < H5F_MEM_PAGE_NTYPES); - -/*------------------------------------------------------------------------- - * Function: H5MF_sects_cb() - * - * Purpose: Iterator callback for each free-space section - * Retrieve address and size into user data - * - * Return: Always succeed - * - * Programmer: Vailin Choi - * July 1st, 2009 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5MF_sects_cb(H5FS_section_info_t *_sect, void *_udata) -{ - H5MF_free_section_t *sect = (H5MF_free_section_t *)_sect; - H5MF_sect_iter_ud_t *udata = (H5MF_sect_iter_ud_t *)_udata; + HDassert(lg_fssinfo_fs_type >= H5F_MEM_PAGE_LARGE_SUPER); + HDassert(lg_fssinfo_fs_type < H5F_MEM_PAGE_NTYPES); - FUNC_ENTER_NOAPI_NOINIT_NOERR + HDassert(NULL == f->shared->fs_man[lg_fshdr_fs_type]); + HDassert(NULL == f->shared->fs_man[lg_fssinfo_fs_type]); - if(udata->sect_idx < udata->sect_count) { - udata->sects[udata->sect_idx].addr = sect->sect_info.addr; - udata->sects[udata->sect_idx].size = sect->sect_info.size; - udata->sect_idx++; - } /* end if */ + if(H5F_addr_defined(f->shared->fs_addr[lg_fshdr_fs_type])) { + if(!H5F_addr_defined(first_srfsm_hdr) || + (H5F_addr_defined(first_srfsm_hdr) && + H5F_addr_lt(f->shared->fs_addr[lg_fshdr_fs_type], first_srfsm_hdr))) + first_srfsm_hdr = f->shared->fs_addr[lg_fshdr_fs_type]; - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5MF_sects_cb() */ + HDassert(NULL == f->shared->fs_man[lg_fshdr_fs_type]); - -/*------------------------------------------------------------------------- - * Function: H5MF_get_free_sections() - * - * Purpose: To iterate over one or all free-space managers for: - * # of sections - * section info as defined in H5F_sect_info_t - * - * Return: SUCCEED/FAIL - * - * Programmer: Vailin Choi - * July 1st, 2009 - * - *------------------------------------------------------------------------- - */ -ssize_t -H5MF_get_free_sections(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, size_t nsects, H5F_sect_info_t *sect_info) -{ - size_t total_sects = 0; /* total number of sections */ - H5MF_sect_iter_ud_t sect_udata; /* User data for callback */ - H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ - H5AC_ring_t curr_ring; /* Current ring value */ - H5AC_ring_t needed_ring; /* Ring value needed for this iteration */ - H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ - H5FD_mem_t start_type, end_type; /* Memory types to iterate over */ - H5FD_mem_t ty; /* Memory type for iteration */ - hbool_t reset_ring = FALSE; /* Whether the ring was set */ - ssize_t ret_value = -1; /* Return value */ + /* open the FSM */ + if(H5MF_open_fstype(f, dxpl_id, lg_fshdr_fs_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") + HDassert(f->shared->fs_man[lg_fshdr_fs_type]); + } /* end if */ - FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + if(lg_fshdr_fs_type != lg_fssinfo_fs_type && H5F_addr_defined(f->shared->fs_addr[lg_fssinfo_fs_type])) { + if(!H5F_addr_defined(first_srfsm_hdr) || + (H5F_addr_defined(first_srfsm_hdr) && + H5F_addr_lt(f->shared->fs_addr[lg_fssinfo_fs_type], first_srfsm_hdr))) + first_srfsm_hdr = f->shared->fs_addr[lg_fssinfo_fs_type]; - /* check args */ - HDassert(f); - HDassert(f->shared); - HDassert(f->shared->lf); + HDassert(NULL == f->shared->fs_man[lg_fssinfo_fs_type]); - /* Determine start/end points for loop */ - if(type == H5FD_MEM_DEFAULT) { - start_type = H5FD_MEM_SUPER; - end_type = H5FD_MEM_NTYPES; + /* open the FSM */ + if(H5MF_open_fstype(f, dxpl_id, lg_fssinfo_fs_type) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space manager") + HDassert(f->shared->fs_man[lg_fssinfo_fs_type]); + } /* end if */ + } /* end if */ + HDassert(H5F_addr_eq(first_srfsm_hdr, f->shared->eoa_pre_fsm_fsalloc)); + + /* 4) Float the FSMs. Ensure that the file space is NOT released. */ + if(f->shared->fs_man[sm_fshdr_fs_type]) { + /* Sanity check: Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[sm_fshdr_fs_type], &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") + + HDassert(H5F_addr_defined(fs_stat.addr)); + HDassert(H5F_addr_defined(fs_stat.sect_addr)); + if(H5FS_free(f, f->shared->fs_man[sm_fshdr_fs_type], dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers") + f->shared->fs_addr[sm_fshdr_fs_type] = HADDR_UNDEF; } /* end if */ - else { - start_type = end_type = type; - H5_INC_ENUM(H5FD_mem_t, end_type); - } /* end else */ - /* Set up user data for section iteration */ - sect_udata.sects = sect_info; - sect_udata.sect_count = nsects; - sect_udata.sect_idx = 0; + if(sm_fshdr_fs_type != sm_fssinfo_fs_type && f->shared->fs_man[sm_fssinfo_fs_type]) { + /* Sanity check: Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[sm_fssinfo_fs_type], &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't get free-space info") - /* Set the ring type in the DXPL. Note that if we are - * scanning a number of different free space managers, - * we may have to change the ring - */ - if((H5MF_ALLOC_TO_FS_TYPE(f, start_type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (H5MF_ALLOC_TO_FS_TYPE(f, start_type) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; - curr_ring = needed_ring; - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &orig_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value") - reset_ring = TRUE; - - /* Iterate over memory types, retrieving the number of sections of each type */ - for(ty = start_type; ty < end_type; H5_INC_ENUM(H5FD_mem_t, ty)) { - hbool_t fs_started = FALSE; + HDassert(H5F_addr_defined(fs_stat.addr)); + HDassert(H5F_addr_defined(fs_stat.sect_addr)); + if(H5FS_free(f, f->shared->fs_man[sm_fssinfo_fs_type], dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't release free-space headers") + f->shared->fs_addr[sm_fssinfo_fs_type] = HADDR_UNDEF; + } /* end if */ - /* test to see if we need to switch rings -- do so if required */ - if((H5MF_ALLOC_TO_FS_TYPE(f, ty) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_HDR)) - || (H5MF_ALLOC_TO_FS_TYPE(f, ty) == H5MF_ALLOC_TO_FS_TYPE(f, H5FD_MEM_FSPACE_SINFO))) - needed_ring = H5AC_RING_MDFSM; - else - needed_ring = H5AC_RING_RDFSM; + if(H5F_PAGED_AGGR(f)) { + if(f->shared->fs_man[lg_fshdr_fs_type]) { + /* Sanity check: Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[lg_fshdr_fs_type], &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") - if(needed_ring != curr_ring) { - if(H5AC_set_ring(dxpl_id, needed_ring, &dxpl, &curr_ring) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set ring value (2)") - curr_ring = needed_ring; + HDassert(H5F_addr_defined(fs_stat.addr)); + HDassert(H5F_addr_defined(fs_stat.sect_addr)); + if(H5FS_free(f, f->shared->fs_man[lg_fshdr_fs_type], dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't float free-space headers") + f->shared->fs_addr[lg_fshdr_fs_type] = HADDR_UNDEF; } /* end if */ - /* Open free space manager of this type, if it isn't already */ - if(!f->shared->fs_man[ty] && H5F_addr_defined(f->shared->fs_addr[ty])) { - if(H5MF__alloc_open(f, dxpl_id, ty) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTOPENOBJ, FAIL, "can't initialize file free space") - HDassert(f->shared->fs_man[ty]); - fs_started = TRUE; - } /* end if */ - - /* Check if f there's free space sections of this type */ - if(f->shared->fs_man[ty]) { - hsize_t hnums = 0; /* Total # of sections */ - size_t nums; /* Total # of sections, cast to a size_t */ - - /* Query how many sections of this type */ - if(H5FS_sect_stats(f->shared->fs_man[ty], NULL, &hnums) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query free space stats") - H5_CHECKED_ASSIGN(nums, size_t, hnums, hsize_t); - - /* Increment total # of sections */ - total_sects += nums; - - /* Check if we should retrieve the section info */ - if(sect_info && nums > 0) { - /* Iterate over all the free space sections of this type, adding them to the user's section info */ - if(H5FS_sect_iterate(f, dxpl_id, f->shared->fs_man[ty], H5MF_sects_cb, §_udata) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADITER, FAIL, "can't iterate over sections") - } /* end if */ + if(lg_fshdr_fs_type != lg_fssinfo_fs_type && f->shared->fs_man[lg_fssinfo_fs_type]) { + /* Sanity check: Query free space manager info for this type */ + if(H5FS_stat_info(f, f->shared->fs_man[lg_fssinfo_fs_type], &fs_stat) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't get free-space info") + + HDassert(H5F_addr_defined(fs_stat.addr)); + HDassert(H5F_addr_defined(fs_stat.sect_addr)); + if(H5FS_free(f, f->shared->fs_man[lg_fssinfo_fs_type], dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't float free-space headers") + f->shared->fs_addr[lg_fssinfo_fs_type] = HADDR_UNDEF; } /* end if */ + } /* end if */ - /* Close the free space manager of this type, if we started it here */ - if(fs_started) - if(H5MF__alloc_close(f, dxpl_id, ty) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCLOSEOBJ, FAIL, "can't close file free space") - } /* end for */ + /* 5) Set EOA equal to f->shared->eoa_pre_fsm_fsalloc, + * and then set f->shared->eoa_pre_fsm_fsalloc to + * HADDR_UNDEF. + * + * If page buffering, verify that the new EOA is + * on a page boundary, and expunge any pages in the + * page buffer after the new EOA. + */ + if(!H5F_PAGED_AGGR(f)) { + /* Verify that the aggregators are still shutdown. */ + HDassert(f->shared->sdata_aggr.tot_size == 0); + HDassert(f->shared->sdata_aggr.addr == 0); + HDassert(f->shared->sdata_aggr.size == 0); - /* Set return value */ - ret_value = (ssize_t)total_sects; + HDassert(f->shared->meta_aggr.tot_size == 0); + HDassert(f->shared->meta_aggr.addr == 0); + HDassert(f->shared->meta_aggr.size == 0); + } /* end if */ + + tail_size = (hsize_t)(eoa - f->shared->eoa_pre_fsm_fsalloc); + + /* Release file space allocated to self referential FSMs */ + if(H5F_free(f, dxpl_id, H5FD_MEM_DEFAULT, f->shared->eoa_pre_fsm_fsalloc, tail_size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "driver free request failed") + if(HADDR_UNDEF == (eoa = H5FD_get_eoa(f->shared->lf, H5FD_MEM_DEFAULT))) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to get EOA") + HDassert(H5F_addr_eq(f->shared->eoa_pre_fsm_fsalloc, eoa)); + + f->shared->eoa_pre_fsm_fsalloc = HADDR_UNDEF; + + HDassert((!H5F_PAGED_AGGR(f)) || (0 == (eoa % f->shared->fs_page_size))); done: /* Reset the ring in the DXPL */ @@ -2349,5 +3911,5 @@ done: HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) -} /* H5MF_get_free_sections() */ +} /* H5MF_tidy_self_referential_fsm_hack() */ diff --git a/src/H5MFaggr.c b/src/H5MFaggr.c index 1510645..de98bf1 100644 --- a/src/H5MFaggr.c +++ b/src/H5MFaggr.c @@ -57,8 +57,10 @@ /********************/ /* Local Prototypes */ /********************/ -static herr_t H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, +static herr_t H5MF__aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr); +static haddr_t H5MF__aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, + H5F_blk_aggr_t *other_aggr, H5FD_mem_t type, hsize_t size); /*********************/ @@ -78,86 +80,9 @@ static herr_t H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, /*------------------------------------------------------------------------- - * Function: H5MF_vfd_alloc - * - * Purpose: Allocate SIZE bytes of file memory via H5FD_alloc() and return - * the relative address where that contiguous chunk of file memory - * exists. - * The TYPE argument describes the purpose for which the storage - * is being requested. - * - * Return: Success: The file address of new chunk. - * Failure: HADDR_UNDEF - * - * Programmer: Quincey Koziol - * January 2, 2017 - * - *------------------------------------------------------------------------- - */ -haddr_t -H5MF_vfd_alloc(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type, hsize_t size, - hbool_t keep_fragment) -{ - haddr_t eoa; /* Initial EOA for the file */ - haddr_t eoa_frag_addr = HADDR_UNDEF; /* Address of fragment at EOA */ - hsize_t eoa_frag_size = 0; /* Size of fragment at EOA */ - haddr_t ret_value = HADDR_UNDEF; /* Return value */ - - FUNC_ENTER_NOAPI(HADDR_UNDEF) -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_type, size); -#endif /* H5MF_ALLOC_DEBUG */ - - /* check arguments */ - HDassert(f); - HDassert(f->shared); - HDassert(f->shared->lf); - HDassert(size > 0); - - /* Get the EOA for the file -- need for sanity check below */ - if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, alloc_type))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, HADDR_UNDEF, "Unable to get eoa") - - /* Check for overlap into temporary allocation space */ - if(H5F_addr_gt((eoa + size), f->shared->tmp_addr)) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "hdr file space alloc will overlap into 'temporary' file space") - - /* Allocate space for the header */ - if(HADDR_UNDEF == (ret_value = H5FD_alloc(f->shared->lf, dxpl_id, alloc_type, f, size, &eoa_frag_addr, &eoa_frag_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space for hdr") - - /* Sanity check for overlapping into file's temporary allocation space */ - HDassert(H5F_addr_le((ret_value + size), f->shared->tmp_addr)); - - /* If the file alignment is 1, there should be no eoa fragment */ - HDassert((eoa_frag_size == 0) || (f->shared->alignment != 1)); - - /* Check if fragment was generated and we want to keep it */ - if(keep_fragment && eoa_frag_size > 0) { - /* Sanity check */ - HDassert(H5F_addr_defined(eoa_frag_addr)); - - /* Put fragment on the free list */ - if(H5MF_xfree(f, alloc_type, dxpl_id, eoa_frag_addr, eoa_frag_size) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free eoa fragment") - } /* end if */ - -done: -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", FUNC, ret_value, size); -#endif /* H5MF_ALLOC_DEBUG */ -#ifdef H5MF_ALLOC_DEBUG_DUMP -H5MF_sects_dump(f, dxpl_id, stderr); -#endif /* H5MF_ALLOC_DEBUG_DUMP */ - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5MF_vfd_alloc() */ - - -/*------------------------------------------------------------------------- * Function: H5MF_aggr_vfd_alloc * - * Purpose: Allocate SIZE bytes of file memory via H5MF_aggr_alloc() + * Purpose: Allocate SIZE bytes of file memory via H5MF__aggr_alloc() * and return the relative address where that contiguous chunk * of file memory exists. * The TYPE argument describes the purpose for which the storage @@ -177,9 +102,9 @@ H5MF_aggr_vfd_alloc(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size haddr_t ret_value = HADDR_UNDEF; /* Return value */ FUNC_ENTER_NOAPI(HADDR_UNDEF) -#ifdef H5MF_ALLOC_DEBUG +#ifdef H5MF_AGGR_DEBUG HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_type, size); -#endif /* H5MF_ALLOC_DEBUG */ +#endif /* H5MF_AGGR_DEBUG */ /* check arguments */ HDassert(f); @@ -190,12 +115,12 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ /* Couldn't find anything from the free space manager, go allocate some */ if(alloc_type != H5FD_MEM_DRAW && alloc_type != H5FD_MEM_GHEAP) { /* Handle metadata differently from "raw" data */ - if(HADDR_UNDEF == (ret_value = H5MF_aggr_alloc(f, dxpl_id, &(f->shared->meta_aggr), &(f->shared->sdata_aggr), alloc_type, size))) + if(HADDR_UNDEF == (ret_value = H5MF__aggr_alloc(f, dxpl_id, &(f->shared->meta_aggr), &(f->shared->sdata_aggr), alloc_type, size))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate metadata") } /* end if */ else { /* Allocate "raw" data: H5FD_MEM_DRAW and H5FD_MEM_GHEAP */ - if(HADDR_UNDEF == (ret_value = H5MF_aggr_alloc(f, dxpl_id, &(f->shared->sdata_aggr), &(f->shared->meta_aggr), H5FD_MEM_DRAW, size))) + if(HADDR_UNDEF == (ret_value = H5MF__aggr_alloc(f, dxpl_id, &(f->shared->sdata_aggr), &(f->shared->meta_aggr), H5FD_MEM_DRAW, size))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate raw data") } /* end else */ @@ -203,19 +128,16 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ HDassert(H5F_addr_le((ret_value + size), f->shared->tmp_addr)); done: -#ifdef H5MF_ALLOC_DEBUG +#ifdef H5MF_AGGR_DEBUG HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", FUNC, ret_value, size); -#endif /* H5MF_ALLOC_DEBUG */ -#ifdef H5MF_ALLOC_DEBUG_DUMP -H5MF_sects_dump(f, dxpl_id, stderr); -#endif /* H5MF_ALLOC_DEBUG_DUMP */ +#endif /* H5MF_AGGR_DEBUG */ FUNC_LEAVE_NOAPI(ret_value) } /* end H5MF_aggr_vfd_alloc() */ /*------------------------------------------------------------------------- - * Function: H5MF_aggr_alloc + * Function: H5MF__aggr_alloc * * Purpose: Try to allocate SIZE bytes of memory from an aggregator * block if possible. @@ -228,8 +150,8 @@ H5MF_sects_dump(f, dxpl_id, stderr); * *------------------------------------------------------------------------- */ -haddr_t -H5MF_aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, +static haddr_t +H5MF__aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, H5F_blk_aggr_t *other_aggr, H5FD_mem_t type, hsize_t size) { haddr_t eoa_frag_addr = HADDR_UNDEF; /* Address of fragment at EOA */ @@ -237,7 +159,7 @@ H5MF_aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, haddr_t eoa = HADDR_UNDEF; /* Initial EOA for the file */ haddr_t ret_value = HADDR_UNDEF; /* Return value */ - FUNC_ENTER_NOAPI(HADDR_UNDEF) + FUNC_ENTER_STATIC #ifdef H5MF_AGGR_DEBUG HDfprintf(stderr, "%s: type = %u, size = %Hu\n", FUNC, (unsigned)type, size); #endif /* H5MF_AGGR_DEBUG */ @@ -257,11 +179,11 @@ HDfprintf(stderr, "%s: type = %u, size = %Hu\n", FUNC, (unsigned)type, size); HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, HADDR_UNDEF, "Unable to get eoa") /* - * If the aggregation feature is enabled for this file and strategy is not H5F_FILE_SPACE_VFD, + * If the aggregation feature is enabled for this file and strategy is not H5F_FILE_SPACE_NONE, * allocate "generic" space and sub-allocate out of that, if possible. - * Otherwise just allocate through H5FD_alloc(). + * Otherwise just allocate through H5F_alloc(). */ - if((f->shared->feature_flags & aggr->feature_flag) && f->shared->fs_strategy != H5F_FILE_SPACE_VFD) { + if((f->shared->feature_flags & aggr->feature_flag) && f->shared->fs_strategy != H5F_FSPACE_STRATEGY_NONE) { haddr_t aggr_frag_addr = HADDR_UNDEF; /* Address of aggregrator fragment */ hsize_t aggr_frag_size = 0; /* Size of aggregator fragment */ hsize_t alignment; /* Alignment of this section */ @@ -273,12 +195,12 @@ HDfprintf(stderr, "%s: aggr = {%a, %Hu, %Hu}\n", FUNC, aggr->addr, aggr->tot_siz #endif /* H5MF_AGGR_DEBUG */ /* Turn off alignment if allocation < threshold */ - alignment = f->shared->alignment; - if(!((alignment > 1) && (size >= f->shared->threshold))) + alignment = H5F_ALIGNMENT(f); + if(!((alignment > 1) && (size >= H5F_THRESHOLD(f)))) alignment = 0; /* no alignment */ /* Generate fragment if aggregator is mis-aligned */ - if(alignment && aggr->addr > 0 && (aggr_mis_align = (aggr->addr + H5FD_get_base_addr(f->shared->lf)) % alignment)) { + if(alignment && H5F_addr_gt(aggr->addr, 0) && (aggr_mis_align = (aggr->addr + H5F_BASE_ADDR(f)) % alignment)) { aggr_frag_addr = aggr->addr; aggr_frag_size = alignment - aggr_mis_align; } /* end if */ @@ -288,7 +210,7 @@ HDfprintf(stderr, "%s: aggr = {%a, %Hu, %Hu}\n", FUNC, aggr->addr, aggr->tot_siz /* Check if the space requested is larger than the space left in the block */ if((size + aggr_frag_size) > aggr->size) { - htri_t was_extended = FALSE; /* Whether the file was extended */ + htri_t extended = FALSE; /* Whether the file was extended */ /* Check if the block asked for is too large for 'normal' aggregator block */ if(size >= aggr->alloc_size) { @@ -298,32 +220,27 @@ HDfprintf(stderr, "%s: aggr = {%a, %Hu, %Hu}\n", FUNC, aggr->addr, aggr->tot_siz if(H5F_addr_gt((aggr->addr + aggr->size + ext_size), f->shared->tmp_addr)) HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space") - if((aggr->addr > 0) && (was_extended = H5FD_try_extend(f->shared->lf, alloc_type, f, dxpl_id, aggr->addr + aggr->size, ext_size)) < 0) + if((aggr->addr > 0) && (extended = H5F_try_extend(f, dxpl_id, alloc_type, (aggr->addr + aggr->size), ext_size)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't extending space") - else if(was_extended) { + else if (extended) { /* aggr->size is unchanged */ ret_value = aggr->addr + aggr_frag_size; aggr->addr += ext_size; aggr->tot_size += ext_size; - } /* end else-if */ - else { - /* Check for overlapping into file's temporary allocation space */ - if(H5F_addr_gt((eoa + size), f->shared->tmp_addr)) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space") - + } else { /* Release "other" aggregator, if it exists, is at the end of the allocated space, * has allocated more than one block and the unallocated space is greater than its * allocation block size. */ if((other_aggr->size > 0) && (H5F_addr_eq((other_aggr->addr + other_aggr->size), eoa)) && (other_aggr->tot_size > other_aggr->size) && ((other_aggr->tot_size - other_aggr->size) >= other_aggr->alloc_size)) { - if(H5MF_aggr_free(f, dxpl_id, other_alloc_type, other_aggr) < 0) + if(H5MF__aggr_free(f, dxpl_id, other_alloc_type, other_aggr) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free aggregation block") } /* end if */ /* Allocate space from the VFD (i.e. at the end of the file) */ - if(HADDR_UNDEF == (ret_value = H5FD_alloc(f->shared->lf, dxpl_id, alloc_type, f, size, &eoa_frag_addr, &eoa_frag_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate aggregation block") + if(HADDR_UNDEF == (ret_value = H5F_alloc(f, dxpl_id, alloc_type, size, &eoa_frag_addr, &eoa_frag_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space") } /* end else */ } /* end if */ else { @@ -341,9 +258,9 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC); if(H5F_addr_gt((aggr->addr + aggr->size + ext_size), f->shared->tmp_addr)) HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space") - if((aggr->addr > 0) && (was_extended = H5FD_try_extend(f->shared->lf, alloc_type, f, dxpl_id, aggr->addr + aggr->size, ext_size)) < 0) + if((aggr->addr > 0) && (extended = H5F_try_extend(f, dxpl_id, alloc_type, (aggr->addr + aggr->size), ext_size)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't extending space") - else if(was_extended) { + else if(extended) { aggr->addr += aggr_frag_size; aggr->size += (ext_size - aggr_frag_size); aggr->tot_size += ext_size; @@ -351,23 +268,19 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC); else { haddr_t new_space; /* Address of new space allocated */ - /* Check for overlapping into file's temporary allocation space */ - if(H5F_addr_gt((eoa + aggr->alloc_size), f->shared->tmp_addr)) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space") - /* Release "other" aggregator, if it exists, is at the end of the allocated space, * has allocated more than one block and the unallocated space is greater than its * allocation block size. */ if((other_aggr->size > 0) && (H5F_addr_eq((other_aggr->addr + other_aggr->size), eoa)) && (other_aggr->tot_size > other_aggr->size) && ((other_aggr->tot_size - other_aggr->size) >= other_aggr->alloc_size)) { - if(H5MF_aggr_free(f, dxpl_id, other_alloc_type, other_aggr) < 0) + if(H5MF__aggr_free(f, dxpl_id, other_alloc_type, other_aggr) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free aggregation block") } /* end if */ /* Allocate space from the VFD (i.e. at the end of the file) */ - if(HADDR_UNDEF == (new_space = H5FD_alloc(f->shared->lf, dxpl_id, alloc_type, f, aggr->alloc_size, &eoa_frag_addr, &eoa_frag_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate aggregation block") + if(HADDR_UNDEF == (new_space = H5F_alloc(f, dxpl_id, alloc_type, aggr->alloc_size, &eoa_frag_addr, &eoa_frag_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space") /* Return the unused portion of the block to a free list */ if(aggr->size > 0) @@ -408,7 +321,7 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC); HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free eoa fragment") /* Freeing any possible fragment due to alignment in the block after extension */ - if(was_extended && aggr_frag_size) + if(extended && aggr_frag_size) if(H5MF_xfree(f, alloc_type, dxpl_id, aggr_frag_addr, aggr_frag_size) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free aggregation fragment") } /* end if */ @@ -425,12 +338,8 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC); } /* end else */ } /* end if */ else { - /* Check for overlapping into file's temporary allocation space */ - if(H5F_addr_gt((eoa + size), f->shared->tmp_addr)) - HGOTO_ERROR(H5E_RESOURCE, H5E_BADRANGE, HADDR_UNDEF, "'normal' file space allocation request will overlap into 'temporary' file space") - /* Allocate data from the file */ - if(HADDR_UNDEF == (ret_value = H5FD_alloc(f->shared->lf, dxpl_id, type, f, size, &eoa_frag_addr, &eoa_frag_size))) + if(HADDR_UNDEF == (ret_value = H5F_alloc(f, dxpl_id, type, size, &eoa_frag_addr, &eoa_frag_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate file space") /* Check if fragment was generated */ @@ -444,15 +353,15 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC); HDassert(H5F_addr_le((ret_value + size), f->shared->tmp_addr)); /* Post-condition sanity check */ - if(f->shared->alignment && size >= f->shared->threshold) - HDassert(!((ret_value + H5FD_get_base_addr(f->shared->lf)) % f->shared->alignment)); + if(H5F_ALIGNMENT(f) && size >= H5F_THRESHOLD(f)) + HDassert(!((ret_value + H5FD_get_base_addr(f->shared->lf)) % H5F_ALIGNMENT(f))); done: #ifdef H5MF_AGGR_DEBUG HDfprintf(stderr, "%s: ret_value = %a\n", FUNC, ret_value); #endif /* H5MF_AGGR_DEBUG */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5MF_aggr_alloc() */ +} /* end H5MF__aggr_alloc() */ /*------------------------------------------------------------------------- @@ -524,7 +433,7 @@ H5MF_aggr_try_extend(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, else { hsize_t extra = (extra_requested < aggr->alloc_size) ? aggr->alloc_size : extra_requested; - if((ret_value = H5FD_try_extend(f->shared->lf, type, f, dxpl_id, (aggr->addr + aggr->size), extra)) < 0) + if((ret_value = H5F_try_extend(f, dxpl_id, type, (aggr->addr + aggr->size), extra)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file") else if(ret_value == TRUE) { /* Shift the aggregator block by the extra requested */ @@ -905,7 +814,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5MF_aggr_free + * Function: H5MF__aggr_free * * Purpose: Free the aggregator's space in the file. * @@ -919,11 +828,11 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr) +H5MF__aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_STATIC /* Sanity check */ HDassert(f); @@ -936,7 +845,7 @@ H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr) HDassert(f->shared->feature_flags & aggr->feature_flag); /* Free the remaining space at EOA in the aggregator */ - if(H5FD_free(f->shared->lf, dxpl_id, type, f, aggr->addr, aggr->size) < 0) + if(H5F_free(f, dxpl_id, type, aggr->addr, aggr->size) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free aggregation block") /* Reset the aggregator */ @@ -946,7 +855,7 @@ H5MF_aggr_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, H5F_blk_aggr_t *aggr) done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5MF_aggr_free() */ +} /* H5MF__aggr_free() */ /*------------------------------------------------------------------------- @@ -978,13 +887,13 @@ H5MF_aggrs_try_shrink_eoa(H5F_t *f, hid_t dxpl_id) if((ma_status = H5MF_aggr_can_shrink_eoa(f, H5FD_MEM_DEFAULT, &(f->shared->meta_aggr))) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query metadata aggregator stats") if(ma_status > 0) - if(H5MF_aggr_free(f, dxpl_id, H5FD_MEM_DEFAULT, &(f->shared->meta_aggr)) < 0) + if(H5MF__aggr_free(f, dxpl_id, H5FD_MEM_DEFAULT, &(f->shared->meta_aggr)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") if((sda_status = H5MF_aggr_can_shrink_eoa(f, H5FD_MEM_DRAW, &(f->shared->sdata_aggr))) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "can't query small data aggregator stats") if(sda_status > 0) - if(H5MF_aggr_free(f, dxpl_id, H5FD_MEM_DRAW, &(f->shared->sdata_aggr)) < 0) + if(H5MF__aggr_free(f, dxpl_id, H5FD_MEM_DRAW, &(f->shared->sdata_aggr)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTSHRINK, FAIL, "can't check for shrinking eoa") ret_value = (ma_status || sda_status); diff --git a/src/H5MFdbg.c b/src/H5MFdbg.c index 57bac9e..817ea00 100644 --- a/src/H5MFdbg.c +++ b/src/H5MFdbg.c @@ -116,7 +116,9 @@ H5MF_sects_debug_cb(H5FS_section_info_t *_sect, void *_udata) /* Print generic section information */ HDfprintf(udata->stream, "%*s%-*s %s\n", udata->indent, "", udata->fwidth, "Section type:", - (sect->sect_info.type == H5MF_FSPACE_SECT_SIMPLE ? "simple" : "unknown")); + (sect->sect_info.type == H5MF_FSPACE_SECT_SIMPLE ? "simple" : + (sect->sect_info.type == H5MF_FSPACE_SECT_SMALL ? "small" : + (sect->sect_info.type == H5MF_FSPACE_SECT_LARGE ? "large" : "unknown")))); HDfprintf(udata->stream, "%*s%-*s %a\n", udata->indent, "", udata->fwidth, "Section address:", sect->sect_info.addr); @@ -171,7 +173,7 @@ H5MF_sects_debug(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr, FILE *stream, int ind for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) if(H5F_addr_eq(f->shared->fs_addr[type], fs_addr)) { if(!f->shared->fs_man[type]) - if(H5MF__alloc_open(f, dxpl_id, type) < 0) + if(H5MF_open_fstype(f, dxpl_id, type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't initialize file free space") if(f->shared->fs_man[type]) { @@ -217,11 +219,6 @@ herr_t H5MF_sects_dump(H5F_t *f, hid_t dxpl_id, FILE *stream) { haddr_t eoa; /* End of allocated space in the file */ - haddr_t ma_addr = HADDR_UNDEF; /* Base "metadata aggregator" address */ - hsize_t ma_size = 0; /* Size of "metadata aggregator" */ - haddr_t sda_addr = HADDR_UNDEF; /* Base "small data aggregator" address */ - hsize_t sda_size = 0; /* Size of "small data aggregator" */ - H5FD_mem_t type; /* Memory type for iteration */ int indent = 0; /* Amount to indent */ int fwidth = 50; /* Field width */ herr_t ret_value = SUCCEED; /* Return value */ @@ -244,56 +241,90 @@ HDfprintf(stderr, "%s: Dumping file free space sections\n", FUNC); HDfprintf(stderr, "%s: for type = H5FD_MEM_DEFAULT, eoa = %a\n", FUNC, eoa); #endif /* H5MF_ALLOC_DEBUG */ - /* Retrieve metadata aggregator info, if available */ - H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: ma_addr = %a, ma_size = %Hu, end of ma = %a\n", FUNC, ma_addr, ma_size, (haddr_t)((ma_addr + ma_size) - 1)); -#endif /* H5MF_ALLOC_DEBUG */ + if(H5F_PAGED_AGGR(f)) { /* File space paging */ + H5F_mem_page_t ptype; /* Memory type for iteration -- page fs */ - /* Retrieve 'small data' aggregator info, if available */ - H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sda_addr, &sda_size); -#ifdef H5MF_ALLOC_DEBUG -HDfprintf(stderr, "%s: sda_addr = %a, sda_size = %Hu, end of sda = %a\n", FUNC, sda_addr, sda_size, (haddr_t)((sda_addr + sda_size) - 1)); -#endif /* H5MF_ALLOC_DEBUG */ - - /* Iterate over all the free space types that have managers and dump each free list's space */ - for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { - /* Print header for type */ - HDfprintf(stream, "%*sFile Free Space Info for type = %u:\n", indent, "", (unsigned)type); - - /* Check for this type being mapped to another type */ - if(H5FD_MEM_DEFAULT == f->shared->fs_type_map[type] || - type == f->shared->fs_type_map[type]) { - /* Retrieve the 'eoa' for this file memory type */ - if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, type))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") - HDfprintf(stream, "%*s%-*s %a\n", indent + 3, "", MAX(0, fwidth - 3), "eoa:", eoa); + for(ptype = H5F_MEM_PAGE_META; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) { + /* Print header for type */ + HDfprintf(stream, "%*sFile Free Space Info for type = %u:\n", indent, "", (unsigned)ptype); /* Print header for sections */ HDfprintf(stream, "%*sSections:\n", indent + 3, ""); /* If there is a free space manager for this type, iterate over them */ - if(f->shared->fs_man[type]) { + if(f->shared->fs_man[ptype]) { H5MF_debug_iter_ud_t udata; /* User data for callbacks */ /* Prepare user data for section iteration callback */ - udata.fspace = f->shared->fs_man[type]; + udata.fspace = f->shared->fs_man[ptype]; udata.stream = stream; udata.indent = indent + 6; udata.fwidth = MAX(0, fwidth - 6); /* Iterate over all the free space sections */ - if(H5FS_sect_iterate(f, dxpl_id, f->shared->fs_man[type], H5MF_sects_debug_cb, &udata) < 0) + if(H5FS_sect_iterate(f, dxpl_id, f->shared->fs_man[ptype], H5MF_sects_debug_cb, &udata) < 0) HGOTO_ERROR(H5E_HEAP, H5E_BADITER, FAIL, "can't iterate over heap's free space") } /* end if */ else /* No sections of this type */ HDfprintf(stream, "%*s<none>\n", indent + 6, ""); - } /* end if */ - else - HDfprintf(stream, "%*sMapped to type = %u\n", indent, "", (unsigned)f->shared->fs_type_map[type]); + } /* end for */ + } /* end if */ + else { /* not file space paging */ + H5FD_mem_t atype; /* Memory type for iteration -- aggr fs */ + haddr_t ma_addr = HADDR_UNDEF; /* Base "metadata aggregator" address */ + hsize_t ma_size = 0; /* Size of "metadata aggregator" */ + haddr_t sda_addr = HADDR_UNDEF; /* Base "small data aggregator" address */ + hsize_t sda_size = 0; /* Size of "small data aggregator" */ + + /* Retrieve metadata aggregator info, if available */ + H5MF_aggr_query(f, &(f->shared->meta_aggr), &ma_addr, &ma_size); +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: ma_addr = %a, ma_size = %Hu, end of ma = %a\n", FUNC, ma_addr, ma_size, (haddr_t)((ma_addr + ma_size) - 1)); +#endif /* H5MF_ALLOC_DEBUG */ + + /* Retrieve 'small data' aggregator info, if available */ + H5MF_aggr_query(f, &(f->shared->sdata_aggr), &sda_addr, &sda_size); +#ifdef H5MF_ALLOC_DEBUG +HDfprintf(stderr, "%s: sda_addr = %a, sda_size = %Hu, end of sda = %a\n", FUNC, sda_addr, sda_size, (haddr_t)((sda_addr + sda_size) - 1)); +#endif /* H5MF_ALLOC_DEBUG */ - } /* end for */ + /* Iterate over all the free space types that have managers and dump each free list's space */ + for(atype = H5FD_MEM_DEFAULT; atype < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, atype)) { + /* Print header for type */ + HDfprintf(stream, "%*sFile Free Space Info for type = %u:\n", indent, "", (unsigned)atype); + + /* Check for this type being mapped to another type */ + if(H5FD_MEM_DEFAULT == f->shared->fs_type_map[atype] || atype == f->shared->fs_type_map[atype]) { + /* Retrieve the 'eoa' for this file memory type */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, atype))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + HDfprintf(stream, "%*s%-*s %a\n", indent + 3, "", MAX(0, fwidth - 3), "eoa:", eoa); + + /* Print header for sections */ + HDfprintf(stream, "%*sSections:\n", indent + 3, ""); + + /* If there is a free space manager for this type, iterate over them */ + if(f->shared->fs.aggr.fs_man[atype]) { + H5MF_debug_iter_ud_t udata; /* User data for callbacks */ + + /* Prepare user data for section iteration callback */ + udata.fspace = f->shared->fs_man[atype]; + udata.stream = stream; + udata.indent = indent + 6; + udata.fwidth = MAX(0, fwidth - 6); + + /* Iterate over all the free space sections */ + if(H5FS_sect_iterate(f, dxpl_id, f->shared->fs_man[atype], H5MF_sects_debug_cb, &udata) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_BADITER, FAIL, "can't iterate over heap's free space") + } /* end if */ + else /* No sections of this type */ + HDfprintf(stream, "%*s<none>\n", indent + 6, ""); + } /* end if */ + else + HDfprintf(stream, "%*sMapped to type = %u\n", indent, "", (unsigned)f->shared->fs_type_map[atype]); + } /* end for */ + } /* end else */ done: HDfprintf(stderr, "%s: Done dumping file free space sections\n", FUNC); diff --git a/src/H5MFpkg.h b/src/H5MFpkg.h index 43fc1cd..c4a0819 100644 --- a/src/H5MFpkg.h +++ b/src/H5MFpkg.h @@ -51,9 +51,38 @@ /* Define this to dump free space tracker contents after they've been modified */ /* #define H5MF_ALLOC_DEBUG_DUMP */ -/* Free space section types for file */ +/* Free-space section types for file */ /* (values stored in free space data structures in file) */ -#define H5MF_FSPACE_SECT_SIMPLE 0 /* Section is a range of actual bytes in file */ +#define H5MF_FSPACE_SECT_SIMPLE 0 /* For non-paged aggregation: section is a range of actual bytes in file */ +#define H5MF_FSPACE_SECT_SMALL 1 /* For paged aggregation: "small" meta/raw data section which is < fsp_size) */ +#define H5MF_FSPACE_SECT_LARGE 2 /* For paged aggregation: "large" Section which is >= fsp_size) */ + +/* For non-paged aggregation: map allocation request type to tracked free-space type */ +/* F -- pointer to H5F_t; T -- H5FD_mem_t */ +#define H5MF_ALLOC_TO_FS_AGGR_TYPE(F, T) \ + ((H5FD_MEM_DEFAULT == (F)->shared->fs_type_map[T]) ? (T) : (F)->shared->fs_type_map[T]) + +/* Get section class type based on size */ +#define H5MF_SECT_CLASS_TYPE(F, S) \ + ((H5F_PAGED_AGGR(F)) ? \ + ((S >= (F)->shared->fs_page_size) ? H5MF_FSPACE_SECT_LARGE : H5MF_FSPACE_SECT_SMALL) : H5MF_FSPACE_SECT_SIMPLE) + +/* Get section class cls */ +#define H5MF_SECT_CLS_TYPE(F, S) \ + ((H5F_PAGED_AGGR(F)) ? \ + ((S >= (F)->shared->fs_page_size) ? \ + H5MF_FSPACE_SECT_CLS_LARGE : H5MF_FSPACE_SECT_CLS_SMALL) : H5MF_FSPACE_SECT_CLS_SIMPLE) + +/* Calculate the mis-aligned fragment */ +#define H5MF_EOA_MISALIGN(F, E, A, FR) \ +{ \ + hsize_t m; \ + \ + if(H5F_addr_gt((E), 0) && ((m) = ((E) + H5F_BASE_ADDR(F)) % (A))) \ + (FR) = (A) - m; \ + else \ + (FR) = 0; \ +} /****************************/ @@ -129,6 +158,15 @@ typedef struct H5MF_sect_ud_t { H5F_blk_aggr_t *aggr; /* Aggregator block to operate on */ } H5MF_sect_ud_t; +/* Information about the current free-space manager to use */ +typedef struct H5MF_fs_t { + H5F_fs_state_t *fs_state; + haddr_t *fs_addr; + H5FS_t **fs_man; + hsize_t align_thres; /* Threshold for alignment */ + hsize_t alignment; /* Alignment */ +} H5MF_fs_t; + /*****************************/ /* Package Private Variables */ @@ -136,6 +174,8 @@ typedef struct H5MF_sect_ud_t { /* H5MF single section inherits serializable properties from H5FS_section_class_t */ H5_DLLVAR H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1]; +H5_DLLVAR H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SMALL[1]; +H5_DLLVAR H5FS_section_class_t H5MF_FSPACE_SECT_CLS_LARGE[1]; /******************************/ @@ -143,22 +183,23 @@ H5_DLLVAR H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1]; /******************************/ /* Allocator routines */ -H5_DLL herr_t H5MF__alloc_start(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type); -H5_DLL herr_t H5MF__alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type); +H5_DLL herr_t H5MF_open_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type); +H5_DLL herr_t H5MF_start_fstype(H5F_t *f, hid_t dxpl_id, H5F_mem_page_t type); + +H5_DLL htri_t H5MF_find_sect(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size, H5FS_t *fspace, haddr_t *addr); +H5_DLL herr_t H5MF_add_sect(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, H5FS_t *fspace, H5MF_free_section_t *node); + H5_DLL herr_t H5MF_sects_dump(H5F_t *f, hid_t dxpl_id, FILE *stream); -/* 'simple' section routines */ -H5_DLL H5MF_free_section_t *H5MF_sect_simple_new(haddr_t sect_off, +H5_DLL void H5MF_alloc_to_fs_type(H5F_t *f, H5FD_mem_t alloc_type, hsize_t size, H5F_mem_page_t *fs_type); + +/* 'simple/small/large' section routines */ +H5_DLL H5MF_free_section_t *H5MF_sect_new(unsigned ctype, haddr_t sect_off, hsize_t sect_size); -H5_DLL htri_t H5MF_sect_simple_can_shrink(const H5FS_section_info_t *_sect, - void *udata); -H5_DLL herr_t H5MF_sect_simple_shrink(H5FS_section_info_t **_sect, - void *udata); -H5_DLL herr_t H5MF_sect_simple_free(H5FS_section_info_t *sect); +H5_DLL herr_t H5MF_sect_free(H5FS_section_info_t *sect); + /* Block aggregator routines */ -H5_DLL haddr_t H5MF_aggr_alloc(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, - H5F_blk_aggr_t *other_aggr, H5FD_mem_t type, hsize_t size); H5_DLL htri_t H5MF_aggr_try_extend(H5F_t *f, hid_t dxpl_id, H5F_blk_aggr_t *aggr, H5FD_mem_t type, haddr_t abs_blk_end, hsize_t extra_requested); H5_DLL htri_t H5MF_aggr_can_absorb(const H5F_t *f, const H5F_blk_aggr_t *aggr, diff --git a/src/H5MFprivate.h b/src/H5MFprivate.h index e258677..70322c3 100644 --- a/src/H5MFprivate.h +++ b/src/H5MFprivate.h @@ -58,8 +58,6 @@ H5_DLL herr_t H5MF_try_close(H5F_t *f, hid_t dxpl_id); /* File space allocation routines */ H5_DLL haddr_t H5MF_alloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, hsize_t size); H5_DLL haddr_t H5MF_aggr_vfd_alloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, hsize_t size); -H5_DLL haddr_t H5MF_vfd_alloc(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type, - hsize_t size, hbool_t keep_fragment); H5_DLL herr_t H5MF_xfree(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t size); H5_DLL herr_t H5MF_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, @@ -80,6 +78,14 @@ H5_DLL htri_t H5MF_aggrs_try_shrink_eoa(H5F_t *f, hid_t dxpl_id); H5_DLL herr_t H5MF_settle_raw_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled); H5_DLL herr_t H5MF_settle_meta_data_fsm(H5F_t *f, hid_t dxpl_id, hbool_t *fsm_settled); +/* This function has to be declared in H5MFprivate.h as it is needed + * in our test code to allow us to manually start a self referential + * free space manager prior to the first file space allocations / + * deallocation without causing assertion failures on the first + * file space allocation / deallocation. + */ +H5_DLL herr_t H5MF_tidy_self_referential_fsm_hack(H5F_t *f, hid_t dxpl_id); + /* Debugging routines */ #ifdef H5MF_DEBUGGING H5_DLL herr_t H5MF_sects_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, diff --git a/src/H5MFsection.c b/src/H5MFsection.c index e5a0cf0..617cb59 100644 --- a/src/H5MFsection.c +++ b/src/H5MFsection.c @@ -57,18 +57,47 @@ /* Local Prototypes */ /********************/ -/* 'simple' section callbacks */ -static H5FS_section_info_t *H5MF_sect_simple_deserialize(const H5FS_section_class_t *cls, +/* 'simple/small/large' section callbacks */ +static H5FS_section_info_t *H5MF_sect_deserialize(const H5FS_section_class_t *cls, hid_t dxpl_id, const uint8_t *buf, haddr_t sect_addr, hsize_t sect_size, unsigned *des_flags); +static herr_t H5MF_sect_valid(const H5FS_section_class_t *cls, + const H5FS_section_info_t *sect, hid_t dxpl_id); +static H5FS_section_info_t *H5MF_sect_split(H5FS_section_info_t *sect, + hsize_t frag_size); + + +/* 'simple' section callbacks */ static htri_t H5MF_sect_simple_can_merge(const H5FS_section_info_t *sect1, const H5FS_section_info_t *sect2, void *udata); -static herr_t H5MF_sect_simple_merge(H5FS_section_info_t *sect1, +static herr_t H5MF_sect_simple_merge(H5FS_section_info_t **sect1, H5FS_section_info_t *sect2, void *udata); -static herr_t H5MF_sect_simple_valid(const H5FS_section_class_t *cls, - const H5FS_section_info_t *sect, hid_t dxpl_id); -static H5FS_section_info_t *H5MF_sect_simple_split(H5FS_section_info_t *sect, - hsize_t frag_size); +static htri_t H5MF_sect_simple_can_shrink(const H5FS_section_info_t *_sect, + void *udata); +static herr_t H5MF_sect_simple_shrink(H5FS_section_info_t **_sect, + void *udata); + + +/* 'small' section callbacks */ +static herr_t H5MF_sect_small_add(H5FS_section_info_t **_sect, unsigned *flags, void *_udata); +static htri_t H5MF_sect_small_can_merge(const H5FS_section_info_t *sect1, + const H5FS_section_info_t *sect2, void *udata); +static herr_t H5MF_sect_small_merge(H5FS_section_info_t **sect1, + H5FS_section_info_t *sect2, void *udata); +static htri_t H5MF_sect_small_can_shrink(const H5FS_section_info_t *_sect, + void *udata); +static herr_t H5MF_sect_small_shrink(H5FS_section_info_t **_sect, + void *udata); + +/* 'large' section callbacks */ +static htri_t H5MF_sect_large_can_merge(const H5FS_section_info_t *sect1, + const H5FS_section_info_t *sect2, void *udata); +static herr_t H5MF_sect_large_merge(H5FS_section_info_t **sect1, + H5FS_section_info_t *sect2, void *udata); +static htri_t H5MF_sect_large_can_shrink(const H5FS_section_info_t *_sect, + void *udata); +static herr_t H5MF_sect_large_shrink(H5FS_section_info_t **_sect, + void *udata); /*********************/ /* Package Variables */ @@ -89,17 +118,68 @@ H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1] = {{ /* Object methods */ NULL, /* Add section */ NULL, /* Serialize section */ - H5MF_sect_simple_deserialize, /* Deserialize section */ + H5MF_sect_deserialize, /* Deserialize section */ H5MF_sect_simple_can_merge, /* Can sections merge? */ H5MF_sect_simple_merge, /* Merge sections */ H5MF_sect_simple_can_shrink, /* Can section shrink container?*/ H5MF_sect_simple_shrink, /* Shrink container w/section */ - H5MF_sect_simple_free, /* Free section */ - H5MF_sect_simple_valid, /* Check validity of section */ - H5MF_sect_simple_split, /* Split section node for alignment */ + H5MF_sect_free, /* Free section */ + H5MF_sect_valid, /* Check validity of section */ + H5MF_sect_split, /* Split section node for alignment */ NULL, /* Dump debugging for section */ }}; +/* Class info for "small" free space sections */ +H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SMALL[1] = {{ + /* Class variables */ + H5MF_FSPACE_SECT_SMALL, /* Section type */ + 0, /* Extra serialized size */ + H5FS_CLS_MERGE_SYM | H5FS_CLS_ADJUST_OK, /* Class flags */ + NULL, /* Class private info */ + + /* Class methods */ + NULL, /* Initialize section class */ + NULL, /* Terminate section class */ + + /* Object methods */ + H5MF_sect_small_add, /* Add section */ + NULL, /* Serialize section */ + H5MF_sect_deserialize, /* Deserialize section */ + H5MF_sect_small_can_merge, /* Can sections merge? */ + H5MF_sect_small_merge, /* Merge sections */ + H5MF_sect_small_can_shrink, /* Can section shrink container?*/ + H5MF_sect_small_shrink, /* Shrink container w/section */ + H5MF_sect_free, /* Free section */ + H5MF_sect_valid, /* Check validity of section */ + H5MF_sect_split, /* Split section node for alignment */ + NULL, /* Dump debugging for section */ +}}; + +/* Class info for "large" free space sections */ +H5FS_section_class_t H5MF_FSPACE_SECT_CLS_LARGE[1] = {{ + /* Class variables */ + H5MF_FSPACE_SECT_LARGE, /* Section type */ + 0, /* Extra serialized size */ + H5FS_CLS_MERGE_SYM | H5FS_CLS_ADJUST_OK, /* Class flags */ + NULL, /* Class private info */ + + /* Class methods */ + NULL, /* Initialize section class */ + NULL, /* Terminate section class */ + + /* Object methods */ + NULL, /* Add section */ + NULL, /* Serialize section */ + H5MF_sect_deserialize, /* Deserialize section */ + H5MF_sect_large_can_merge, /* Can sections merge? */ + H5MF_sect_large_merge, /* Merge sections */ + H5MF_sect_large_can_shrink, /* Can section shrink container?*/ + H5MF_sect_large_shrink, /* Shrink container w/section */ + H5MF_sect_free, /* Free section */ + H5MF_sect_valid, /* Check validity of section */ + H5MF_sect_split, /* Split section node for alignment */ + NULL, /* Dump debugging for section */ +}}; /*****************************/ /* Library Private Variables */ @@ -113,12 +193,15 @@ H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1] = {{ /* Declare a free list to manage the H5MF_free_section_t struct */ H5FL_DEFINE(H5MF_free_section_t); +/* + * "simple/small/large" section callbacks + */ /*------------------------------------------------------------------------- - * Function: H5MF_sect_simple_new + * Function: H5MF_sect_new * - * Purpose: Create a new 'simple' section and return it to the caller + * Purpose: Create a new section of "ctype" and return it to the caller * * Return: Pointer to new section on success/NULL on failure * @@ -129,9 +212,9 @@ H5FL_DEFINE(H5MF_free_section_t); *------------------------------------------------------------------------- */ H5MF_free_section_t * -H5MF_sect_simple_new(haddr_t sect_off, hsize_t sect_size) +H5MF_sect_new(unsigned ctype, haddr_t sect_off, hsize_t sect_size) { - H5MF_free_section_t *sect = NULL; /* 'Simple' free space section to add */ + H5MF_free_section_t *sect; /* 'Simple' free space section to add */ H5MF_free_section_t *ret_value = NULL; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -148,7 +231,7 @@ H5MF_sect_simple_new(haddr_t sect_off, hsize_t sect_size) sect->sect_info.size = sect_size; /* Set the section's class & state */ - sect->sect_info.type = H5MF_FSPACE_SECT_SIMPLE; + sect->sect_info.type = ctype; sect->sect_info.state = H5FS_SECT_LIVE; /* Set return value */ @@ -156,13 +239,43 @@ H5MF_sect_simple_new(haddr_t sect_off, hsize_t sect_size) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5MF_sect_simple_new() */ +} /* end H5MF_sect_new() */ /*------------------------------------------------------------------------- - * Function: H5MF_sect_simple_deserialize + * Function: H5MF_sect_free * - * Purpose: Deserialize a buffer into a "live" single section + * Purpose: Free a 'simple/small/large' section node + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Tuesday, January 8, 2008 + * + *------------------------------------------------------------------------- + */ +herr_t +H5MF_sect_free(H5FS_section_info_t *_sect) +{ + H5MF_free_section_t *sect = (H5MF_free_section_t *)_sect; /* File free section */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check arguments. */ + HDassert(sect); + + /* Release the section */ + sect = H5FL_FREE(H5MF_free_section_t, sect); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5MF_sect_free() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_deserialize + * + * Purpose: Deserialize a buffer into a "live" section * * Return: Success: non-negative * Failure: negative @@ -173,7 +286,7 @@ done: *------------------------------------------------------------------------- */ static H5FS_section_info_t * -H5MF_sect_simple_deserialize(const H5FS_section_class_t H5_ATTR_UNUSED *cls, +H5MF_sect_deserialize(const H5FS_section_class_t *cls, hid_t H5_ATTR_UNUSED dxpl_id, const uint8_t H5_ATTR_UNUSED *buf, haddr_t sect_addr, hsize_t sect_size, unsigned H5_ATTR_UNUSED *des_flags) { @@ -183,11 +296,12 @@ H5MF_sect_simple_deserialize(const H5FS_section_class_t H5_ATTR_UNUSED *cls, FUNC_ENTER_NOAPI_NOINIT /* Check arguments. */ + HDassert(cls); HDassert(H5F_addr_defined(sect_addr)); HDassert(sect_size); /* Create free space section for block */ - if(NULL == (sect = H5MF_sect_simple_new(sect_addr, sect_size))) + if(NULL == (sect = H5MF_sect_new(cls->type, sect_addr, sect_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't initialize free space section") /* Set return value */ @@ -195,10 +309,80 @@ H5MF_sect_simple_deserialize(const H5FS_section_class_t H5_ATTR_UNUSED *cls, done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5MF_sect_simple_deserialize() */ +} /* H5MF_sect_deserialize() */ /*------------------------------------------------------------------------- + * Function: H5MF_sect_valid + * + * Purpose: Check the validity of a section + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Tuesday, January 8, 2008 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF_sect_valid(const H5FS_section_class_t H5_ATTR_UNUSED *cls, + const H5FS_section_info_t +#ifdef NDEBUG + H5_ATTR_UNUSED +#endif /* NDEBUG */ + *_sect, hid_t H5_ATTR_UNUSED dxpl_id) +{ +#ifndef NDEBUG + const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */ +#endif /* NDEBUG */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check arguments. */ + HDassert(sect); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5MF_sect_valid() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_split + * + * Purpose: Split SECT into 2 sections: fragment for alignment & the aligned section + * SECT's addr and size are updated to point to the aligned section + * + * Return: Success: the fragment for aligning sect + * Failure: null + * + * Programmer: Vailin Choi, July 29, 2008 + * + *------------------------------------------------------------------------- + */ +static H5FS_section_info_t * +H5MF_sect_split(H5FS_section_info_t *sect, hsize_t frag_size) +{ + H5MF_free_section_t *ret_value; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Allocate space for new section */ + if(NULL == (ret_value = H5MF_sect_new(sect->type, sect->addr, frag_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't initialize free space section") + + /* Set new section's info */ + sect->addr += frag_size; + sect->size -= frag_size; + +done: + FUNC_LEAVE_NOAPI((H5FS_section_info_t *)ret_value) +} /* end H5MF_sect_split() */ + +/* + * "simple" section callbacks + */ + +/*------------------------------------------------------------------------- * Function: H5MF_sect_simple_can_merge * * Purpose: Can two sections of this type merge? @@ -252,10 +436,10 @@ H5MF_sect_simple_can_merge(const H5FS_section_info_t *_sect1, *------------------------------------------------------------------------- */ static herr_t -H5MF_sect_simple_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, +H5MF_sect_simple_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2, void H5_ATTR_UNUSED *_udata) { - H5MF_free_section_t *sect1 = (H5MF_free_section_t *)_sect1; /* File free section */ + H5MF_free_section_t **sect1 = (H5MF_free_section_t **)_sect1; /* File free section */ H5MF_free_section_t *sect2 = (H5MF_free_section_t *)_sect2; /* File free section */ herr_t ret_value = SUCCEED; /* Return value */ @@ -263,16 +447,16 @@ H5MF_sect_simple_merge(H5FS_section_info_t *_sect1, H5FS_section_info_t *_sect2, /* Check arguments. */ HDassert(sect1); - HDassert(sect1->sect_info.type == H5MF_FSPACE_SECT_SIMPLE); + HDassert((*sect1)->sect_info.type == H5MF_FSPACE_SECT_SIMPLE); HDassert(sect2); HDassert(sect2->sect_info.type == H5MF_FSPACE_SECT_SIMPLE); - HDassert(H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr)); + HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr)); /* Add second section's size to first section */ - sect1->sect_info.size += sect2->sect_info.size; + (*sect1)->sect_info.size += sect2->sect_info.size; /* Get rid of second section */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)sect2) < 0) + if(H5MF_sect_free((H5FS_section_info_t *)sect2) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node") done: @@ -293,7 +477,7 @@ done: * *------------------------------------------------------------------------- */ -htri_t +static htri_t H5MF_sect_simple_can_shrink(const H5FS_section_info_t *_sect, void *_udata) { const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */ @@ -392,7 +576,7 @@ done: * *------------------------------------------------------------------------- */ -herr_t +static herr_t H5MF_sect_simple_shrink(H5FS_section_info_t **_sect, void *_udata) { H5MF_free_section_t **sect = (H5MF_free_section_t **)_sect; /* File free section */ @@ -411,8 +595,8 @@ H5MF_sect_simple_shrink(H5FS_section_info_t **_sect, void *_udata) /* Sanity check */ HDassert(H5F_INTENT(udata->f) & H5F_ACC_RDWR); - /* Release section's space at EOA with file driver */ - if(H5FD_free(udata->f->shared->lf, udata->dxpl_id, udata->alloc_type, udata->f, (*sect)->sect_info.addr, (*sect)->sect_info.size) < 0) + /* Release section's space at EOA */ + if(H5F_free(udata->f, udata->dxpl_id, udata->alloc_type, (*sect)->sect_info.addr, (*sect)->sect_info.size) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "driver free request failed") } /* end if */ else { @@ -427,7 +611,7 @@ H5MF_sect_simple_shrink(H5FS_section_info_t **_sect, void *_udata) /* Check for freeing section */ if(udata->shrink != H5MF_SHRINK_SECT_ABSORB_AGGR) { /* Free section */ - if(H5MF_sect_simple_free((H5FS_section_info_t *)*sect) < 0) + if(H5MF_sect_free((H5FS_section_info_t *)*sect) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") /* Mark section as freed, for free space manager */ @@ -438,100 +622,474 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5MF_sect_simple_shrink() */ +/* + * "small" section callbacks + */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_small_add + * + * Purpose: Perform actions on a small "meta" action before adding it to the free space manager: + * 1) Drop the section if it is at page end and its size <= page end threshold + * 2) Adjust section size to include page end threshold if + * (section size + threshold) is at page end + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF_sect_small_add(H5FS_section_info_t **_sect, unsigned *flags, void *_udata) +{ + H5MF_free_section_t **sect = (H5MF_free_section_t **)_sect; /* Fractal heap free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + haddr_t sect_end; + hsize_t rem, prem; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Entering, section {%a, %Hu}\n", FUNC, (*sect)->sect_info.addr, (*sect)->sect_info.size); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Do not adjust the section raw data or global heap data */ + if(udata->alloc_type == H5FD_MEM_DRAW || udata->alloc_type == H5FD_MEM_GHEAP) + HGOTO_DONE(ret_value); + + sect_end = (*sect)->sect_info.addr + (*sect)->sect_info.size; + rem = sect_end % udata->f->shared->fs_page_size; + prem = udata->f->shared->fs_page_size - rem; + + /* Drop the section if it is at page end and its size is <= pgend threshold */ + if(!rem && (*sect)->sect_info.size <= H5F_PGEND_META_THRES(udata->f) && (*flags & H5FS_ADD_RETURNED_SPACE)) { + if(H5MF_sect_free((H5FS_section_info_t *)(*sect)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node") + *sect = NULL; + *flags &= (unsigned)~H5FS_ADD_RETURNED_SPACE; + *flags |= H5FS_PAGE_END_NO_ADD; +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: section is dropped\n", FUNC); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + } /* end if */ + /* Adjust the section if it is not at page end but its size + pgend threshold is at page end */ + else + if(prem <= H5F_PGEND_META_THRES(udata->f)) { + (*sect)->sect_info.size += prem; +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: section is adjusted {%a, %Hu}\n", FUNC, (*sect)->sect_info.addr, (*sect)->sect_info.size); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_small_add() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_small_can_shrink + * + * Purpose: Can this section shrink the container? + * + * Note: A small section is allowed to shrink only at closing. + * + * Return: Success: non-negative (TRUE/FALSE) + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5MF_sect_small_can_shrink(const H5FS_section_info_t *_sect, void *_udata) +{ + const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + haddr_t eoa; /* End of address space in the file */ + haddr_t end; /* End of section to extend */ + htri_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check arguments. */ + HDassert(sect); + HDassert(udata); + HDassert(udata->f); + + /* Retrieve the end of the file's address space */ + if(HADDR_UNDEF == (eoa = H5FD_get_eoa(udata->f->shared->lf, udata->alloc_type))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* Compute address of end of section to check */ + end = sect->sect_info.addr + sect->sect_info.size; + + /* Check if the section is exactly at the end of the allocated space in the file */ + if(H5F_addr_eq(end, eoa) && sect->sect_info.size == udata->f->shared->fs_page_size) { + udata->shrink = H5MF_SHRINK_EOA; + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: section {%a, %Hu}, shrinks file, eoa = %a\n", FUNC, sect->sect_info.addr, sect->sect_info.size, eoa); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + /* Indicate shrinking can occur */ + HGOTO_DONE(TRUE) + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_small_can_shrink() */ + /*------------------------------------------------------------------------- - * Function: H5MF_sect_simple_free + * Function: H5MF_sect_small_shrink * - * Purpose: Free a 'single' section node + * Purpose: Shrink container with section * * Return: Success: non-negative - * Failure: negative + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 * - * Programmer: Quincey Koziol - * Tuesday, January 8, 2008 + *------------------------------------------------------------------------- + */ +static herr_t +H5MF_sect_small_shrink(H5FS_section_info_t **_sect, void *_udata) +{ + H5MF_free_section_t **sect = (H5MF_free_section_t **)_sect; /* File free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check arguments. */ + HDassert(sect); + HDassert((*sect)->sect_info.type == H5MF_FSPACE_SECT_SMALL); + HDassert(udata); + HDassert(udata->f); + HDassert(udata->shrink == H5MF_SHRINK_EOA); + HDassert(H5F_INTENT(udata->f) & H5F_ACC_RDWR); + + /* Release section's space at EOA */ + if(H5F_free(udata->f, udata->dxpl_id, udata->alloc_type, (*sect)->sect_info.addr, (*sect)->sect_info.size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "driver free request failed") + + /* Free section */ + if(H5MF_sect_free((H5FS_section_info_t *)*sect) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") + + /* Mark section as freed, for free space manager */ + *sect = NULL; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_small_shrink() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_small_can_merge + * + * Purpose: Can two sections of this type merge? + * + * Note: Second section must be "after" first section + * The "merged" section cannot cross page boundary. + * + * Return: Success: non-negative (TRUE/FALSE) + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 * *------------------------------------------------------------------------- */ -herr_t -H5MF_sect_simple_free(H5FS_section_info_t *_sect) +static htri_t +H5MF_sect_small_can_merge(const H5FS_section_info_t *_sect1, + const H5FS_section_info_t *_sect2, void *_udata) { - H5MF_free_section_t *sect = (H5MF_free_section_t *)_sect; /* File free section */ + const H5MF_free_section_t *sect1 = (const H5MF_free_section_t *)_sect1; /* File free section */ + const H5MF_free_section_t *sect2 = (const H5MF_free_section_t *)_sect2; /* File free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + htri_t ret_value = FALSE; /* Return value */ FUNC_ENTER_NOAPI_NOINIT_NOERR /* Check arguments. */ - HDassert(sect); + HDassert(sect1); + HDassert(sect2); + HDassert(sect1->sect_info.type == sect2->sect_info.type); /* Checks "MERGE_SYM" flag */ + HDassert(H5F_addr_lt(sect1->sect_info.addr, sect2->sect_info.addr)); - /* Release the section */ - sect = H5FL_FREE(H5MF_free_section_t, sect); + /* Check if second section adjoins first section */ + ret_value = H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr); + if(ret_value > 0) + /* If they are on different pages, couldn't merge */ + if((sect1->sect_info.addr / udata->f->shared->fs_page_size) != (((sect2->sect_info.addr + sect2->sect_info.size - 1) / udata->f->shared->fs_page_size))) + ret_value = FALSE; - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5MF_sect_simple_free() */ +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Leaving: ret_value = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_small_can_merge() */ /*------------------------------------------------------------------------- - * Function: H5MF_sect_simple_valid + * Function: H5MF_sect_small_merge * - * Purpose: Check the validity of a section + * Purpose: Merge two sections of this type + * + * Note: Second section always merges into first node. + * If the size of the "merged" section is equal to file space page size, + * free the section. * * Return: Success: non-negative * Failure: negative * - * Programmer: Quincey Koziol - * Tuesday, January 8, 2008 + * Programmer: Vailin Choi; Dec 2012 * *------------------------------------------------------------------------- */ static herr_t -H5MF_sect_simple_valid(const H5FS_section_class_t H5_ATTR_UNUSED *cls, - const H5FS_section_info_t -#ifdef NDEBUG - H5_ATTR_UNUSED -#endif /* NDEBUG */ - *_sect, hid_t H5_ATTR_UNUSED dxpl_id) +H5MF_sect_small_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2, + void *_udata) { -#ifndef NDEBUG - const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */ -#endif /* NDEBUG */ + H5MF_free_section_t **sect1 = (H5MF_free_section_t **)_sect1; /* File free section */ + H5MF_free_section_t *sect2 = (H5MF_free_section_t *)_sect2; /* File free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check arguments. */ + HDassert(sect1); + HDassert((*sect1)->sect_info.type == H5MF_FSPACE_SECT_SMALL); + HDassert(sect2); + HDassert(sect2->sect_info.type == H5MF_FSPACE_SECT_SMALL); + HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr)); + + /* Add second section's size to first section */ + (*sect1)->sect_info.size += sect2->sect_info.size; + + if((*sect1)->sect_info.size == udata->f->shared->fs_page_size) { + if(H5MF_xfree(udata->f, udata->alloc_type, udata->dxpl_id, (*sect1)->sect_info.addr, (*sect1)->sect_info.size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free merged section") + + /* Need to free possible metadata page in the PB cache */ + /* This is in response to the data corruption bug from fheap.c with page buffering + page strategy */ + /* Note: Large metadata page bypasses the PB cache */ + /* Note: Update of raw data page (large or small sized) is handled by the PB cache */ + if(udata->f->shared->page_buf != NULL && udata->alloc_type != H5FD_MEM_DRAW) + if(H5PB_remove_entry(udata->f, (*sect1)->sect_info.addr) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "can't free merged section") + + if(H5MF_sect_free((H5FS_section_info_t *)(*sect1)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node") + *sect1 = NULL; + } /* end if */ + + /* Get rid of second section */ + if(H5MF_sect_free((H5FS_section_info_t *)sect2) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_small_merge() */ + +/* + * "Large" section callbacks + */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_large_can_merge (same as H5MF_sect_simple_can_merge) + * + * Purpose: Can two sections of this type merge? + * + * Note: Second section must be "after" first section + * + * Return: Success: non-negative (TRUE/FALSE) + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5MF_sect_large_can_merge(const H5FS_section_info_t *_sect1, + const H5FS_section_info_t *_sect2, void H5_ATTR_UNUSED *_udata) +{ + const H5MF_free_section_t *sect1 = (const H5MF_free_section_t *)_sect1; /* File free section */ + const H5MF_free_section_t *sect2 = (const H5MF_free_section_t *)_sect2; /* File free section */ + htri_t ret_value = FALSE; /* Return value */ FUNC_ENTER_NOAPI_NOINIT_NOERR /* Check arguments. */ + HDassert(sect1); + HDassert(sect2); + HDassert(sect1->sect_info.type == sect2->sect_info.type); /* Checks "MERGE_SYM" flag */ + HDassert(H5F_addr_lt(sect1->sect_info.addr, sect2->sect_info.addr)); + + ret_value = H5F_addr_eq(sect1->sect_info.addr + sect1->sect_info.size, sect2->sect_info.addr); + +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: Leaving: ret_value = %t\n", FUNC, ret_value); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_large_can_merge() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_large_merge (same as H5MF_sect_simple_merge) + * + * Purpose: Merge two sections of this type + * + * Note: Second section always merges into first node + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5MF_sect_large_merge(H5FS_section_info_t **_sect1, H5FS_section_info_t *_sect2, + void H5_ATTR_UNUSED *_udata) +{ + H5MF_free_section_t **sect1 = (H5MF_free_section_t **)_sect1; /* File free section */ + H5MF_free_section_t *sect2 = (H5MF_free_section_t *)_sect2; /* File free section */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check arguments. */ + HDassert(sect1); + HDassert((*sect1)->sect_info.type == H5MF_FSPACE_SECT_LARGE); + HDassert(sect2); + HDassert(sect2->sect_info.type == H5MF_FSPACE_SECT_LARGE); + HDassert(H5F_addr_eq((*sect1)->sect_info.addr + (*sect1)->sect_info.size, sect2->sect_info.addr)); + + /* Add second section's size to first section */ + (*sect1)->sect_info.size += sect2->sect_info.size; + + /* Get rid of second section */ + if(H5MF_sect_free((H5FS_section_info_t *)sect2) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free section node") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_large_merge() */ + + +/*------------------------------------------------------------------------- + * Function: H5MF_sect_large_can_shrink + * + * Purpose: Can this section shrink the container? + * + * Return: Success: non-negative (TRUE/FALSE) + * Failure: negative + * + * Programmer: Vailin Choi; Dec 2012 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5MF_sect_large_can_shrink(const H5FS_section_info_t *_sect, void *_udata) +{ + const H5MF_free_section_t *sect = (const H5MF_free_section_t *)_sect; /* File free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + haddr_t eoa; /* End of address space in the file */ + haddr_t end; /* End of section to extend */ + htri_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check arguments. */ HDassert(sect); + HDassert(sect->sect_info.type == H5MF_FSPACE_SECT_LARGE); + HDassert(udata); + HDassert(udata->f); - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5MF_sect_simple_valid() */ + /* Retrieve the end of the file's address space */ + if(HADDR_UNDEF == (eoa = H5FD_get_eoa(udata->f->shared->lf, udata->alloc_type))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* Compute address of end of section to check */ + end = sect->sect_info.addr + sect->sect_info.size; + + /* Check if the section is exactly at the end of the allocated space in the file */ + if(H5F_addr_eq(end, eoa) && sect->sect_info.size >= udata->f->shared->fs_page_size) { + /* Set the shrinking type */ + udata->shrink = H5MF_SHRINK_EOA; +#ifdef H5MF_ALLOC_DEBUG_MORE +HDfprintf(stderr, "%s: section {%a, %Hu}, shrinks file, eoa = %a\n", FUNC, sect->sect_info.addr, sect->sect_info.size, eoa); +#endif /* H5MF_ALLOC_DEBUG_MORE */ + /* Indicate shrinking can occur */ + HGOTO_DONE(TRUE) + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_large_can_shrink() */ + /*------------------------------------------------------------------------- - * Function: H5MF_sect_simple_split + * Function: H5MF_sect_large_shrink * - * Purpose: Split SECT into 2 sections: fragment for alignment & the aligned section - * SECT's addr and size are updated to point to the aligned section + * Purpose: Shrink a large-sized section * - * Return: Success: the fragment for aligning sect - * Failure: null + * Return: Success: non-negative + * Failure: negative * - * Programmer: Vailin Choi, July 29, 2008 + * Programmer: Vailin Choi; Dec 2012 * *------------------------------------------------------------------------- */ -static H5FS_section_info_t * -H5MF_sect_simple_split(H5FS_section_info_t *sect, hsize_t frag_size) +static herr_t +H5MF_sect_large_shrink(H5FS_section_info_t **_sect, void *_udata) { - H5MF_free_section_t *ret_value = NULL; /* Return value */ + H5MF_free_section_t **sect = (H5MF_free_section_t **)_sect; /* File free section */ + H5MF_sect_ud_t *udata = (H5MF_sect_ud_t *)_udata; /* User data for callback */ + hsize_t frag_size = 0; /* Fragment size */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT - /* Allocate space for new section */ - if(NULL == (ret_value = H5MF_sect_simple_new(sect->addr, frag_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't initialize free space section") + /* Check arguments. */ + HDassert(sect); + HDassert((*sect)->sect_info.type == H5MF_FSPACE_SECT_LARGE); + HDassert(udata); + HDassert(udata->f); + HDassert(udata->shrink == H5MF_SHRINK_EOA); + HDassert(H5F_INTENT(udata->f) & H5F_ACC_RDWR); + HDassert(H5F_PAGED_AGGR(udata->f)); - /* Set new section's info */ - sect->addr += frag_size; - sect->size -= frag_size; + /* Calculate possible mis-aligned fragment */ + H5MF_EOA_MISALIGN(udata->f, (*sect)->sect_info.addr, udata->f->shared->fs_page_size, frag_size); + + /* Free full pages from EOA */ + /* Retain partial page in the free-space manager so as to keep EOA at page boundary */ + if(H5F_free(udata->f, udata->dxpl_id, udata->alloc_type, (*sect)->sect_info.addr+frag_size, (*sect)->sect_info.size-frag_size) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, FAIL, "driver free request failed") + + if(frag_size) /* Adjust section size for the partial page */ + (*sect)->sect_info.size = frag_size; + else { + /* Free section */ + if(H5MF_sect_free((H5FS_section_info_t *)*sect) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTRELEASE, FAIL, "can't free simple section node") + + /* Mark section as freed, for free space manager */ + *sect = NULL; + } /* end else */ done: - FUNC_LEAVE_NOAPI((H5FS_section_info_t *)ret_value) -} /* end H5MF_sect_simple_split() */ + FUNC_LEAVE_NOAPI(ret_value) +} /* H5MF_sect_large_shrink() */ diff --git a/src/H5Oalloc.c b/src/H5Oalloc.c index 4f98cfa..40c9c38 100644 --- a/src/H5Oalloc.c +++ b/src/H5Oalloc.c @@ -2023,6 +2023,7 @@ H5O_merge_null(H5F_t *f, hid_t dxpl_id, H5O_t *oh) /* Second message has been merged, delete it */ if(merged_msg) { H5O_chunk_proxy_t *curr_chk_proxy; /* Chunk that message is in */ + htri_t result; /* Release any information/memory for second message */ H5O_msg_free_mesg(curr_msg2); @@ -2050,6 +2051,13 @@ H5O_merge_null(H5F_t *f, hid_t dxpl_id, H5O_t *oh) /* (Don't bother reducing size of message array for now -QAK) */ oh->nmesgs--; + /* The merge null message might span the entire chunk: scan for empty chunk to remove */ + if((result = H5O_remove_empty_chunks(f, dxpl_id, oh)) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPACK, FAIL, "can't remove empty chunk") + else if(result > 0) + /* Get out of loop */ + break; + /* If the merged message is too large, shrink the chunk */ if(curr_msg->raw_size >= H5O_MESG_MAX_SIZE) if(H5O_alloc_shrink_chunk(f, dxpl_id, oh, curr_msg->chunkno) < 0) diff --git a/src/H5Ocache_image.c b/src/H5Ocache_image.c index 65f6aa2..bea99a2 100644 --- a/src/H5Ocache_image.c +++ b/src/H5Ocache_image.c @@ -33,6 +33,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* Files */ #include "H5FLprivate.h" /* Free Lists */ #include "H5Opkg.h" /* Object headers */ #include "H5MFprivate.h" /* File space management */ @@ -290,9 +291,50 @@ H5O__mdci_delete(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, void *_mesg) HDassert(mesg); /* Free file space for cache image */ - if(H5F_addr_defined(mesg->addr)) - if(H5MF_xfree(f, H5FD_MEM_SUPER, dxpl_id, mesg->addr, mesg->size) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to free file space for cache image block") + if(H5F_addr_defined(mesg->addr)) { + /* The space for the cache image block was allocated directly + * from the VFD layer at the end of file. As this was the + * last file space allocation before shutdown, the cache image + * should still be the last item in the file. + * + * If the hack to work around the self referential free space + * manager issue is in use, file space for the non-empty self + * referential free space managers was also allocated from VFD + * layer at the end of file. Since these allocations directly + * preceeded the cache image allocation they should be directly + * adjacent to the cache image block at the end of file. + * + * In this case, just call H5MF_tidy_self_referential_fsm_hack(). + * + * That routine will float the self referential free space + * managers, and reduce the eoa to its value just prior to + * allocation of space for same. Since the cache image appears + * just after the self referential free space managers, this + * will release the file space for the cache image as well. + * + * Note that in this case, there must not have been any file + * space allocations / deallocations prior to the free of the + * cache image. Verify this to the extent possible. + * + * If the hack to work around the persistant self referential + * free space manager issue is NOT in use, just call H5MF_xfree() + * to release the cache iamge. In principle, we should be able + * to just reduce the EOA to the base address of the cache + * image block, as there shouldn't be any file space allocation + * before the first metadata cache access. However, given + * time constraints, I don't want to go there now. + */ + if(H5F_FIRST_ALLOC_DEALLOC(f)) { + HDassert(HADDR_UNDEF !=H5F_EOA_PRE_FSM_FSALLOC(f)); + HDassert(H5F_addr_ge(mesg->addr, H5F_EOA_PRE_FSM_FSALLOC(f))); + if(H5MF_tidy_self_referential_fsm_hack(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "tidy of self referential fsm hack failed") + } /* end if */ + else { + if(H5MF_xfree(f, H5FD_MEM_SUPER, dxpl_id, mesg->addr, mesg->size) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to free file space for cache image block") + } /* end else */ + } /* end if */ done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Ofsinfo.c b/src/H5Ofsinfo.c index 938d319..37ec627 100644 --- a/src/H5Ofsinfo.c +++ b/src/H5Ofsinfo.c @@ -23,12 +23,13 @@ * *------------------------------------------------------------------------- */ - +#define H5F_FRIEND /*suppress error about including H5Fpkg */ #include "H5Omodule.h" /* This source code file is part of the H5O module */ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* File access */ #include "H5FLprivate.h" /* Free lists */ #include "H5Opkg.h" /* Object headers */ @@ -66,7 +67,8 @@ const H5O_msg_class_t H5O_MSG_FSINFO[1] = {{ }}; /* Current version of free-space manager info information */ -#define H5O_FSINFO_VERSION 0 +#define H5O_FSINFO_VERSION_0 0 +#define H5O_FSINFO_VERSION_1 1 /* Declare a free list to manage the H5O_fsinfo_t struct */ H5FL_DEFINE_STATIC(H5O_fsinfo_t); @@ -85,12 +87,13 @@ H5FL_DEFINE_STATIC(H5O_fsinfo_t); *------------------------------------------------------------------------- */ static void * -H5O_fsinfo_decode(H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5_ATTR_UNUSED *open_oh, +H5O_fsinfo_decode(H5F_t *f, hid_t dxpl_id, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, unsigned H5_ATTR_UNUSED *ioflags, const uint8_t *p) { - H5O_fsinfo_t *fsinfo = NULL; /* free-space manager info */ - H5FD_mem_t type; /* Memory type for iteration */ - void *ret_value = NULL; /* Return value */ + H5O_fsinfo_t *fsinfo = NULL; /* File space info message */ + H5F_mem_page_t ptype; /* Memory type for iteration */ + unsigned vers; /* message version */ + void *ret_value = NULL; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -98,26 +101,83 @@ H5O_fsinfo_decode(H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5_ATTR_UNUSED * HDassert(f); HDassert(p); - /* Version of message */ - if(*p++ != H5O_FSINFO_VERSION) - HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "bad version number for message") - /* Allocate space for message */ if(NULL == (fsinfo = H5FL_CALLOC(H5O_fsinfo_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - fsinfo->strategy = (H5F_file_space_type_t)*p++; /* file space strategy */ - H5F_DECODE_LENGTH(f, p, fsinfo->threshold); /* free space section size threshold */ + for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + fsinfo->fs_addr[ptype - 1] = HADDR_UNDEF; - /* Addresses of free space managers: only exist for H5F_FILE_SPACE_ALL_PERSIST */ - if(fsinfo->strategy == H5F_FILE_SPACE_ALL_PERSIST) { - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - H5F_addr_decode(f, &p, &(fsinfo->fs_addr[type-1])); - } /* end if */ - else { - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - fsinfo->fs_addr[type-1] = HADDR_UNDEF; - } /* end else */ + /* Version of message */ + vers = *p++; + + if(vers == H5O_FSINFO_VERSION_0) { + H5F_file_space_type_t strategy; /* Strategy */ + hsize_t threshold; /* Threshold */ + H5FD_mem_t type; /* Memory type for iteration */ + + fsinfo->persist = H5F_FREE_SPACE_PERSIST_DEF; + fsinfo->threshold = H5F_FREE_SPACE_THRESHOLD_DEF; + fsinfo->page_size = H5F_FILE_SPACE_PAGE_SIZE_DEF; + fsinfo->pgend_meta_thres = H5F_FILE_SPACE_PGEND_META_THRES; + fsinfo->eoa_pre_fsm_fsalloc = HADDR_UNDEF; + + strategy = (H5F_file_space_type_t)*p++; /* File space strategy */ + H5F_DECODE_LENGTH(f, p, threshold); /* Free-space section threshold */ + + /* Map version 0 (deprecated) to version 1 message */ + switch(strategy) { + + case H5F_FILE_SPACE_ALL_PERSIST: + fsinfo->strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; + fsinfo->persist = TRUE; + fsinfo->threshold = threshold; + if(HADDR_UNDEF == (fsinfo->eoa_pre_fsm_fsalloc = H5F_get_eoa(f, H5FD_MEM_DEFAULT)) ) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to get file size") + for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) + H5F_addr_decode(f, &p, &(fsinfo->fs_addr[type-1])); + break; + + case H5F_FILE_SPACE_ALL: + fsinfo->strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; + fsinfo->threshold = threshold; + break; + + case H5F_FILE_SPACE_AGGR_VFD: + fsinfo->strategy = H5F_FSPACE_STRATEGY_AGGR; + break; + + case H5F_FILE_SPACE_VFD: + fsinfo->strategy = H5F_FSPACE_STRATEGY_NONE; + break; + + case H5F_FILE_SPACE_NTYPES: + case H5F_FILE_SPACE_DEFAULT: + default: + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file space strategy") + } /* end switch */ + + fsinfo->mapped = TRUE; + + } else { + HDassert(vers == H5O_FSINFO_VERSION_1); + + fsinfo->strategy = (H5F_fspace_strategy_t)*p++; /* File space strategy */ + fsinfo->persist = *p++; /* Free-space persist or not */ + H5F_DECODE_LENGTH(f, p, fsinfo->threshold); /* Free-space section threshold */ + + H5F_DECODE_LENGTH(f, p, fsinfo->page_size); /* File space page size */ + UINT16DECODE(p, fsinfo->pgend_meta_thres); /* Page end metdata threshold */ + H5F_addr_decode(f, &p, &(fsinfo->eoa_pre_fsm_fsalloc)); /* EOA before free-space header and section info */ + + /* Decode addresses of free space managers, if persisting */ + if(fsinfo->persist) { + for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + H5F_addr_decode(f, &p, &(fsinfo->fs_addr[ptype - 1])); + } /* end if */ + + fsinfo->mapped = FALSE; + } /* Set return value */ ret_value = fsinfo; @@ -145,7 +205,7 @@ static herr_t H5O_fsinfo_encode(H5F_t *f, hbool_t H5_ATTR_UNUSED disable_shared, uint8_t *p, const void *_mesg) { const H5O_fsinfo_t *fsinfo = (const H5O_fsinfo_t *)_mesg; - H5FD_mem_t type; /* Memory type for iteration */ + H5F_mem_page_t ptype; /* Memory type for iteration */ FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -154,14 +214,20 @@ H5O_fsinfo_encode(H5F_t *f, hbool_t H5_ATTR_UNUSED disable_shared, uint8_t *p, c HDassert(p); HDassert(fsinfo); - *p++ = H5O_FSINFO_VERSION; /* message version */ - *p++ = fsinfo->strategy; /* file space strategy */ - H5F_ENCODE_LENGTH(f, p, fsinfo->threshold); /* free-space section size threshold */ + *p++ = H5O_FSINFO_VERSION_1; /* message version */ + *p++ = fsinfo->strategy; /* File space strategy */ + *p++ = (unsigned char)fsinfo->persist; /* Free-space persist or not */ + H5F_ENCODE_LENGTH(f, p, fsinfo->threshold); /* Free-space section size threshold */ + + H5F_ENCODE_LENGTH(f, p, fsinfo->page_size); /* File space page size */ + UINT16ENCODE(p, fsinfo->pgend_meta_thres); /* Page end metadata threshold */ + H5F_addr_encode(f, &p, fsinfo->eoa_pre_fsm_fsalloc); /* EOA before free-space header and section info */ - /* Addresses of free space managers: only exist for H5F_FILE_SPACE_ALL_PERSIST */ - if(fsinfo->strategy == H5F_FILE_SPACE_ALL_PERSIST) { - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - H5F_addr_encode(f, &p, fsinfo->fs_addr[type-1]); + /* Store addresses of free-space managers, if persisting */ + if(fsinfo->persist) { + /* Addresses of free-space managers */ + for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + H5F_addr_encode(f, &p, fsinfo->fs_addr[ptype - 1]); } /* end if */ FUNC_LEAVE_NOAPI(SUCCEED) @@ -224,19 +290,19 @@ static size_t H5O_fsinfo_size(const H5F_t *f, hbool_t H5_ATTR_UNUSED disable_shared, const void *_mesg) { const H5O_fsinfo_t *fsinfo = (const H5O_fsinfo_t *)_mesg; - size_t fs_addr_size = 0; size_t ret_value = 0; /* Return value */ FUNC_ENTER_NOAPI_NOINIT_NOERR - - /* Addresses of free-space managers exist only for H5F_FILE_SPACE_ALL_PERSIST type */ - if(H5F_FILE_SPACE_ALL_PERSIST == fsinfo->strategy) - fs_addr_size = (H5FD_MEM_NTYPES - 1) * (size_t)H5F_SIZEOF_ADDR(f); - - ret_value = 2 /* Version & strategy */ - + (size_t)H5F_SIZEOF_SIZE(f) /* Threshold */ - + fs_addr_size; /* Addresses of free-space managers */ + ret_value = 3 /* Version, strategy & persist */ + + (size_t)H5F_SIZEOF_SIZE(f) /* Free-space section threshold */ + + (size_t)H5F_SIZEOF_SIZE(f) /* File space page size */ + + 2 /* Page end meta threshold */ + + (size_t)H5F_SIZEOF_ADDR(f); + + /* Free-space manager addresses */ + if(fsinfo->persist) + ret_value += (H5F_MEM_PAGE_NTYPES - 1) * (size_t)H5F_SIZEOF_ADDR(f); FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_fsinfo_size() */ @@ -282,7 +348,7 @@ H5O_fsinfo_debug(H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id, const vo int indent, int fwidth) { const H5O_fsinfo_t *fsinfo = (const H5O_fsinfo_t *) _mesg; - H5FD_mem_t type; /* Memory type for iteration */ + H5F_mem_page_t ptype; /* Free-space types for iteration */ FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -293,16 +359,48 @@ H5O_fsinfo_debug(H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id, const vo HDassert(indent >= 0); HDassert(fwidth >= 0); - HDfprintf(stream, "%*s%-*s %u\n", indent, "", fwidth, - "File space strategy:", fsinfo->strategy); + HDfprintf(stream, "%*s%-*s ", indent, "", fwidth, "File space strategy:"); + switch(fsinfo->strategy) { + case H5F_FSPACE_STRATEGY_FSM_AGGR: + HDfprintf(stream, "%s\n", "H5F_FSPACE_STRATEGY_FSM_AGGR"); + break; + + case H5F_FSPACE_STRATEGY_PAGE: + HDfprintf(stream, "%s\n", "H5F_FSPACE_STRATEGY_PAGE"); + break; + + case H5F_FSPACE_STRATEGY_AGGR: + HDfprintf(stream, "%s\n", "H5F_FSPACE_STRATEGY_AGGR"); + break; + + case H5F_FSPACE_STRATEGY_NONE: + HDfprintf(stream, "%s\n", "H5F_FSPACE_STRATEGY_NONE"); + break; + + case H5F_FSPACE_STRATEGY_NTYPES: + default: + HDfprintf(stream, "%s\n", "unknown"); + } /* end switch */ + + HDfprintf(stream, "%*s%-*s %t\n", indent, "", fwidth, + "Free-space persist:", fsinfo->persist); HDfprintf(stream, "%*s%-*s %Hu\n", indent, "", fwidth, - "Free space section threshold:", fsinfo->threshold); + "Free-space section threshold:", fsinfo->threshold); + + HDfprintf(stream, "%*s%-*s %Hu\n", indent, "", fwidth, + "File space page size:", fsinfo->page_size); + + HDfprintf(stream, "%*s%-*s %u\n", indent, "", fwidth, + "Page end metadata threshold:", fsinfo->pgend_meta_thres); + + HDfprintf(stream, "%*s%-*s %a\n", indent, "", fwidth, + "eoa_pre_fsm_fsalloc:", fsinfo->eoa_pre_fsm_fsalloc); - if(fsinfo->strategy == H5F_FILE_SPACE_ALL_PERSIST) { - for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) - HDfprintf(stream, "%*s%-*s %a\n", indent, "", fwidth, - "Free space manager address:", fsinfo->fs_addr[type-1]); + if(fsinfo->persist) { + for(ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; H5_INC_ENUM(H5F_mem_page_t, ptype)) + HDfprintf(stream, "%*s%-*s %a\n", indent, "", fwidth, + "Free space manager address:", fsinfo->fs_addr[ptype-1]); } /* end if */ FUNC_LEAVE_NOAPI(SUCCEED) diff --git a/src/H5Omessage.c b/src/H5Omessage.c index 376c888..7e6463a 100644 --- a/src/H5Omessage.c +++ b/src/H5Omessage.c @@ -2253,3 +2253,55 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_flush_msgs() */ + +/*------------------------------------------------------------------------- + * Function: H5O_msg_get_flags + * + * Purpose: Queries a message's message flags in the object header + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin; Jan 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_msg_get_flags(const H5O_loc_t *loc, unsigned type_id, hid_t dxpl_id, uint8_t *flags) +{ + H5O_t *oh = NULL; /* Object header to use */ + const H5O_msg_class_t *type; /* Actual H5O class type for the ID */ + H5O_mesg_t *idx_msg; /* Pointer to message to modify */ + unsigned idx; /* Index of message to modify */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* check args */ + HDassert(loc); + HDassert(loc->file); + HDassert(H5F_addr_defined(loc->addr)); + HDassert(type_id < NELMTS(H5O_msg_class_g)); + type = H5O_msg_class_g[type_id]; /* map the type ID to the actual type object */ + HDassert(type); + + /* Get the object header */ + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to protect object header") + + /* Locate message of correct type */ + for(idx = 0, idx_msg = &oh->mesg[0]; idx < oh->nmesgs; idx++, idx_msg++) + if(type == idx_msg->type) + break; + + if(idx == oh->nmesgs) + HGOTO_ERROR(H5E_OHDR, H5E_NOTFOUND, FAIL, "message type not found") + + /* Set return value */ + *flags = idx_msg->flags; + +done: + if(oh && H5O_unprotect(loc, dxpl_id, oh, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to release object header") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_msg_get_flags() */ diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 71c512c..f0fbe72 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -791,9 +791,16 @@ typedef unsigned H5O_unknown_t; /* Original message type ID */ * (Data structure in memory) */ typedef struct H5O_fsinfo_t { - H5F_file_space_type_t strategy; /* File space strategy */ - hsize_t threshold; /* Free space section threshold */ - haddr_t fs_addr[H5FD_MEM_NTYPES-1]; /* Addresses of free space managers */ + H5F_fspace_strategy_t strategy; /* File space strategy */ + hbool_t persist; /* Persisting free-space or not */ + hsize_t threshold; /* Free-space section threshold */ + hsize_t page_size; /* For paged aggregation: file space page size */ + size_t pgend_meta_thres; /* For paged aggregation: page end metadata threshold */ + haddr_t eoa_pre_fsm_fsalloc; /* For paged aggregation: the eoa before free-space headers & sinfo */ + haddr_t fs_addr[H5F_MEM_PAGE_NTYPES - 1]; /* 13 addresses of free-space managers */ + /* For non-paged aggregation: only 6 addresses are used */ + hbool_t mapped; /* Not stored */ + /* Indicate the message is mapped from version 0 to version 1 */ } H5O_fsinfo_t; /* @@ -920,6 +927,7 @@ H5_DLL void* H5O_msg_decode(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, unsigned type_id, const unsigned char *buf); H5_DLL herr_t H5O_msg_delete(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, unsigned type_id, void *mesg); +H5_DLL herr_t H5O_msg_get_flags(const H5O_loc_t *loc, unsigned type_id, hid_t dxpl_id, uint8_t *flags); /* Object metadata flush/refresh routines */ H5_DLL herr_t H5O_flush_common(H5O_loc_t *oloc, hid_t obj_id, hid_t dxpl_id); diff --git a/src/H5PB.c b/src/H5PB.c new file mode 100644 index 0000000..c67ae59 --- /dev/null +++ b/src/H5PB.c @@ -0,0 +1,1533 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5PB.c + * + * Purpose: Page Buffer routines. + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + +#define H5F_FRIEND /*suppress error about including H5Fpkg */ +#include "H5PBmodule.h" /* This source code file is part of the H5PB module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* Files */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5Iprivate.h" /* IDs */ +#include "H5PBpkg.h" /* File access */ +#include "H5SLprivate.h" /* Skip List */ + + +/****************/ +/* Local Macros */ +/****************/ +#define H5PB__PREPEND(page_ptr, head_ptr, tail_ptr, len) { \ + if((head_ptr) == NULL) { \ + (head_ptr) = (page_ptr); \ + (tail_ptr) = (page_ptr); \ + } /* end if */ \ + else { \ + (head_ptr)->prev = (page_ptr); \ + (page_ptr)->next = (head_ptr); \ + (head_ptr) = (page_ptr); \ + } /* end else */ \ + (len)++; \ +} /* H5PB__PREPEND() */ + +#define H5PB__REMOVE(page_ptr, head_ptr, tail_ptr, len) { \ + if((head_ptr) == (page_ptr)) { \ + (head_ptr) = (page_ptr)->next; \ + if((head_ptr) != NULL) \ + (head_ptr)->prev = NULL; \ + } /* end if */ \ + else \ + (page_ptr)->prev->next = (page_ptr)->next; \ + if((tail_ptr) == (page_ptr)) { \ + (tail_ptr) = (page_ptr)->prev; \ + if((tail_ptr) != NULL) \ + (tail_ptr)->next = NULL; \ + } /* end if */ \ + else \ + (page_ptr)->next->prev = (page_ptr)->prev; \ + page_ptr->next = NULL; \ + page_ptr->prev = NULL; \ + (len)--; \ +} + +#define H5PB__INSERT_LRU(page_buf, page_ptr) { \ + HDassert(page_buf); \ + HDassert(page_ptr); \ + /* insert the entry at the head of the list. */ \ + H5PB__PREPEND((page_ptr), (page_buf)->LRU_head_ptr, \ + (page_buf)->LRU_tail_ptr, (page_buf)->LRU_list_len) \ +} + +#define H5PB__REMOVE_LRU(page_buf, page_ptr) { \ + HDassert(page_buf); \ + HDassert(page_ptr); \ + /* remove the entry from the list. */ \ + H5PB__REMOVE((page_ptr), (page_buf)->LRU_head_ptr, \ + (page_buf)->LRU_tail_ptr, (page_buf)->LRU_list_len) \ +} + +#define H5PB__MOVE_TO_TOP_LRU(page_buf, page_ptr) { \ + HDassert(page_buf); \ + HDassert(page_ptr); \ + /* Remove entry and insert at the head of the list. */ \ + H5PB__REMOVE((page_ptr), (page_buf)->LRU_head_ptr, \ + (page_buf)->LRU_tail_ptr, (page_buf)->LRU_list_len) \ + H5PB__PREPEND((page_ptr), (page_buf)->LRU_head_ptr, \ + (page_buf)->LRU_tail_ptr, (page_buf)->LRU_list_len) \ +} + + +/******************/ +/* Local Typedefs */ +/******************/ + +/* Iteration context for destroying page buffer */ +typedef struct { + H5PB_t *page_buf; + hbool_t actual_slist; +} H5PB_ud1_t; + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ +static herr_t H5PB__insert_entry(H5PB_t *page_buf, H5PB_entry_t *page_entry); +static htri_t H5PB__make_space(const H5F_io_info2_t *fio_info, H5PB_t *page_buf, H5FD_mem_t inserted_type); +static herr_t H5PB__write_entry(const H5F_io_info2_t *fio_info, H5PB_entry_t *page_entry); + + +/*********************/ +/* Package Variables */ +/*********************/ + +/* Package initialization variable */ +hbool_t H5_PKG_INIT_VAR = FALSE; + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ +/* Declare a free list to manage the H5PB_t struct */ +H5FL_DEFINE_STATIC(H5PB_t); + +/* Declare a free list to manage the H5PB_entry_t struct */ +H5FL_DEFINE_STATIC(H5PB_entry_t); + + + +/*------------------------------------------------------------------------- + * Function: H5PB_reset_stats + * + * Purpose: This function was created without documentation. + * What follows is my best understanding of Mohamad's intent. + * + * Reset statistics collected for the page buffer layer. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_reset_stats(H5PB_t *page_buf) +{ + FUNC_ENTER_NOAPI_NOERR + + /* Sanity checks */ + HDassert(page_buf); + + page_buf->accesses[0] = 0; + page_buf->accesses[1] = 0; + page_buf->hits[0] = 0; + page_buf->hits[1] = 0; + page_buf->misses[0] = 0; + page_buf->misses[1] = 0; + page_buf->evictions[0] = 0; + page_buf->evictions[1] = 0; + page_buf->bypasses[0] = 0; + page_buf->bypasses[1] = 0; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5PB_reset_stats() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_get_stats + * + * Purpose: This function was created without documentation. + * What follows is my best understanding of Mohamad's intent. + * + * Retrieve statistics collected about page accesses for the page buffer layer. + * --accesses: the number of metadata and raw data accesses to the page buffer layer + * --hits: the number of metadata and raw data hits in the page buffer layer + * --misses: the number of metadata and raw data misses in the page buffer layer + * --evictions: the number of metadata and raw data evictions from the page buffer layer + * --bypasses: the number of metadata and raw data accesses that bypass the page buffer layer + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_get_stats(const H5PB_t *page_buf, unsigned accesses[2], unsigned hits[2], + unsigned misses[2], unsigned evictions[2], unsigned bypasses[2]) +{ + FUNC_ENTER_NOAPI_NOERR + + /* Sanity checks */ + HDassert(page_buf); + + accesses[0] = page_buf->accesses[0]; + accesses[1] = page_buf->accesses[1]; + hits[0] = page_buf->hits[0]; + hits[1] = page_buf->hits[1]; + misses[0] = page_buf->misses[0]; + misses[1] = page_buf->misses[1]; + evictions[0] = page_buf->evictions[0]; + evictions[1] = page_buf->evictions[1]; + bypasses[0] = page_buf->bypasses[0]; + bypasses[1] = page_buf->bypasses[1]; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5PB_get_stats */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_print_stats() + * + * Purpose: This function was created without documentation. + * What follows is my best understanding of Mohamad's intent. + * + * Print out statistics collected for the page buffer layer. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_print_stats(const H5PB_t *page_buf) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(page_buf); + + printf("PAGE BUFFER STATISTICS:\n"); + + printf("******* METADATA\n"); + printf("\t Total Accesses: %u\n", page_buf->accesses[0]); + printf("\t Hits: %u\n", page_buf->hits[0]); + printf("\t Misses: %u\n", page_buf->misses[0]); + printf("\t Evictions: %u\n", page_buf->evictions[0]); + printf("\t Bypasses: %u\n", page_buf->bypasses[0]); + printf("\t Hit Rate = %f%%\n", ((double)page_buf->hits[0]/(page_buf->accesses[0] - page_buf->bypasses[0]))*100); + printf("*****************\n\n"); + + printf("******* RAWDATA\n"); + printf("\t Total Accesses: %u\n", page_buf->accesses[1]); + printf("\t Hits: %u\n", page_buf->hits[1]); + printf("\t Misses: %u\n", page_buf->misses[1]); + printf("\t Evictions: %u\n", page_buf->evictions[1]); + printf("\t Bypasses: %u\n", page_buf->bypasses[1]); + printf("\t Hit Rate = %f%%\n", ((double)page_buf->hits[1]/(page_buf->accesses[1]-page_buf->bypasses[0]))*100); + printf("*****************\n\n"); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5PB_print_stats */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_create + * + * Purpose: Create and setup the PB on the file. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_create(H5F_t *f, size_t size, unsigned page_buf_min_meta_perc, unsigned page_buf_min_raw_perc) +{ + H5PB_t *page_buf = NULL; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + + /* Check args */ + if(f->shared->fs_strategy != H5F_FSPACE_STRATEGY_PAGE) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "Enabling Page Buffering requires PAGE file space strategy") + /* round down the size if it is larger than the page size */ + else if(size > f->shared->fs_page_size) { + hsize_t temp_size; + + temp_size = (size / f->shared->fs_page_size) * f->shared->fs_page_size; + H5_CHECKED_ASSIGN(size, size_t, temp_size, hsize_t); + } /* end if */ + else if(0 != size % f->shared->fs_page_size) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTINIT, FAIL, "Page Buffer size must be >= to the page size") + + /* Allocate the new page buffering structure */ + if(NULL == (page_buf = H5FL_CALLOC(H5PB_t))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "memory allocation failed") + + page_buf->max_size = size; + H5_CHECKED_ASSIGN(page_buf->page_size, size_t, f->shared->fs_page_size, hsize_t); + page_buf->min_meta_perc = page_buf_min_meta_perc; + page_buf->min_raw_perc = page_buf_min_raw_perc; + + /* Calculate the minimum page count for metadata and raw data + * based on the fractions provided + */ + page_buf->min_meta_count = (unsigned)((size * page_buf_min_meta_perc) / (f->shared->fs_page_size * 100)); + page_buf->min_raw_count = (unsigned)((size * page_buf_min_raw_perc) / (f->shared->fs_page_size * 100)); + + if(NULL == (page_buf->slist_ptr = H5SL_create(H5SL_TYPE_HADDR, NULL))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTCREATE, FAIL, "can't create skip list") + if(NULL == (page_buf->mf_slist_ptr = H5SL_create(H5SL_TYPE_HADDR, NULL))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTCREATE, FAIL, "can't create skip list") + + if(NULL == (page_buf->page_fac = H5FL_fac_init(page_buf->page_size))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTINIT, FAIL, "can't create page factory") + + f->shared->page_buf = page_buf; + +done: + if(ret_value < 0) { + if(page_buf != NULL) { + if(page_buf->slist_ptr != NULL) + H5SL_close(page_buf->slist_ptr); + if(page_buf->mf_slist_ptr != NULL) + H5SL_close(page_buf->mf_slist_ptr); + if(page_buf->page_fac != NULL) + H5FL_fac_term(page_buf->page_fac); + page_buf = H5FL_FREE(H5PB_t, page_buf); + } /* end if */ + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5PB_create */ + + +/*------------------------------------------------------------------------- + * Function: H5PB__flush_cb + * + * Purpose: Callback to flush PB skiplist entries. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PB__flush_cb(void *item, void H5_ATTR_UNUSED *key, void *_op_data) +{ + H5PB_entry_t *page_entry = (H5PB_entry_t *)item; /* Pointer to page entry node */ + const H5F_io_info2_t *fio_info = (const H5F_io_info2_t *)_op_data; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(page_entry); + HDassert(fio_info); + + /* Flush the page if it's dirty */ + if(page_entry->is_dirty) + if(H5PB__write_entry(fio_info, page_entry) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "file write failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5PB__flush_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_flush + * + * Purpose: Flush/Free all the PB entries to the file. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_flush(const H5F_io_info2_t *fio_info) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(fio_info); + HDassert(fio_info->f); + HDassert(fio_info->meta_dxpl); + HDassert(fio_info->raw_dxpl); + + /* Flush all the entries in the PB skiplist, if we have write access on the file */ + if(fio_info->f->shared->page_buf && (H5F_ACC_RDWR & H5F_INTENT(fio_info->f))) { + H5PB_t *page_buf = fio_info->f->shared->page_buf; + + /* Iterate over all entries in page buffer skip list */ + if(H5SL_iterate(page_buf->slist_ptr, H5PB__flush_cb, (void *)fio_info)) + HGOTO_ERROR(H5E_PAGEBUF, H5E_BADITER, FAIL, "can't flush page buffer skip list") + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5PB_flush */ + + +/*------------------------------------------------------------------------- + * Function: H5PB__dest_cb + * + * Purpose: Callback to free PB skiplist entries. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PB__dest_cb(void *item, void H5_ATTR_UNUSED *key, void *_op_data) +{ + H5PB_entry_t *page_entry = (H5PB_entry_t *)item; /* Pointer to page entry node */ + H5PB_ud1_t *op_data = (H5PB_ud1_t *)_op_data; + + FUNC_ENTER_STATIC_NOERR + + /* Sanity checking */ + HDassert(page_entry); + HDassert(op_data); + HDassert(op_data->page_buf); + + /* Remove entry from LRU list */ + if(op_data->actual_slist) { + H5PB__REMOVE_LRU(op_data->page_buf, page_entry) + page_entry->page_buf_ptr = H5FL_FAC_FREE(op_data->page_buf->page_fac, page_entry->page_buf_ptr); + } /* end if */ + + /* Free page entry */ + page_entry = H5FL_FREE(H5PB_entry_t, page_entry); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5PB__dest_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_dest + * + * Purpose: destroy the PB on the file. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_dest(H5F_t *f) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + + /* Destroy page buffer info, if there is any */ + if(f->shared->page_buf) { + H5PB_t *page_buf = f->shared->page_buf; + H5PB_ud1_t op_data; /* Iteration context */ + + /* Set up context info */ + op_data.page_buf = page_buf; + + /* Destroy the skip list containing all the entries in the PB */ + op_data.actual_slist = TRUE; + if(H5SL_destroy(page_buf->slist_ptr, H5PB__dest_cb, &op_data)) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTCLOSEOBJ, FAIL, "can't destroy page buffer skip list") + + /* Destroy the skip list containing the new entries */ + op_data.actual_slist = FALSE; + if(H5SL_destroy(page_buf->mf_slist_ptr, H5PB__dest_cb, &op_data)) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTCLOSEOBJ, FAIL, "can't destroy page buffer skip list") + + /* Destroy the page factory */ + if(H5FL_fac_term(page_buf->page_fac) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTRELEASE, FAIL, "can't destroy page buffer page factory") + +#ifdef QAK +H5PB_print_stats(page_buf); +#endif /* QAK */ + + f->shared->page_buf = H5FL_FREE(H5PB_t, page_buf); + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5PB_dest */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_add_new_page + * + * Purpose: Add a new page to the new page skip list. This is called + * from the MF layer when a new page is allocated to + * indicate to the page buffer layer that a read of the page + * from the file is not necessary since it's an empty page. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_add_new_page(H5F_t *f, H5FD_mem_t type, haddr_t page_addr) +{ + H5PB_t *page_buf = f->shared->page_buf; + H5PB_entry_t *page_entry = NULL; /* pointer to the corresponding page entry */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(page_buf); + + /* If there is an existing page, this means that at some point the + * file free space manager freed and re-allocated a page at the same + * address. No need to do anything here then... + */ + /* MSC - to be safe, might want to dig in the MF layer and remove + * the page when it is freed from this list if it still exists and + * remove this check + */ + if(NULL == H5SL_search(page_buf->mf_slist_ptr, &(page_addr))) { + /* Create the new PB entry */ + if(NULL == (page_entry = H5FL_CALLOC(H5PB_entry_t))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "memory allocation failed") + + /* Initialize page fields */ + page_entry->addr = page_addr; + page_entry->type = (H5F_mem_page_t)type; + page_entry->is_dirty = FALSE; + + /* Insert entry in skip list */ + if(H5SL_insert(page_buf->mf_slist_ptr, page_entry, &(page_entry->addr)) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_BADVALUE, FAIL, "Can't insert entry in skip list") + } /* end if */ + +done: + if(ret_value < 0) + if(page_entry) + page_entry = H5FL_FREE(H5PB_entry_t, page_entry); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5PB_add_new_page */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_update_entry + * + * Purpose: In PHDF5, entries that are written by other processes and just + * marked clean by this process have to have their corresponding + * pages updated if they exist in the page buffer. + * This routine checks and update the pages. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_update_entry(H5PB_t *page_buf, haddr_t addr, size_t size, const void *buf) +{ + H5PB_entry_t *page_entry; /* Pointer to the corresponding page entry */ + haddr_t page_addr; + + FUNC_ENTER_NOAPI_NOERR + + /* Sanity checks */ + HDassert(page_buf); + HDassert(size <= page_buf->page_size); + HDassert(buf); + + /* calculate the aligned address of the first page */ + page_addr = (addr / page_buf->page_size) * page_buf->page_size; + + /* search for the page and update if found */ + page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&page_addr)); + if(page_entry) { + haddr_t offset; + + HDassert(addr + size <= page_addr + page_buf->page_size); + offset = addr - page_addr; + HDmemcpy((uint8_t *)page_entry->page_buf_ptr + offset, buf, size); + + /* move to top of LRU list */ + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + } /* end if */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5PB_update_entry */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_remove_entry + * + * Purpose: Remove possible metadata entry with ADDR from the PB cache. + * This is in response to the data corruption bug from fheap.c + * with page buffering + page strategy. + * Note: Large metadata page bypasses the PB cache. + * Note: Update of raw data page (large or small sized) is handled by the PB cache. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Feb 2017 + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_remove_entry(const H5F_t *f, haddr_t addr) +{ + H5PB_t *page_buf = f->shared->page_buf; + H5PB_entry_t *page_entry = NULL; /* pointer to the page entry being searched */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(page_buf); + + /* Search for address in the skip list */ + page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&addr)); + + /* If found, remove the entry from the PB cache */ + if(page_entry) { + HDassert(page_entry->type != H5F_MEM_PAGE_DRAW); + if(NULL == H5SL_remove(page_buf->slist_ptr, &(page_entry->addr))) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "Page Entry is not in skip list") + + /* Remove from LRU list */ + H5PB__REMOVE_LRU(page_buf, page_entry) + HDassert(H5SL_count(page_buf->slist_ptr) == page_buf->LRU_list_len); + + page_buf->meta_count--; + + page_entry->page_buf_ptr = H5FL_FAC_FREE(page_buf->page_fac, page_entry->page_buf_ptr); + page_entry = H5FL_FREE(H5PB_entry_t, page_entry); + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5PB_remove_entry */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_read + * + * Purpose: Reads in the data from the page containing it if it exists + * in the PB cache; otherwise reads in the page through the VFD. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_read(const H5F_io_info2_t *fio_info, H5FD_mem_t type, haddr_t addr, + size_t size, void *buf/*out*/) +{ + H5PB_t *page_buf; /* Page buffering info for this file */ + H5PB_entry_t *page_entry; /* Pointer to the corresponding page entry */ + H5FD_io_info_t fdio_info; /* File driver I/O info */ + haddr_t first_page_addr, last_page_addr; /* Addresses of the first and last pages covered by I/O */ + haddr_t offset; + haddr_t search_addr; /* Address of current page */ + hsize_t num_touched_pages; /* Number of pages accessed */ + size_t access_size; + hbool_t bypass_pb = FALSE; /* Whether to bypass page buffering */ + hsize_t i; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(fio_info); + + /* Get pointer to page buffer info for this file */ + page_buf = fio_info->f->shared->page_buf; + +#ifdef H5_HAVE_PARALLEL + if(H5F_HAS_FEATURE(fio_info->f, H5FD_FEAT_HAS_MPI)) { +#if 1 + bypass_pb = TRUE; +#else + /* MSC - why this stopped working ? */ + int mpi_size; + + if((mpi_size = H5F_mpi_get_size(fio_info->f)) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "can't retrieve MPI communicator size") + if(1 != mpi_size) + bypass_pb = TRUE; +#endif + } /* end if */ +#endif + + /* If page buffering is disabled, or the I/O size is larger than that of a + * single page, or if this is a parallel raw data access, bypass page + * buffering. + */ + if(NULL == page_buf || size >= page_buf->page_size || + (bypass_pb && H5FD_MEM_DRAW == type)) { + if(H5F__accum_read(fio_info, type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_READERROR, FAIL, "read through metadata accumulator failed") + + /* Update statistics */ + if(page_buf) { + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->bypasses[1] ++; + else + page_buf->bypasses[0] ++; + } /* end if */ + + /* If page buffering is disabled, or if this is a large metadata access, + * or if this is parallel raw data access, we are done here + */ + if(NULL == page_buf || (size >= page_buf->page_size && H5FD_MEM_DRAW != type) || + (bypass_pb && H5FD_MEM_DRAW == type)) + HGOTO_DONE(SUCCEED) + } /* end if */ + + /* Update statistics */ + if(page_buf) { + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->accesses[1]++; + else + page_buf->accesses[0]++; + } /* end if */ + + /* Calculate the aligned address of the first page */ + first_page_addr = (addr / page_buf->page_size) * page_buf->page_size; + + /* For Raw data calculate the aligned address of the last page and + * the number of pages accessed if more than 1 page is accessed + */ + if(H5FD_MEM_DRAW == type) { + last_page_addr = ((addr + size - 1) / page_buf->page_size) * page_buf->page_size; + + /* How many pages does this write span */ + num_touched_pages = (last_page_addr / page_buf->page_size + 1) - + (first_page_addr / page_buf->page_size); + if(first_page_addr == last_page_addr) { + HDassert(1 == num_touched_pages); + last_page_addr = HADDR_UNDEF; + } /* end if */ + } /* end if */ + /* Otherwise set last page addr to HADDR_UNDEF */ + else { + num_touched_pages = 1; + last_page_addr = HADDR_UNDEF; + } /* end else */ + + /* Translate to file driver I/O info object */ + fdio_info.file = fio_info->f->shared->lf; + fdio_info.meta_dxpl = fio_info->meta_dxpl; + fdio_info.raw_dxpl = fio_info->raw_dxpl; + + /* Copy raw data from dirty pages into the read buffer if the read + request spans pages in the page buffer*/ + if(H5FD_MEM_DRAW == type && size >= page_buf->page_size) { + H5SL_node_t *node; + + /* For each touched page in the page buffer, check if it + * exists in the page Buffer and is dirty. If it does, we + * update the buffer with what's in the page so we get the up + * to date data into the buffer after the big read from the file. + */ + node = H5SL_find(page_buf->slist_ptr, (void *)(&first_page_addr)); + for(i = 0; i < num_touched_pages; i++) { + search_addr = i*page_buf->page_size + first_page_addr; + + /* if we still haven't located a starting page, search again */ + if(!node && i!=0) + node = H5SL_find(page_buf->slist_ptr, (void *)(&search_addr)); + + /* if the current page is in the Page Buffer, do the updates */ + if(node) { + page_entry = (H5PB_entry_t *)H5SL_item(node); + + HDassert(page_entry); + + /* If the current page address falls out of the access + block, then there are no more pages to go over */ + if(page_entry->addr >= addr + size) + break; + + HDassert(page_entry->addr == search_addr); + + if(page_entry->is_dirty) { + /* special handling for the first page if it is not a full page access */ + if(i == 0 && first_page_addr != addr) { + offset = addr - first_page_addr; + HDassert(page_buf->page_size > offset); + + HDmemcpy(buf, (uint8_t *)page_entry->page_buf_ptr + offset, + page_buf->page_size - (size_t)offset); + + /* move to top of LRU list */ + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + } /* end if */ + /* special handling for the last page if it is not a full page access */ + else if(num_touched_pages > 1 && i == num_touched_pages-1 && search_addr < addr+size) { + offset = (num_touched_pages-2)*page_buf->page_size + + (page_buf->page_size - (addr - first_page_addr)); + + HDmemcpy((uint8_t *)buf + offset, page_entry->page_buf_ptr, + (size_t)((addr + size) - last_page_addr)); + + /* move to top of LRU list */ + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + } /* end else-if */ + /* copy the entire fully accessed pages */ + else { + offset = i*page_buf->page_size; + + HDmemcpy((uint8_t *)buf+(i*page_buf->page_size) , page_entry->page_buf_ptr, + page_buf->page_size); + } /* end else */ + } /* end if */ + node = H5SL_next(node); + } /* end if */ + } /* end for */ + } /* end if */ + else { + /* A raw data access could span 1 or 2 PB entries at this point so + we need to handle that */ + HDassert(1 == num_touched_pages || 2 == num_touched_pages); + for(i = 0 ; i < num_touched_pages; i++) { + haddr_t buf_offset; + + /* Calculate the aligned address of the page to search for it in the skip list */ + search_addr = (0==i ? first_page_addr : last_page_addr); + + /* Calculate the access size if the access spans more than 1 page */ + if(1 == num_touched_pages) + access_size = size; + else + access_size = (0 == i ? (size_t)((first_page_addr + page_buf->page_size) - addr) : (size - access_size)); + + /* Lookup the page in the skip list */ + page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&search_addr)); + + /* if found */ + if(page_entry) { + offset = (0 == i ? addr - page_entry->addr : 0); + buf_offset = (0 == i ? 0 : size - access_size); + + /* copy the requested data from the page into the input buffer */ + HDmemcpy((uint8_t *)buf + buf_offset, (uint8_t *)page_entry->page_buf_ptr + offset, access_size); + + /* Update LRU */ + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + + /* Update statistics */ + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->hits[1]++; + else + page_buf->hits[0]++; + } /* end if */ + /* if not found */ + else { + void *new_page_buf = NULL; + size_t page_size = page_buf->page_size; + haddr_t eoa; + + /* make space for new entry */ + if((H5SL_count(page_buf->slist_ptr) * page_buf->page_size) >= page_buf->max_size) { + htri_t can_make_space; + + /* check if we can make space in page buffer */ + if((can_make_space = H5PB__make_space(fio_info, page_buf, type)) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "make space in Page buffer Failed") + + /* if make_space returns 0, then we can't use the page + buffer for this I/O and we need to bypass */ + if(0 == can_make_space) { + /* make space can't return FALSE on second touched page since the first is of the same type */ + HDassert(0 == i); + + /* read entire block from VFD and return */ + if(H5FD_read(&fdio_info, type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_READERROR, FAIL, "driver read request failed") + + /* Break out of loop */ + break; + } /* end if */ + } /* end if */ + + /* Read page from VFD */ + if(NULL == (new_page_buf = H5FL_FAC_MALLOC(page_buf->page_fac))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTALLOC, FAIL, "memory allocation failed for page buffer entry") + + /* Read page through the VFD layer, but make sure we don't read past the EOA. */ + + /* Retrieve the 'eoa' for the file */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(fio_info->f, type))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* If the entire page falls outside the EOA, then fail */ + if(search_addr > eoa) + HGOTO_ERROR(H5E_PAGEBUF, H5E_BADVALUE, FAIL, "reading an entire page that is outside the file EOA") + + /* Adjust the read size to not go beyond the EOA */ + if(search_addr + page_size > eoa) + page_size = (size_t)(eoa - search_addr); + + /* Read page from VFD */ + if(H5FD_read(&fdio_info, type, search_addr, page_size, new_page_buf) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_READERROR, FAIL, "driver read request failed") + + /* Copy the requested data from the page into the input buffer */ + offset = (0 == i ? addr - search_addr : 0); + buf_offset = (0 == i ? 0 : size - access_size); + HDmemcpy((uint8_t *)buf + buf_offset, (uint8_t *)new_page_buf + offset, access_size); + + /* Create the new PB entry */ + if(NULL == (page_entry = H5FL_CALLOC(H5PB_entry_t))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "memory allocation failed") + + page_entry->page_buf_ptr = new_page_buf; + page_entry->addr = search_addr; + page_entry->type = (H5F_mem_page_t)type; + page_entry->is_dirty = FALSE; + + /* Insert page into PB */ + if(H5PB__insert_entry(page_buf, page_entry) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTSET, FAIL, "error inserting new page in page buffer") + + /* Update statistics */ + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->misses[1]++; + else + page_buf->misses[0]++; + } /* end else */ + } /* end for */ + } /* end else */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PB_read() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB_write + * + * Purpose: Write data into the Page Buffer. If the page exists in the + * cache, update it; otherwise read it from disk, update it, and + * insert into cache. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +herr_t +H5PB_write(const H5F_io_info2_t *fio_info, H5FD_mem_t type, haddr_t addr, + size_t size, const void *buf) +{ + H5PB_t *page_buf; /* Page buffering info for this file */ + H5PB_entry_t *page_entry; /* Pointer to the corresponding page entry */ + H5FD_io_info_t fdio_info; /* File driver I/O info */ + haddr_t first_page_addr, last_page_addr; /* Addresses of the first and last pages covered by I/O */ + haddr_t offset; + haddr_t search_addr; /* Address of current page */ + hsize_t num_touched_pages; /* Number of pages accessed */ + size_t access_size; + hbool_t bypass_pb = FALSE; /* Whether to bypass page buffering */ + hsize_t i; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(fio_info); + HDassert(fio_info->f); + + /* Get pointer to page buffer info for this file */ + page_buf = fio_info->f->shared->page_buf; + +#ifdef H5_HAVE_PARALLEL + if(H5F_HAS_FEATURE(fio_info->f, H5FD_FEAT_HAS_MPI)) { +#if 1 + bypass_pb = TRUE; +#else + /* MSC - why this stopped working ? */ + int mpi_size; + + if((mpi_size = H5F_mpi_get_size(fio_info->f)) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "can't retrieve MPI communicator size") + if(1 != mpi_size) + bypass_pb = TRUE; +#endif + } /* end if */ +#endif + + /* If page buffering is disabled, or the I/O size is larger than that of a + * single page, or if this is a parallel raw data access, bypass page + * buffering. + */ + if(NULL == page_buf || size >= page_buf->page_size || bypass_pb) { + if(H5F__accum_write(fio_info, type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "write through metadata accumulator failed") + + /* Update statistics */ + if(page_buf) { + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->bypasses[1]++; + else + page_buf->bypasses[0]++; + } /* end if */ + + /* If page buffering is disabled, or if this is a large metadata access, + * or if this is a parallel raw data access, we are done here + */ + if(NULL == page_buf || (size >= page_buf->page_size && H5FD_MEM_DRAW != type) || + (bypass_pb && H5FD_MEM_DRAW == type)) + HGOTO_DONE(SUCCEED) + +#ifdef H5_HAVE_PARALLEL + if(bypass_pb) { + if(H5PB_update_entry(page_buf, addr, size, buf) > 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTUPDATE, FAIL, "failed to update PB with metadata cache") + HGOTO_DONE(SUCCEED) + } /* end if */ +#endif + } /* end if */ + + /* Update statistics */ + if(page_buf) { + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->accesses[1]++; + else + page_buf->accesses[0]++; + } /* end if */ + + /* Calculate the aligned address of the first page */ + first_page_addr = (addr / page_buf->page_size) * page_buf->page_size; + + /* For raw data calculate the aligned address of the last page and + * the number of pages accessed if more than 1 page is accessed + */ + if(H5FD_MEM_DRAW == type) { + last_page_addr = (addr + size - 1) / page_buf->page_size * page_buf->page_size; + + /* how many pages does this write span */ + num_touched_pages = (last_page_addr/page_buf->page_size + 1) - + (first_page_addr / page_buf->page_size); + if(first_page_addr == last_page_addr) { + HDassert(1 == num_touched_pages); + last_page_addr = HADDR_UNDEF; + } /* end if */ + } /* end if */ + /* Otherwise set last page addr to HADDR_UNDEF */ + else { + num_touched_pages = 1; + last_page_addr = HADDR_UNDEF; + } /* end else */ + + /* Translate to file driver I/O info object */ + fdio_info.file = fio_info->f->shared->lf; + fdio_info.meta_dxpl = fio_info->meta_dxpl; + fdio_info.raw_dxpl = fio_info->raw_dxpl; + + /* Check if existing pages for raw data need to be updated since raw data access is not atomic */ + if(H5FD_MEM_DRAW == type && size >= page_buf->page_size) { + /* For each touched page, check if it exists in the page buffer, and + * update it with the data in the buffer to keep it up to date + */ + for(i = 0; i < num_touched_pages; i++) { + search_addr = i * page_buf->page_size + first_page_addr; + + /* Special handling for the first page if it is not a full page update */ + if(i == 0 && first_page_addr != addr) { + /* Lookup the page in the skip list */ + page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&search_addr)); + if(page_entry) { + offset = addr - first_page_addr; + HDassert(page_buf->page_size > offset); + + /* Update page's data */ + HDmemcpy((uint8_t *)page_entry->page_buf_ptr + offset, buf, page_buf->page_size - (size_t)offset); + + /* Mark page dirty and push to top of LRU */ + page_entry->is_dirty = TRUE; + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + } /* end if */ + } /* end if */ + /* Special handling for the last page if it is not a full page update */ + else if(num_touched_pages > 1 && i == (num_touched_pages - 1) && + (search_addr + page_buf->page_size) != (addr + size)) { + HDassert(search_addr+page_buf->page_size > addr+size); + + /* Lookup the page in the skip list */ + page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&search_addr)); + if(page_entry) { + offset = (num_touched_pages - 2) * page_buf->page_size + + (page_buf->page_size - (addr - first_page_addr)); + + /* Update page's data */ + HDmemcpy(page_entry->page_buf_ptr, (const uint8_t *)buf + offset, + (size_t)((addr + size) - last_page_addr)); + + /* Mark page dirty and push to top of LRU */ + page_entry->is_dirty = TRUE; + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + } /* end if */ + } /* end else-if */ + /* Discard all fully written pages from the page buffer */ + else { + page_entry = (H5PB_entry_t *)H5SL_remove(page_buf->slist_ptr, (void *)(&search_addr)); + if(page_entry) { + /* Remove from LRU list */ + H5PB__REMOVE_LRU(page_buf, page_entry) + + /* Decrement page count of appropriate type */ + if(H5F_MEM_PAGE_DRAW == page_entry->type || H5F_MEM_PAGE_GHEAP == page_entry->type) + page_buf->raw_count--; + else + page_buf->meta_count--; + + /* Free page info */ + page_entry->page_buf_ptr = H5FL_FAC_FREE(page_buf->page_fac, page_entry->page_buf_ptr); + page_entry = H5FL_FREE(H5PB_entry_t, page_entry); + } /* end if */ + } /* end else */ + } /* end for */ + } /* end if */ + else { + /* An access could span 1 or 2 PBs at this point so we need to handle that */ + HDassert(1 == num_touched_pages || 2 == num_touched_pages); + for(i = 0; i < num_touched_pages; i++) { + haddr_t buf_offset; + + /* Calculate the aligned address of the page to search for it in the skip list */ + search_addr = (0 == i ? first_page_addr : last_page_addr); + + /* Calculate the access size if the access spans more than 1 page */ + if(1 == num_touched_pages) + access_size = size; + else + access_size = (0 == i ? (size_t)(first_page_addr + page_buf->page_size - addr) : (size - access_size)); + + /* Lookup the page in the skip list */ + page_entry = (H5PB_entry_t *)H5SL_search(page_buf->slist_ptr, (void *)(&search_addr)); + + /* If found */ + if(page_entry) { + offset = (0 == i ? addr - page_entry->addr : 0); + buf_offset = (0 == i ? 0 : size - access_size); + + /* Copy the requested data from the input buffer into the page */ + HDmemcpy((uint8_t *)page_entry->page_buf_ptr + offset, (const uint8_t *)buf + buf_offset, access_size); + + /* Mark page dirty and push to top of LRU */ + page_entry->is_dirty = TRUE; + H5PB__MOVE_TO_TOP_LRU(page_buf, page_entry) + + /* Update statistics */ + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->hits[1]++; + else + page_buf->hits[0]++; + } /* end if */ + /* If not found */ + else { + void *new_page_buf; + size_t page_size = page_buf->page_size; + + /* Make space for new entry */ + if((H5SL_count(page_buf->slist_ptr) * page_buf->page_size) >= page_buf->max_size) { + htri_t can_make_space; + + /* Check if we can make space in page buffer */ + if((can_make_space = H5PB__make_space(fio_info, page_buf, type)) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_NOSPACE, FAIL, "make space in Page buffer Failed") + + /* If make_space returns 0, then we can't use the page + * buffer for this I/O and we need to bypass + */ + if(0 == can_make_space) { + HDassert(0 == i); + + /* Write to VFD and return */ + if(H5FD_write(&fdio_info, type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "driver write request failed") + + /* Break out of loop */ + break; + } /* end if */ + } /* end if */ + + /* Don't bother searching if there is no write access */ + if(H5F_ACC_RDWR & H5F_INTENT(fio_info->f)) + /* Lookup & remove the page from the new skip list page if + * it exists to see if this is a new page from the MF layer + */ + page_entry = (H5PB_entry_t *)H5SL_remove(page_buf->mf_slist_ptr, (void *)(&search_addr)); + + /* Calculate offset into the buffer of the page and the user buffer */ + offset = (0 == i ? addr - search_addr : 0); + buf_offset = (0 == i ? 0 : size - access_size); + + /* If found, then just update the buffer pointer to the newly allocate buffer */ + if(page_entry) { + /* Allocate space for the page buffer */ + if(NULL == (new_page_buf = H5FL_FAC_MALLOC(page_buf->page_fac))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTALLOC, FAIL, "memory allocation failed for page buffer entry") + HDmemset(new_page_buf, 0, (size_t)offset); + HDmemset((uint8_t *)new_page_buf + offset + access_size, 0, page_size - ((size_t)offset + access_size)); + + page_entry->page_buf_ptr = new_page_buf; + + /* Update statistics */ + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->hits[1]++; + else + page_buf->hits[0]++; + } /* end if */ + /* Otherwise read page through the VFD layer, but make sure we don't read past the EOA. */ + else { + haddr_t eoa, eof = HADDR_UNDEF; + + /* Allocate space for the page buffer */ + if(NULL == (new_page_buf = H5FL_FAC_CALLOC(page_buf->page_fac))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTALLOC, FAIL, "memory allocation failed for page buffer entry") + + /* Create the new loaded PB entry */ + if(NULL == (page_entry = H5FL_CALLOC(H5PB_entry_t))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTALLOC, FAIL, "memory allocation failed") + + page_entry->page_buf_ptr = new_page_buf; + page_entry->addr = search_addr; + page_entry->type = (H5F_mem_page_t)type; + + /* Retrieve the 'eoa' for the file */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(fio_info->f, type))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* If the entire page falls outside the EOA, then fail */ + if(search_addr > eoa) + HGOTO_ERROR(H5E_PAGEBUF, H5E_BADVALUE, FAIL, "writing to a page that is outside the file EOA") + + /* Retrieve the 'eof' for the file - The MPI-VFD EOF + * returned will most likely be HADDR_UNDEF, so skip + * that check. + */ + if(!H5F_HAS_FEATURE(fio_info->f, H5FD_FEAT_HAS_MPI)) + if(HADDR_UNDEF == (eof = H5FD_get_eof(fio_info->f->shared->lf, H5FD_MEM_DEFAULT))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eof request failed") + + /* Adjust the read size to not go beyond the EOA */ + if(search_addr + page_size > eoa) + page_size = (size_t)(eoa - search_addr); + + if(search_addr < eof) { + if(H5FD_read(&fdio_info, type, search_addr, page_size, new_page_buf) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_READERROR, FAIL, "driver read request failed") + + /* Update statistics */ + if(type == H5FD_MEM_DRAW || type == H5FD_MEM_GHEAP) + page_buf->misses[1]++; + else + page_buf->misses[0]++; + } /* end if */ + } /* end else */ + + /* Copy the requested data from the page into the input buffer */ + HDmemcpy((uint8_t *)new_page_buf + offset, (const uint8_t *)buf+buf_offset, access_size); + + /* Page is dirty now */ + page_entry->is_dirty = TRUE; + + /* Insert page into PB, evicting other pages as necessary */ + if(H5PB__insert_entry(page_buf, page_entry) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTSET, FAIL, "error inserting new page in page buffer") + } /* end else */ + } /* end for */ + } /* end else */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PB_write() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB__insert_entry() + * + * Purpose: ??? + * + * This function was created without documentation. + * What follows is my best understanding of Mohamad's intent. + * + * Insert the supplied page into the page buffer, both the + * skip list and the LRU. + * + * As best I can tell, this function imposes no limit on the + * number of entries in the page buffer beyond an assertion + * failure it the page count exceeds the limit. + * + * JRM -- 12/22/16 + * + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PB__insert_entry(H5PB_t *page_buf, H5PB_entry_t *page_entry) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Insert entry in skip list */ + if(H5SL_insert(page_buf->slist_ptr, page_entry, &(page_entry->addr)) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTINSERT, FAIL, "can't insert entry in skip list") + HDassert(H5SL_count(page_buf->slist_ptr) * page_buf->page_size <= page_buf->max_size); + + /* Increment appropriate page count */ + if(H5F_MEM_PAGE_DRAW == page_entry->type || H5F_MEM_PAGE_GHEAP == page_entry->type) + page_buf->raw_count++; + else + page_buf->meta_count++; + + /* Insert entry in LRU */ + H5PB__INSERT_LRU(page_buf, page_entry) + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PB__insert_entry() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB__make_space() + * + * Purpose: ??? + * + * This function was created without documentation. + * What follows is my best understanding of Mohamad's intent. + * + * If necessary and if possible, evict a page from the page + * buffer to make space for the supplied page. Depending on + * the page buffer configuration and contents, and the page + * supplied this may or may not be possible. + * + * JRM -- 12/22/16 + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +static htri_t +H5PB__make_space(const H5F_io_info2_t *fio_info, H5PB_t *page_buf, + H5FD_mem_t inserted_type) +{ + H5PB_entry_t *page_entry; /* Pointer to page eviction candidate */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(fio_info); + HDassert(page_buf); + + /* Get oldest entry */ + page_entry = page_buf->LRU_tail_ptr; + + if(H5FD_MEM_DRAW == inserted_type) { + /* If threshould is 100% metadata and page buffer is full of + metadata, then we can't make space for raw data */ + if(0 == page_buf->raw_count && page_buf->min_meta_count == page_buf->meta_count) { + HDassert(page_buf->meta_count * page_buf->page_size == page_buf->max_size); + HGOTO_DONE(FALSE) + } /* end if */ + + /* check the metadata threshold before evicting metadata items */ + while(1) { + if(page_entry->prev && H5F_MEM_PAGE_META == page_entry->type && + page_buf->min_meta_count >= page_buf->meta_count) + page_entry = page_entry->prev; + else + break; + } /* end while */ + } /* end if */ + else { + /* If threshould is 100% raw data and page buffer is full of + raw data, then we can't make space for meta data */ + if(0 == page_buf->meta_count && page_buf->min_raw_count == page_buf->raw_count) { + HDassert(page_buf->raw_count * page_buf->page_size == page_buf->max_size); + HGOTO_DONE(FALSE) + } /* end if */ + + /* check the raw data threshold before evicting raw data items */ + while(1) { + if(page_entry->prev && (H5F_MEM_PAGE_DRAW == page_entry->type || H5F_MEM_PAGE_GHEAP == page_entry->type) && + page_buf->min_raw_count >= page_buf->raw_count) + page_entry = page_entry->prev; + else + break; + } /* end while */ + } /* end else */ + + /* Remove from page index */ + if(NULL == H5SL_remove(page_buf->slist_ptr, &(page_entry->addr))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_BADVALUE, FAIL, "Tail Page Entry is not in skip list") + + /* Remove entry from LRU list */ + H5PB__REMOVE_LRU(page_buf, page_entry) + HDassert(H5SL_count(page_buf->slist_ptr) == page_buf->LRU_list_len); + + /* Decrement appropriate page type counter */ + if(H5F_MEM_PAGE_DRAW == page_entry->type || H5F_MEM_PAGE_GHEAP == page_entry->type) + page_buf->raw_count--; + else + page_buf->meta_count--; + + /* Flush page if dirty */ + if(page_entry->is_dirty) + if(H5PB__write_entry(fio_info, page_entry) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "file write failed") + + /* Update statistics */ + if(page_entry->type == H5F_MEM_PAGE_DRAW || H5F_MEM_PAGE_GHEAP == page_entry->type) + page_buf->evictions[1]++; + else + page_buf->evictions[0]++; + + /* Release page */ + page_entry->page_buf_ptr = H5FL_FAC_FREE(page_buf->page_fac, page_entry->page_buf_ptr); + page_entry = H5FL_FREE(H5PB_entry_t, page_entry); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PB__make_space() */ + + +/*------------------------------------------------------------------------- + * Function: H5PB__write_entry() + * + * Purpose: ??? + * + * This function was created without documentation. + * What follows is my best understanding of Mohamad's intent. + * + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PB__write_entry(const H5F_io_info2_t *fio_info, H5PB_entry_t *page_entry) +{ + haddr_t eoa; /* Current EOA for the file */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(fio_info); + HDassert(fio_info->f); + HDassert(page_entry); + + /* Retrieve the 'eoa' for the file */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(fio_info->f, page_entry->type))) + HGOTO_ERROR(H5E_PAGEBUF, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* If the starting address of the page is larger than + * the EOA, then the entire page is discarded without writing. + */ + if(page_entry->addr <= eoa) { + H5FD_io_info_t fdio_info; /* File driver I/O info */ + size_t page_size = fio_info->f->shared->page_buf->page_size; + + /* Adjust the page length if it exceeds the EOA */ + if((page_entry->addr + page_size) > eoa) + page_size = (size_t)(eoa - page_entry->addr); + + /* Translate to file driver I/O info object */ + fdio_info.file = fio_info->f->shared->lf; + fdio_info.meta_dxpl = fio_info->meta_dxpl; + fdio_info.raw_dxpl = fio_info->raw_dxpl; + + if(H5FD_write(&fdio_info, page_entry->type, page_entry->addr, page_size, page_entry->page_buf_ptr) < 0) + HGOTO_ERROR(H5E_PAGEBUF, H5E_WRITEERROR, FAIL, "file write failed") + } /* end if */ + + page_entry->is_dirty = FALSE; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PB__write_entry() */ + diff --git a/src/H5PBmodule.h b/src/H5PBmodule.h new file mode 100644 index 0000000..35da3f6 --- /dev/null +++ b/src/H5PBmodule.h @@ -0,0 +1,34 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Quincey Koziol <koziol@hdfgroup.org> + * Saturday, September 12, 2015 + * + * Purpose: This file contains declarations which define macros for the + * H5PB package. Including this header means that the source file + * is part of the H5PB package. + */ +#ifndef _H5PBmodule_H +#define _H5PBmodule_H + +/* Define the proper control macros for the generic FUNC_ENTER/LEAVE and error + * reporting macros. + */ +#define H5PB_MODULE +#define H5_MY_PKG H5PB +#define H5_MY_PKG_ERR H5E_RESOURCE +#define H5_MY_PKG_INIT NO + +#endif /* _H5PBmodule_H */ diff --git a/src/H5PBpkg.h b/src/H5PBpkg.h new file mode 100644 index 0000000..976f18d --- /dev/null +++ b/src/H5PBpkg.h @@ -0,0 +1,60 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#if !(defined H5PB_FRIEND || defined H5PB_MODULE) +#error "Do not include this file outside the H5PB package!" +#endif + +#ifndef _H5PBpkg_H +#define _H5PBpkg_H + +/* Get package's private header */ +#include "H5PBprivate.h" + +/* Other private headers needed by this file */ + +/**************************/ +/* Package Private Macros */ +/**************************/ + + +/****************************/ +/* Package Private Typedefs */ +/****************************/ + +typedef struct H5PB_entry_t { + void *page_buf_ptr; /* Pointer to the buffer containing the data */ + haddr_t addr; /* Address of the page in the file */ + H5F_mem_page_t type; /* Type of the page entry (H5F_MEM_PAGE_RAW/META) */ + hbool_t is_dirty; /* Flag indicating whether the page has dirty data or not */ + + /* Fields supporting replacement policies */ + struct H5PB_entry_t *next; /* next pointer in the LRU list */ + struct H5PB_entry_t *prev; /* previous pointer in the LRU list */ +} H5PB_entry_t; + + +/*****************************/ +/* Package Private Variables */ +/*****************************/ + + +/******************************/ +/* Package Private Prototypes */ +/******************************/ + + +#endif /* _H5PBpkg_H */ + diff --git a/src/H5PBprivate.h b/src/H5PBprivate.h new file mode 100644 index 0000000..3b5dcae --- /dev/null +++ b/src/H5PBprivate.h @@ -0,0 +1,108 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5PBprivate.h + * June 2014 + * Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ + +#ifndef _H5PBprivate_H +#define _H5PBprivate_H + +/* Include package's public header */ +#ifdef NOT_YET +#include "H5PBpublic.h" +#endif /* NOT_YET */ + +/* Private headers needed by this header */ +#include "H5private.h" /* Generic Functions */ +#include "H5Fprivate.h" /* File access */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5SLprivate.h" /* Skip List */ + + +/**************************/ +/* Library Private Macros */ +/**************************/ + + +/****************************/ +/* Library Private Typedefs */ +/****************************/ + +/* Forward declaration for a page buffer entry */ +struct H5PB_entry_t; + +/* Typedef for the main structure for the page buffer */ +typedef struct H5PB_t { + size_t max_size; /* The total page buffer size */ + size_t page_size; /* Size of a single page */ + unsigned min_meta_perc; /* Minimum ratio of metadata entries required before evicting meta entries */ + unsigned min_raw_perc; /* Minimum ratio of raw data entries required before evicting raw entries */ + unsigned meta_count; /* Number of entries for metadata */ + unsigned raw_count; /* Number of entries for raw data */ + unsigned min_meta_count; /* Minimum # of entries for metadata */ + unsigned min_raw_count; /* Minimum # of entries for raw data */ + + H5SL_t *slist_ptr; /* Skip list with all the active page entries */ + H5SL_t *mf_slist_ptr; /* Skip list containing newly allocated page entries inserted from the MF layer */ + + size_t LRU_list_len; /* Number of entries in the LRU (identical to slist_ptr count) */ + struct H5PB_entry_t *LRU_head_ptr; /* Head pointer of the LRU */ + struct H5PB_entry_t *LRU_tail_ptr; /* Tail pointer of the LRU */ + + H5FL_fac_head_t *page_fac; /* Factory for allocating pages */ + + /* Statistics */ + unsigned accesses[2]; + unsigned hits[2]; + unsigned misses[2]; + unsigned evictions[2]; + unsigned bypasses[2]; +} H5PB_t; + +/*****************************/ +/* Library-private Variables */ +/*****************************/ + + +/***************************************/ +/* Library-private Function Prototypes */ +/***************************************/ + +/* General routines */ +H5_DLL herr_t H5PB_create(H5F_t *file, size_t page_buffer_size, unsigned page_buf_min_meta_perc, unsigned page_buf_min_raw_perc); +H5_DLL herr_t H5PB_flush(const H5F_io_info2_t *fio_info); +H5_DLL herr_t H5PB_dest(H5F_t *file); +H5_DLL herr_t H5PB_add_new_page(H5F_t *f, H5FD_mem_t type, haddr_t page_addr); +H5_DLL herr_t H5PB_update_entry(H5PB_t *page_buf, haddr_t addr, size_t size, const void *buf); +H5_DLL herr_t H5PB_remove_entry(const H5F_t *f, haddr_t addr); +H5_DLL herr_t H5PB_read(const H5F_io_info2_t *fio_info, H5FD_mem_t type, + haddr_t addr, size_t size, void *buf/*out*/); +H5_DLL herr_t H5PB_write(const H5F_io_info2_t *f, H5FD_mem_t type, haddr_t addr, + size_t size, const void *buf); + +/* Statistics routines */ +H5_DLL herr_t H5PB_reset_stats(H5PB_t *page_buf); +H5_DLL herr_t H5PB_get_stats(const H5PB_t *page_buf, unsigned accesses[2], + unsigned hits[2], unsigned misses[2], unsigned evictions[2], unsigned bypasses[2]); +H5_DLL herr_t H5PB_print_stats(const H5PB_t *page_buf); + +#endif /* !_H5PBprivate_H */ + diff --git a/src/H5Pdeprec.c b/src/H5Pdeprec.c index ecf2bea..d225dfa 100644 --- a/src/H5Pdeprec.c +++ b/src/H5Pdeprec.c @@ -488,5 +488,135 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Pget_version() */ +/*------------------------------------------------------------------------- + * Function: H5Pset_file_space + * + * Purpose: It is mapped to H5Pset_file_space_strategy(). + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Jan 2017 + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold) +{ + + H5F_fspace_strategy_t new_strategy; /* File space strategy type */ + hbool_t new_persist = H5F_FREE_SPACE_PERSIST_DEF; /* Persisting free-space or not */ + hsize_t new_threshold = H5F_FREE_SPACE_THRESHOLD_DEF; /* Free-space section threshold */ + H5F_file_space_type_t in_strategy = strategy; /* Input strategy */ + hsize_t in_threshold = threshold; /* Input threshold */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "iFfh", plist_id, strategy, threshold); + + if((unsigned)in_strategy >= H5F_FILE_SPACE_NTYPES) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid strategy") + /* + * For 1.10.0 H5Pset_file_space: + * If strategy is zero, the property is not changed; + * the existing strategy is retained. + * If threshold is zero, the property is not changed; + * the existing threshold is retained. + */ + if(!in_strategy) + H5Pget_file_space(plist_id, &in_strategy, NULL); + if(!in_threshold) + H5Pget_file_space(plist_id, NULL, &in_threshold); + + switch(in_strategy) { + case H5F_FILE_SPACE_ALL_PERSIST: + new_strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; + new_persist = TRUE; + new_threshold = in_threshold; + break; + + case H5F_FILE_SPACE_ALL: + new_strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; + new_threshold = in_threshold; + break; + + case H5F_FILE_SPACE_AGGR_VFD: + new_strategy = H5F_FSPACE_STRATEGY_AGGR; + break; + + case H5F_FILE_SPACE_VFD: + new_strategy = H5F_FSPACE_STRATEGY_NONE; + break; + + case H5F_FILE_SPACE_NTYPES: + case H5F_FILE_SPACE_DEFAULT: + default: + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file space strategy") + } + + if(H5Pset_file_space_strategy(plist_id, new_strategy, new_persist, new_threshold) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set file space strategy") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pset_file_space() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_file_space + * + * Purpose: It is mapped to H5Pget_file_space_strategy(). + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Jan 2017 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold) +{ + H5F_fspace_strategy_t new_strategy; /* File space strategy type */ + hbool_t new_persist; /* Persisting free-space or not */ + hsize_t new_threshold; /* Free-space section threshold */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "i*Ff*h", plist_id, strategy, threshold); + + /* Get current file space info */ + if(H5Pget_file_space_strategy(plist_id, &new_strategy, &new_persist, &new_threshold) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file space strategy") + + /* Get value(s) */ + if(strategy) { + switch(new_strategy) { + + case H5F_FSPACE_STRATEGY_FSM_AGGR: + if(new_persist) + *strategy = H5F_FILE_SPACE_ALL_PERSIST; + else + *strategy = H5F_FILE_SPACE_ALL; + break; + + case H5F_FSPACE_STRATEGY_AGGR: + *strategy = H5F_FILE_SPACE_AGGR_VFD; + break; + + case H5F_FSPACE_STRATEGY_NONE: + *strategy = H5F_FILE_SPACE_VFD; + break; + + case H5F_FSPACE_STRATEGY_PAGE: + case H5F_FSPACE_STRATEGY_NTYPES: + default: + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file space strategy") + } + } + + if(threshold) + *threshold = new_threshold; + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pget_file_space() */ #endif /* H5_NO_DEPRECATED_SYMBOLS */ diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 7ea0fe0..7865fdf 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -80,18 +80,18 @@ #define H5F_ACS_PREEMPT_READ_CHUNKS_DEC H5P__decode_double /* Definition for threshold for alignment */ #define H5F_ACS_ALIGN_THRHD_SIZE sizeof(hsize_t) -#define H5F_ACS_ALIGN_THRHD_DEF 1 +#define H5F_ACS_ALIGN_THRHD_DEF H5F_ALIGN_THRHD_DEF #define H5F_ACS_ALIGN_THRHD_ENC H5P__encode_hsize_t #define H5F_ACS_ALIGN_THRHD_DEC H5P__decode_hsize_t /* Definition for alignment */ #define H5F_ACS_ALIGN_SIZE sizeof(hsize_t) -#define H5F_ACS_ALIGN_DEF 1 +#define H5F_ACS_ALIGN_DEF H5F_ALIGN_DEF #define H5F_ACS_ALIGN_ENC H5P__encode_hsize_t #define H5F_ACS_ALIGN_DEC H5P__decode_hsize_t /* Definition for minimum metadata allocation block size (when aggregating metadata allocations. */ #define H5F_ACS_META_BLOCK_SIZE_SIZE sizeof(hsize_t) -#define H5F_ACS_META_BLOCK_SIZE_DEF 2048 +#define H5F_ACS_META_BLOCK_SIZE_DEF H5F_META_BLOCK_SIZE_DEF #define H5F_ACS_META_BLOCK_SIZE_ENC H5P__encode_hsize_t #define H5F_ACS_META_BLOCK_SIZE_DEC H5P__decode_hsize_t /* Definition for maximum sieve buffer size (when data sieving @@ -103,7 +103,7 @@ /* Definition for minimum "small data" allocation block size (when aggregating "small" raw data allocations. */ #define H5F_ACS_SDATA_BLOCK_SIZE_SIZE sizeof(hsize_t) -#define H5F_ACS_SDATA_BLOCK_SIZE_DEF 2048 +#define H5F_ACS_SDATA_BLOCK_SIZE_DEF H5F_SDATA_BLOCK_SIZE_DEF #define H5F_ACS_SDATA_BLOCK_SIZE_ENC H5P__encode_hsize_t #define H5F_ACS_SDATA_BLOCK_SIZE_DEC H5P__decode_hsize_t /* Definition for garbage-collect references */ @@ -231,6 +231,21 @@ #define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_ENC H5P__facc_cache_image_config_enc #define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_DEC H5P__facc_cache_image_config_dec #define H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_CMP H5P__facc_cache_image_config_cmp +/* Definition for total size of page buffer(bytes) */ +#define H5F_ACS_PAGE_BUFFER_SIZE_SIZE sizeof(size_t) +#define H5F_ACS_PAGE_BUFFER_SIZE_DEF 0 +#define H5F_ACS_PAGE_BUFFER_SIZE_ENC H5P__encode_size_t +#define H5F_ACS_PAGE_BUFFER_SIZE_DEC H5P__decode_size_t +/* Definition for minimum metadata size of page buffer(bytes) */ +#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_SIZE sizeof(unsigned) +#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEF 0 +#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_ENC H5P__encode_unsigned +#define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEC H5P__decode_unsigned +/* Definition for minimum raw data size of page buffer(bytes) */ +#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_SIZE sizeof(unsigned) +#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF 0 +#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_ENC H5P__encode_unsigned +#define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEC H5P__decode_unsigned /******************/ @@ -359,6 +374,9 @@ static const H5P_coll_md_read_flag_t H5F_def_coll_md_read_flag_g = H5F_ACS_COLL_ static const hbool_t H5F_def_coll_md_write_flag_g = H5F_ACS_COLL_MD_WRITE_FLAG_DEF; /* Default setting for the collective metedata write flag */ #endif /* H5_HAVE_PARALLEL */ static const H5AC_cache_image_config_t H5F_def_mdc_initCacheImageCfg_g = H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_DEF; /* Default metadata cache image settings */ +static const size_t H5F_def_page_buf_size_g = H5F_ACS_PAGE_BUFFER_SIZE_DEF; /* Default page buffer size */ +static const unsigned H5F_def_page_buf_min_meta_perc_g = H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEF; /* Default page buffer minimum metadata size */ +static const unsigned H5F_def_page_buf_min_raw_perc_g = H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF; /* Default page buffer minumum raw data size */ /*------------------------------------------------------------------------- @@ -574,6 +592,22 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, H5F_ACS_META_CACHE_INIT_IMAGE_CONFIG_CMP, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the size of the page buffer size */ + if(H5P_register_real(pclass, H5F_ACS_PAGE_BUFFER_SIZE_NAME, H5F_ACS_PAGE_BUFFER_SIZE_SIZE, &H5F_def_page_buf_size_g, + NULL, NULL, NULL, H5F_ACS_PAGE_BUFFER_SIZE_ENC, H5F_ACS_PAGE_BUFFER_SIZE_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the size of the page buffer minimum metadata size */ + if(H5P_register_real(pclass, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_SIZE, &H5F_def_page_buf_min_meta_perc_g, + NULL, NULL, NULL, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_ENC, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the size of the page buffer minimum raw data size */ + if(H5P_register_real(pclass, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_SIZE, &H5F_def_page_buf_min_raw_perc_g, + NULL, NULL, NULL, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_ENC, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__facc_reg_prop() */ @@ -4681,3 +4715,93 @@ done: } /* end H5Pget_coll_metadata_write() */ #endif /* H5_HAVE_PARALLEL */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_page_buffer_size + * + * Purpose: Set the maximum page buffering size. This has to be a + * multiple of the page allocation size which must be enabled; + * otherwise file create/open will fail. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * June 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_page_buffer_size(hid_t plist_id, size_t buf_size, unsigned min_meta_perc, unsigned min_raw_perc) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE4("e", "izIuIu", plist_id, buf_size, min_meta_perc, min_raw_perc); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + if(min_meta_perc > 100) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Minimum metadata fractions must be between 0 and 100 inclusive") + if(min_raw_perc > 100) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Minimum rawdata fractions must be between 0 and 100 inclusive") + + if(min_meta_perc + min_raw_perc > 100) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Sum of minimum metadata and raw data fractions can't be bigger than 100"); + + /* Set size */ + if(H5P_set(plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &buf_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET,FAIL, "can't set page buffer size") + if(H5P_set(plist, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, &min_meta_perc) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET,FAIL, "can't set percentage of min metadata entries") + if(H5P_set(plist, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, &min_raw_perc) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET,FAIL, "can't set percentage of min rawdata entries") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_page_buffer_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_page_buffer_size + * + * Purpose: Retrieves the maximum page buffer size. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Mohamad Chaarawi + * June 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_page_buffer_size(hid_t plist_id, size_t *buf_size, unsigned *min_meta_perc, unsigned *min_raw_perc) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE4("e", "i*z*Iu*Iu", plist_id, buf_size, min_meta_perc, min_raw_perc); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get size */ + + if(buf_size) + if(H5P_get(plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, buf_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET,FAIL, "can't get page buffer size") + if(min_meta_perc) + if(H5P_get(plist, H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME, min_meta_perc) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET,FAIL, "can't get page buffer minimum metadata percent") + if(min_raw_perc) + if(H5P_get(plist, H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME, min_raw_perc) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET,FAIL, "can't get page buffer minimum raw data percent") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_page_buffer_size() */ + diff --git a/src/H5Pfcpl.c b/src/H5Pfcpl.c index d451982..c778801 100644 --- a/src/H5Pfcpl.c +++ b/src/H5Pfcpl.c @@ -97,14 +97,23 @@ #define H5F_CRT_SHMSG_BTREE_MIN_ENC H5P__encode_unsigned #define H5F_CRT_SHMSG_BTREE_MIN_DEC H5P__decode_unsigned /* Definitions for file space handling strategy */ -#define H5F_CRT_FILE_SPACE_STRATEGY_SIZE sizeof(unsigned) -#define H5F_CRT_FILE_SPACE_STRATEGY_DEF H5F_FILE_SPACE_STRATEGY_DEF -#define H5F_CRT_FILE_SPACE_STRATEGY_ENC H5P__encode_unsigned -#define H5F_CRT_FILE_SPACE_STRATEGY_DEC H5P__decode_unsigned -#define H5F_CRT_FREE_SPACE_THRESHOLD_SIZE sizeof(hsize_t) -#define H5F_CRT_FREE_SPACE_THRESHOLD_DEF H5F_FREE_SPACE_THRESHOLD_DEF -#define H5F_CRT_FREE_SPACE_THRESHOLD_ENC H5P__encode_hsize_t -#define H5F_CRT_FREE_SPACE_THRESHOLD_DEC H5P__decode_hsize_t +#define H5F_CRT_FILE_SPACE_STRATEGY_SIZE sizeof(H5F_fspace_strategy_t) +#define H5F_CRT_FILE_SPACE_STRATEGY_DEF H5F_FILE_SPACE_STRATEGY_DEF +#define H5F_CRT_FILE_SPACE_STRATEGY_ENC H5P__fcrt_fspace_strategy_enc +#define H5F_CRT_FILE_SPACE_STRATEGY_DEC H5P__fcrt_fspace_strategy_dec +#define H5F_CRT_FREE_SPACE_PERSIST_SIZE sizeof(hbool_t) +#define H5F_CRT_FREE_SPACE_PERSIST_DEF H5F_FREE_SPACE_PERSIST_DEF +#define H5F_CRT_FREE_SPACE_PERSIST_ENC H5P__encode_hbool_t +#define H5F_CRT_FREE_SPACE_PERSIST_DEC H5P__decode_hbool_t +#define H5F_CRT_FREE_SPACE_THRESHOLD_SIZE sizeof(hsize_t) +#define H5F_CRT_FREE_SPACE_THRESHOLD_DEF H5F_FREE_SPACE_THRESHOLD_DEF +#define H5F_CRT_FREE_SPACE_THRESHOLD_ENC H5P__encode_hsize_t +#define H5F_CRT_FREE_SPACE_THRESHOLD_DEC H5P__decode_hsize_t +/* Definitions for file space page size in support of level-2 page caching */ +#define H5F_CRT_FILE_SPACE_PAGE_SIZE_SIZE sizeof(hsize_t) +#define H5F_CRT_FILE_SPACE_PAGE_SIZE_DEF H5F_FILE_SPACE_PAGE_SIZE_DEF +#define H5F_CRT_FILE_SPACE_PAGE_SIZE_ENC H5P__encode_hsize_t +#define H5F_CRT_FILE_SPACE_PAGE_SIZE_DEC H5P__decode_hsize_t /******************/ @@ -131,6 +140,8 @@ static herr_t H5P__fcrt_shmsg_index_types_enc(const void *value, void **_pp, siz static herr_t H5P__fcrt_shmsg_index_types_dec(const void **_pp, void *value); static herr_t H5P__fcrt_shmsg_index_minsize_enc(const void *value, void **_pp, size_t *size); static herr_t H5P__fcrt_shmsg_index_minsize_dec(const void **_pp, void *value); +static herr_t H5P__fcrt_fspace_strategy_enc(const void *value, void **_pp, size_t *size); +static herr_t H5P__fcrt_fspace_strategy_dec(const void **_pp, void *_value); /*********************/ @@ -178,8 +189,10 @@ static const unsigned H5F_def_sohm_index_flags_g[H5O_SHMESG_MAX_NINDEXES] = H static const unsigned H5F_def_sohm_index_minsizes_g[H5O_SHMESG_MAX_NINDEXES] = H5F_CRT_SHMSG_INDEX_MINSIZE_DEF; static const unsigned H5F_def_sohm_list_max_g = H5F_CRT_SHMSG_LIST_MAX_DEF; static const unsigned H5F_def_sohm_btree_min_g = H5F_CRT_SHMSG_BTREE_MIN_DEF; -static const unsigned H5F_def_file_space_strategy_g = H5F_CRT_FILE_SPACE_STRATEGY_DEF; +static const H5F_fspace_strategy_t H5F_def_file_space_strategy_g = H5F_CRT_FILE_SPACE_STRATEGY_DEF; +static const hbool_t H5F_def_free_space_persist_g = H5F_CRT_FREE_SPACE_PERSIST_DEF; static const hsize_t H5F_def_free_space_threshold_g = H5F_CRT_FREE_SPACE_THRESHOLD_DEF; +static const hsize_t H5F_def_file_space_page_size_g = H5F_CRT_FILE_SPACE_PAGE_SIZE_DEF; @@ -267,12 +280,24 @@ H5P_fcrt_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the free-space persist flag */ + if(H5P_register_real(pclass, H5F_CRT_FREE_SPACE_PERSIST_NAME, H5F_CRT_FREE_SPACE_PERSIST_SIZE, &H5F_def_free_space_persist_g, + NULL, NULL, NULL, H5F_CRT_FREE_SPACE_PERSIST_ENC, H5F_CRT_FREE_SPACE_PERSIST_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the free space section threshold */ if(H5P_register_real(pclass, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, H5F_CRT_FREE_SPACE_THRESHOLD_SIZE, &H5F_def_free_space_threshold_g, NULL, NULL, NULL, H5F_CRT_FREE_SPACE_THRESHOLD_ENC, H5F_CRT_FREE_SPACE_THRESHOLD_DEC, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the file space page size */ + if(H5P_register_real(pclass, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, H5F_CRT_FILE_SPACE_PAGE_SIZE_SIZE, &H5F_def_file_space_page_size_g, + NULL, NULL, NULL, H5F_CRT_FILE_SPACE_PAGE_SIZE_ENC, H5F_CRT_FILE_SPACE_PAGE_SIZE_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P_fcrt_reg_prop() */ @@ -1246,15 +1271,13 @@ H5Pget_shared_mesg_phase_change(hid_t plist_id, unsigned *max_list, unsigned *mi if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_CREATE))) HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID"); - /* Get value */ - if (max_list) { + /* Get value(s) */ + if(max_list) if(H5P_get(plist, H5F_CRT_SHMSG_LIST_MAX_NAME, max_list) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get list maximum"); - } - if (min_btree) { + if(min_btree) if(H5P_get(plist, H5F_CRT_SHMSG_BTREE_MIN_NAME, min_btree) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get SOHM information"); - } done: FUNC_LEAVE_API(ret_value) @@ -1262,15 +1285,12 @@ done: /*------------------------------------------------------------------------- - * Function: H5Pset_file_space + * Function: H5Pset_file_space_strategy * - * Purpose: Sets the strategy that the library employs in managing file space. - * If strategy is zero, the property is not changed; the existing - * strategy is retained. - * Sets the threshold value that the file's free space - * manager(s) will use to track free space sections. - * If threshold is zero, the property is not changed; the existing - * threshold is retained. + * Purpose: Sets the "strategy" that the library employs in managing file space + * Sets the "persist" value as to persist free-space or not + * Sets the "threshold" value that the free space manager(s) will use to track free space sections. + * Ignore "persist" and "threshold" for strategies that do not use free-space managers * * Return: Non-negative on success/Negative on failure * @@ -1279,15 +1299,16 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold) +H5Pset_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t strategy, hbool_t persist, hsize_t threshold) { H5P_genplist_t *plist; /* Property list pointer */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE3("e", "iFfh", plist_id, strategy, threshold); + H5TRACE4("e", "iFfbh", plist_id, strategy, persist, threshold); - if((unsigned)strategy >= H5F_FILE_SPACE_NTYPES) + /* Check arguments */ + if(strategy >= H5F_FSPACE_STRATEGY_NTYPES) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid strategy") /* Get the plist structure */ @@ -1295,24 +1316,28 @@ H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t thresh HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") /* Set value(s), if non-zero */ - if(strategy) - if(H5P_set(plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &strategy) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set file space strategy") - if(threshold) - if(H5P_set(plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &threshold) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set free-space threshold") + if(H5P_set(plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, &strategy) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set file space strategy") + + /* Ignore persist and threshold settings for strategies that do not use FSM */ + if(strategy == H5F_FSPACE_STRATEGY_FSM_AGGR || strategy == H5F_FSPACE_STRATEGY_PAGE) { + if(H5P_set(plist, H5F_CRT_FREE_SPACE_PERSIST_NAME, &persist) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set free-space persisting status") + + if(H5P_set(plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, &threshold) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set free-space threshold") + } /* end if */ done: FUNC_LEAVE_API(ret_value) -} /* H5Pset_file_space() */ +} /* H5Pset_file_space_strategy() */ /*------------------------------------------------------------------------- - * Function: H5Pget_file_space + * Function: H5Pget_file_space_strategy * - * Purpose: Retrieves the strategy that the library uses in managing file space. - * Retrieves the threshold value that the file's free space - * managers use to track free space sections. + * Purpose: Retrieves the strategy, persist, and threshold that the library + * uses in managing file space. * * Return: Non-negative on success/Negative on failure * @@ -1321,13 +1346,13 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold) +H5Pget_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t *strategy, hbool_t *persist, hsize_t *threshold) { H5P_genplist_t *plist; /* Property list pointer */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE3("e", "i*Ff*h", plist_id, strategy, threshold); + H5TRACE4("e", "i*Ff*b*h", plist_id, strategy, persist, threshold); /* Get the plist structure */ if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_CREATE))) @@ -1337,11 +1362,158 @@ H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *thre if(strategy) if(H5P_get(plist, H5F_CRT_FILE_SPACE_STRATEGY_NAME, strategy) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file space strategy") + if(persist) + if(H5P_get(plist, H5F_CRT_FREE_SPACE_PERSIST_NAME, persist) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get free-space persisting status") if(threshold) if(H5P_get(plist, H5F_CRT_FREE_SPACE_THRESHOLD_NAME, threshold) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get free-space threshold") done: FUNC_LEAVE_API(ret_value) -} /* H5Pget_file_space() */ +} /* H5Pget_file_space_strategy() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__fcrt_fspace_strategy_enc + * + * Purpose: Callback routine which is called whenever the free-space + * strategy property in the file creation property list + * is encoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Quincey Koziol + * Friday, December 27, 2013 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__fcrt_fspace_strategy_enc(const void *value, void **_pp, size_t *size) +{ + const H5F_fspace_strategy_t *strategy = (const H5F_fspace_strategy_t *)value; /* Create local alias for values */ + uint8_t **pp = (uint8_t **)_pp; + + FUNC_ENTER_STATIC_NOERR + + /* Sanity check */ + HDassert(strategy); + HDassert(size); + + if(NULL != *pp) + /* Encode free-space strategy */ + *(*pp)++ = (uint8_t)*strategy; + + /* Size of free-space strategy */ + (*size)++; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__fcrt_fspace_strategy_enc() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__fcrt_fspace_strategy_dec + * + * Purpose: Callback routine which is called whenever the free-space + * strategy property in the file creation property list + * is decoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Quincey Koziol + * Friday, December 27, 2013 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__fcrt_fspace_strategy_dec(const void **_pp, void *_value) +{ + H5F_fspace_strategy_t *strategy = (H5F_fspace_strategy_t *)_value; /* Free-space strategy */ + const uint8_t **pp = (const uint8_t **)_pp; + + FUNC_ENTER_STATIC_NOERR + + /* Sanity checks */ + HDassert(pp); + HDassert(*pp); + HDassert(strategy); + + /* Decode free-space strategy */ + *strategy = (H5F_fspace_strategy_t)*(*pp)++; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__fcrt_fspace_strategy_dec() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_file_space_page_size + * + * Purpose: Sets the file space page size for paged aggregation. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; August 2012 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_file_space_page_size(hid_t plist_id, hsize_t fsp_size) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ih", plist_id, fsp_size); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_CREATE))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + if(fsp_size < H5F_FILE_SPACE_PAGE_SIZE_MIN) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "cannot set file space page size to less than 512") + + /* Set the value*/ + if(H5P_set(plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, &fsp_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set file space page size") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pset_file_space_page_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_file_space_page_size + * + * Purpose: Retrieves the file space page size for aggregating small metadata + * or raw data in the parameter "fsp_size". + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; August 2012 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_file_space_page_size(hid_t plist_id, hsize_t *fsp_size) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*h", plist_id, fsp_size); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_CREATE))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get value */ + if(fsp_size) + if(H5P_get(plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, fsp_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file space page size") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pget_file_space_page_size() */ diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 5aa8301..50b9eb0 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -300,8 +300,10 @@ H5_DLL herr_t H5Pset_shared_mesg_index(hid_t plist_id, unsigned index_num, unsig H5_DLL herr_t H5Pget_shared_mesg_index(hid_t plist_id, unsigned index_num, unsigned *mesg_type_flags, unsigned *min_mesg_size); H5_DLL herr_t H5Pset_shared_mesg_phase_change(hid_t plist_id, unsigned max_list, unsigned min_btree); H5_DLL herr_t H5Pget_shared_mesg_phase_change(hid_t plist_id, unsigned *max_list, unsigned *min_btree); -H5_DLL herr_t H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold); -H5_DLL herr_t H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold); +H5_DLL herr_t H5Pset_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t strategy, hbool_t persist, hsize_t threshold); +H5_DLL herr_t H5Pget_file_space_strategy(hid_t plist_id, H5F_fspace_strategy_t *strategy, hbool_t *persist, hsize_t *threshold); +H5_DLL herr_t H5Pset_file_space_page_size(hid_t plist_id, hsize_t fsp_size); +H5_DLL herr_t H5Pget_file_space_page_size(hid_t plist_id, hsize_t *fsp_size); /* File access property list (FAPL) routines */ H5_DLL herr_t H5Pset_alignment(hid_t fapl_id, hsize_t threshold, @@ -367,6 +369,8 @@ H5_DLL herr_t H5Pget_coll_metadata_write(hid_t plist_id, hbool_t *is_collective) #endif /* H5_HAVE_PARALLEL */ H5_DLL herr_t H5Pset_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t *config_ptr); H5_DLL herr_t H5Pget_mdc_image_config(hid_t plist_id, H5AC_cache_image_config_t *config_ptr /*out*/); +H5_DLL herr_t H5Pset_page_buffer_size(hid_t plist_id, size_t buf_size, unsigned min_meta_per, unsigned min_raw_per); +H5_DLL herr_t H5Pget_page_buffer_size(hid_t plist_id, size_t *buf_size, unsigned *min_meta_per, unsigned *min_raw_per); /* Dataset creation property list (DCPL) routines */ H5_DLL herr_t H5Pset_layout(hid_t plist_id, H5D_layout_t layout); @@ -534,6 +538,8 @@ H5_DLL herr_t H5Pget_filter_by_id1(hid_t plist_id, H5Z_filter_t id, H5_DLL herr_t H5Pget_version(hid_t plist_id, unsigned *boot/*out*/, unsigned *freelist/*out*/, unsigned *stab/*out*/, unsigned *shhdr/*out*/); +H5_DLL herr_t H5Pset_file_space(hid_t plist_id, H5F_file_space_type_t strategy, hsize_t threshold); +H5_DLL herr_t H5Pget_file_space(hid_t plist_id, H5F_file_space_type_t *strategy, hsize_t *threshold); #endif /* H5_NO_DEPRECATED_SYMBOLS */ #ifdef __cplusplus @@ -626,7 +626,7 @@ H5Z__flush_file_cb(void *obj_ptr, hid_t H5_ATTR_UNUSED obj_id, void H5_ATTR_UNUS /* Call the flush routine for mounted file hierarchies. Do a global flush * if the file is opened for write */ if(H5F_ACC_RDWR & H5F_INTENT((H5F_t *)obj_ptr)) { - if(H5F_flush_mounts((H5F_t *)obj_ptr, H5AC_ind_read_dxpl_id) < 0) + if(H5F_flush_mounts((H5F_t *)obj_ptr, H5AC_ind_read_dxpl_id, H5AC_rawdata_dxpl_id) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFLUSH, FAIL, "unable to flush file hierarchy") } /* end if */ diff --git a/src/H5err.txt b/src/H5err.txt index 48069a5..b2ff3bd 100644 --- a/src/H5err.txt +++ b/src/H5err.txt @@ -77,6 +77,7 @@ MAJOR, H5E_SOHM, Shared Object Header Messages MAJOR, H5E_EARRAY, Extensible Array MAJOR, H5E_FARRAY, Fixed Array MAJOR, H5E_PLUGIN, Plugin for dynamically loaded library +MAJOR, H5E_PAGEBUF, Page Buffering MAJOR, H5E_NONE_MAJOR, No error # Sections (for grouping minor errors) diff --git a/src/H5private.h b/src/H5private.h index 00de96c..61ecc5b 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -1386,7 +1386,13 @@ typedef off_t h5_stat_size_t; #ifndef HDstrtol #define HDstrtol(S,R,N) strtol(S,R,N) #endif /* HDstrtol */ -H5_DLL int64_t HDstrtoll (const char *s, const char **rest, int base); +#ifndef HDstrtoll + #ifdef H5_HAVE_STRTOLL + #define HDstrtoll(S,R,N) strtoll(S,R,N) + #else + H5_DLL int64_t HDstrtoll (const char *s, const char **rest, int base); + #endif /* H5_HAVE_STRTOLL */ +#endif /* HDstrtoll */ #ifndef HDstrtoul #define HDstrtoul(S,R,N) strtoul(S,R,N) #endif /* HDstrtoul */ diff --git a/src/H5system.c b/src/H5system.c index ac323c0..1f92e19 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -32,10 +32,10 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5Fprivate.h" /* File access */ -#include "H5MMprivate.h" /* Memory management */ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File access */ +#include "H5MMprivate.h" /* Memory management */ /****************/ @@ -474,6 +474,7 @@ HDfprintf(FILE *stream, const char *fmt, ...) * *------------------------------------------------------------------------- */ +#ifndef HDstrtoll int64_t HDstrtoll(const char *s, const char **rest, int base) { @@ -549,7 +550,7 @@ HDstrtoll(const char *s, const char **rest, int base) *rest = s; return acc; } /* end HDstrtoll() */ - +#endif /*------------------------------------------------------------------------- * Function: HDrand/HDsrand @@ -604,7 +605,7 @@ void HDsrand(unsigned int seed) #ifdef H5_HAVE_FCNTL int Pflock(int fd, int operation) { - + struct flock flk; /* Set the lock type */ @@ -649,18 +650,18 @@ Nflock(int H5_ATTR_UNUSED fd, int H5_ATTR_UNUSED operation) { /*------------------------------------------------------------------------- - * Function: H5_make_time + * Function: H5_make_time * - * Purpose: Portability routine to abstract converting a 'tm' struct into - * a time_t value. + * Purpose: Portability routine to abstract converting a 'tm' struct into + * a time_t value. * - * Note: This is a little problematic because mktime() operates on - * local times. We convert to local time and then figure out the - * adjustment based on the local time zone and daylight savings - * setting. + * Note: This is a little problematic because mktime() operates on + * local times. We convert to local time and then figure out the + * adjustment based on the local time zone and daylight savings + * setting. * - * Return: Success: The value of timezone - * Failure: -1 + * Return: Success: The value of timezone + * Failure: -1 * * Programmer: Quincey Koziol * November 18, 2015 @@ -1138,7 +1139,7 @@ H5_combine_path(const char* path1, const char* path2, char **full_name /*out*/) if(NULL == (*full_name = (char *)H5MM_strdup(path2))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") - } /* end if */ + } /* end if */ else if(H5_CHECK_ABS_PATH(path2)) { /* On windows path2 is a path absolute name */ diff --git a/src/H5trace.c b/src/H5trace.c index 44b2ed5..48c357a 100644 --- a/src/H5trace.c +++ b/src/H5trace.c @@ -895,32 +895,28 @@ H5_trace(const double *returning, const char *func, const char *type, ...) fprintf(out, "NULL"); } /* end if */ else { - H5F_file_space_type_t fs_type = (H5F_file_space_type_t)va_arg(ap, int); + H5F_fspace_strategy_t fs_strategy = (H5F_fspace_strategy_t)va_arg(ap, int); - switch(fs_type) { - case H5F_FILE_SPACE_DEFAULT: - fprintf(out, "H5F_FILE_SPACE_DEFAULT"); + switch(fs_strategy) { + case H5F_FSPACE_STRATEGY_FSM_AGGR: + fprintf(out, "H5F_FSPACE_STRATEGY_FSM_AGGR"); break; - case H5F_FILE_SPACE_ALL_PERSIST: - fprintf(out, "H5F_FILE_SPACE_ALL_PERSIST"); + case H5F_FSPACE_STRATEGY_PAGE: + fprintf(out, "H5F_FSPACE_STRATEGY_PAGE"); break; - case H5F_FILE_SPACE_ALL: - fprintf(out, "H5F_FILE_SPACE_ALL"); + case H5F_FSPACE_STRATEGY_AGGR: + fprintf(out, "H5F_FSPACE_STRATEGY_AGGR"); break; - case H5F_FILE_SPACE_AGGR_VFD: - fprintf(out, "H5F_FILE_SPACE_AGGR_VFD"); + case H5F_FSPACE_STRATEGY_NONE: + fprintf(out, "H5F_FSPACE_STRATEGY_NONE"); break; - case H5F_FILE_SPACE_VFD: - fprintf(out, "H5F_FILE_SPACE_VFD"); - break; - - case H5F_FILE_SPACE_NTYPES: + case H5F_FSPACE_STRATEGY_NTYPES: default: - fprintf(out, "%ld", (long)fs_type); + fprintf(out, "%ld", (long)fs_strategy); break; } /* end switch */ } /* end else */ @@ -1003,6 +999,15 @@ H5_trace(const double *returning, const char *func, const char *type, ...) } /* end else */ break; + case 't': + if(ptr) { + if(vp) + fprintf(out, "0x%lx", (unsigned long)vp); + else + fprintf(out, "NULL"); + } /* end if */ + break; + case 'v': if(ptr) { if(vp) diff --git a/src/H5win32defs.h b/src/H5win32defs.h index b419f06..e005b51 100644 --- a/src/H5win32defs.h +++ b/src/H5win32defs.h @@ -23,6 +23,11 @@ * */ +/* + * _MSC_VER = 1900 VS2015 + * _MSC_VER = 1800 VS2013 + * _MSC_VER = 1700 VS2012 + */ #ifdef H5_HAVE_WIN32_API typedef struct _stati64 h5_stat_t; @@ -54,13 +59,22 @@ typedef __int64 h5_stat_size_t; #define HDsleep(S) Sleep(S*1000) #define HDstat(S,B) _stati64(S,B) #define HDstrcasecmp(A,B) _stricmp(A,B) -#define HDstrtoull(S,R,N) _strtoui64(S,R,N) #define HDstrdup(S) _strdup(S) #define HDtzset() _tzset() #define HDunlink(S) _unlink(S) #define HDwrite(F,M,Z) _write(F,M,Z) #ifdef H5_HAVE_VISUAL_STUDIO + +#if (_MSC_VER < 1800) + #ifndef H5_HAVE_STRTOLL + #define HDstrtoll(S,R,N) _strtoi64(S,R,N) + #endif /* H5_HAVE_STRTOLL */ + #ifndef H5_HAVE_STRTOULL + #define HDstrtoull(S,R,N) _strtoui64(S,R,N) + #endif /* H5_HAVE_STRTOULL */ +#endif /* MSC_VER < 1800 */ + /* * The (void*) cast just avoids a compiler warning in H5_HAVE_VISUAL_STUDIO */ diff --git a/src/Makefile.am b/src/Makefile.am index 69b54b4..59df4c2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -60,7 +60,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5F.c H5Fint.c H5Faccum.c H5Fcwfs.c \ H5Fdbg.c H5Fdeprec.c H5Fefc.c H5Ffake.c H5Fio.c \ H5Fmount.c H5Fquery.c \ - H5Fsfile.c H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \ + H5Fsfile.c H5Fspace.c H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \ H5FA.c H5FAcache.c H5FAdbg.c H5FAdblock.c H5FAdblkpage.c H5FAhdr.c \ H5FAint.c H5FAstat.c H5FAtest.c \ H5FD.c H5FDcore.c \ @@ -98,6 +98,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5Pfapl.c H5Pfcpl.c H5Pfmpl.c \ H5Pgcpl.c H5Pint.c \ H5Plapl.c H5Plcpl.c H5Pocpl.c H5Pocpypl.c H5Pstrcpl.c H5Ptest.c \ + H5PB.c \ H5PL.c \ H5R.c H5Rdeprec.c \ H5UC.c \ |