diff options
| author | Frank Willmore <Frank.Willmore@hdfgroup.org> | 2016-12-19 18:09:38 (GMT) |
|---|---|---|
| committer | Frank Willmore <Frank.Willmore@hdfgroup.org> | 2016-12-19 18:09:38 (GMT) |
| commit | d0a7400a344394f2a17157d2c8250adfbd57e93a (patch) | |
| tree | 29d1e3be8400846f432656b09d360556d789ed8e /src | |
| parent | a2e93075f2ec86dad367a8aafadc20f6386f9b0e (diff) | |
| parent | ab3963b28e59419f8e857ec224dd4efa3ea6dd8e (diff) | |
| download | hdf5-d0a7400a344394f2a17157d2c8250adfbd57e93a.zip hdf5-d0a7400a344394f2a17157d2c8250adfbd57e93a.tar.gz hdf5-d0a7400a344394f2a17157d2c8250adfbd57e93a.tar.bz2 | |
Merging in latest from upstream (HDFFV/hdf5:refs/heads/develop)
* commit 'ab3963b28e59419f8e857ec224dd4efa3ea6dd8e': (214 commits)
Merge SWMR-related testing to existing tests.
Bring over tweak for missing environment variable.
Update CMake configuration files with SWMR accumulator changes.
Add missing accumulator test.
Merge SWMR-oriented accumulator tests from revise_chunks to develop.
Bring Java SWMR changes from revise_chunks to develop branch
Snapshot version 1.9 release 235
Change dlopen from RTLD_NOW to RTLD_LAZY
Fix bad implementation of Windows nanosleep equivalent.
Removed NDEBUG guards from H5AC test functions.
Bring SWMR-related tools changes from revise_chunks to develop.
Bring over changes from revise_chunks that cleanup recent SWMR changes from code review feedback.
Updated the H5FS cache code to grab the correct tag and modified the freespace test to use dxpls that have been tagged with the H5AC__FREESPACE_TAG global tag instead of H5AC_ind_read_dxpl_id. The library code now expects the owner of the free space manager to tag it so the owner-less free space managers in the freespace tag had to be tagged with *something* to avoid cache errors.
Updated the comment for the valgrind fix.
Fixed a valgrind problem in file shutdown exposed by the swmr.c test.
Reduce timeout as normal run time on windows is less then 10 min
Updated the icc flags (C flags only).
Another Java oversight (sorry, don't have Java configured on my Mac)
Correct oversight in Java test and remove direct VFD from SWMR supported drivers.
Bring SWMR support in to the main development branch. (Finally!) More tests and the tool and API wrappers will be coming in over the weekend.
...
Diffstat (limited to 'src')
143 files changed, 17042 insertions, 8517 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c0f934e..55de5ea 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -44,7 +44,10 @@ IDE_GENERATED_PROPERTIES ("H5A" "${H5A_HDRS}" "${H5A_SOURCES}" ) set (H5AC_SOURCES ${HDF5_SRC_DIR}/H5AC.c + ${HDF5_SRC_DIR}/H5ACdbg.c + ${HDF5_SRC_DIR}/H5AClog.c ${HDF5_SRC_DIR}/H5ACmpio.c + ${HDF5_SRC_DIR}/H5ACproxy_entry.c ) set (H5AC_HDRS @@ -71,6 +74,8 @@ set (H5B2_SOURCES ${HDF5_SRC_DIR}/H5B2dbg.c ${HDF5_SRC_DIR}/H5B2hdr.c ${HDF5_SRC_DIR}/H5B2int.c + ${HDF5_SRC_DIR}/H5B2internal.c + ${HDF5_SRC_DIR}/H5B2leaf.c ${HDF5_SRC_DIR}/H5B2stat.c ${HDF5_SRC_DIR}/H5B2test.c ) @@ -83,7 +88,9 @@ IDE_GENERATED_PROPERTIES ("H5B2" "${H5B2_HDRS}" "${H5B2_SOURCES}" ) set (H5C_SOURCES ${HDF5_SRC_DIR}/H5C.c + ${HDF5_SRC_DIR}/H5Cdbg.c ${HDF5_SRC_DIR}/H5Cepoch.c + ${HDF5_SRC_DIR}/H5Clog.c ${HDF5_SRC_DIR}/H5Cmpio.c ${HDF5_SRC_DIR}/H5Cquery.c ${HDF5_SRC_DIR}/H5Ctag.c @@ -205,6 +212,7 @@ set (H5FA_SOURCES ${HDF5_SRC_DIR}/H5FAdblkpage.c ${HDF5_SRC_DIR}/H5FAdblock.c ${HDF5_SRC_DIR}/H5FAhdr.c + ${HDF5_SRC_DIR}/H5FAint.c ${HDF5_SRC_DIR}/H5FAstat.c ${HDF5_SRC_DIR}/H5FAtest.c ) @@ -227,6 +235,7 @@ set (H5FD_SOURCES ${HDF5_SRC_DIR}/H5FDsec2.c ${HDF5_SRC_DIR}/H5FDspace.c ${HDF5_SRC_DIR}/H5FDstdio.c + ${HDF5_SRC_DIR}/H5FDtest.c ${HDF5_SRC_DIR}/H5FDwindows.c ) @@ -267,6 +276,7 @@ set (H5FS_SOURCES ${HDF5_SRC_DIR}/H5FS.c ${HDF5_SRC_DIR}/H5FScache.c ${HDF5_SRC_DIR}/H5FSdbg.c + ${HDF5_SRC_DIR}/H5FSint.c ${HDF5_SRC_DIR}/H5FSsection.c ${HDF5_SRC_DIR}/H5FSstat.c ${HDF5_SRC_DIR}/H5FStest.c @@ -959,5 +969,6 @@ if (HDF5_EXPORTED_TARGETS) ARCHIVE DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT libraries RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT libraries FRAMEWORK DESTINATION ${HDF5_INSTALL_FWRK_DIR} COMPONENT libraries + INCLUDES DESTINATION include ) endif (HDF5_EXPORTED_TARGETS) @@ -32,7 +32,7 @@ /****************/ #include "H5ACmodule.h" /* This source code file is part of the H5AC module */ -#define H5F_FRIEND /* Suppress error about including H5Fpkg */ +#define H5F_FRIEND /* Suppress error about including H5Fpkg */ /***********/ @@ -70,8 +70,6 @@ static herr_t H5AC__ext_config_2_int_config(H5AC_cache_config_t *ext_conf_ptr, #if H5AC_DO_TAGGING_SANITY_CHECKS static herr_t H5AC__verify_tag(hid_t dxpl_id, const H5AC_class_t * type); #endif /* H5AC_DO_TAGGING_SANITY_CHECKS */ -static herr_t H5AC__open_trace_file(H5AC_t *cache_ptr, const char *trace_file_name); -static herr_t H5AC__close_trace_file(H5AC_t *cache_ptr); /*********************/ @@ -137,6 +135,7 @@ static const char *H5AC_entry_type_names[H5AC_NTYPES] = "fixed array data block pages", "superblock", "driver info", + "proxy entry", "test entry" /* for testing only -- not used for actual files */ }; @@ -485,6 +484,16 @@ H5AC_create(const H5F_t *f, H5AC_cache_config_t *config_ptr) HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "H5C_set_prefix() failed") #endif /* H5_HAVE_PARALLEL */ + /* Turn on metadata cache logging, if being used */ + if(H5F_USE_MDC_LOGGING(f)) { + if(H5C_set_up_logging(f->shared->cache, H5F_MDC_LOG_LOCATION(f), H5F_START_MDC_LOG_ON_ACCESS(f)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "mdc logging setup failed") + + /* Write the log header regardless of current logging status */ + if(H5AC__write_create_cache_log_msg(f->shared->cache) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + } /* end if */ + /* Set the cache parameters */ if(H5AC_set_cache_auto_resize_config(f->shared->cache, config_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "auto resize configuration failed") @@ -549,9 +558,17 @@ H5AC_dest(H5F_t *f, hid_t dxpl_id) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5AC__close_trace_file() failed.") #endif /* H5AC__TRACE_FILE_ENABLED */ + if(H5F_USE_MDC_LOGGING(f)) { + /* Write the log footer regardless of current logging status */ + if(H5AC__write_destroy_cache_log_msg(f->shared->cache) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + if(H5C_tear_down_logging(f->shared->cache) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "mdc logging tear-down failed") + } /* end if */ + #ifdef H5_HAVE_PARALLEL /* destroying the cache, so clear all collective entries */ - if(H5C_clear_coll_entries(f->shared->cache, 0) < 0) + if(H5C_clear_coll_entries(f->shared->cache, FALSE) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5C_clear_coll_entries() failed.") aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(f->shared->cache); @@ -588,6 +605,52 @@ done: /*------------------------------------------------------------------------- + * Function: H5AC_evict + * + * Purpose: Evict all entries except the pinned entries + * in the cache. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * Dec 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_evict(H5F_t *f, hid_t dxpl_id) +{ + hbool_t log_enabled; /* TRUE if logging was set up */ + hbool_t curr_logging; /* TRUE if currently logging */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->cache); + + /* Check if log messages are being emitted */ + if(H5C_get_logging_status(f->shared->cache, &log_enabled, &curr_logging) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to get logging status") + + /* Evict all entries in the cache except the pinned superblock entry */ + if(H5C_evict(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "can't evict cache") + +done: + + /* If currently logging, generate a message */ + if(curr_logging) + if(H5AC__write_evict_cache_log_msg(f->shared->cache, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to emit log message") + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_evict() */ + + +/*------------------------------------------------------------------------- * Function: H5AC_expunge_entry * * Purpose: Expunge the target entry from the cache without writing it @@ -609,6 +672,8 @@ H5AC_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, char trace[128] = ""; FILE * trace_file_ptr = NULL; #endif /* H5AC__TRACE_FILE_ENABLED */ + hbool_t log_enabled; /* TRUE if logging was set up */ + hbool_t curr_logging; /* TRUE if currently logging */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -621,6 +686,10 @@ H5AC_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, HDassert(type->serialize); HDassert(H5F_addr_defined(addr)); + /* Check if log messages are being emitted */ + if(H5C_get_logging_status(f->shared->cache, &log_enabled, &curr_logging) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to get logging status") + #if H5AC__TRACE_FILE_ENABLED { H5AC_t * cache_ptr = f->shared->cache; @@ -643,6 +712,11 @@ done: HDfprintf(trace_file_ptr, "%s %d\n", trace, (int)ret_value); #endif /* H5AC__TRACE_FILE_ENABLED */ + /* If currently logging, generate a message */ + if(curr_logging) + if(H5AC__write_expunge_entry_log_msg(f->shared->cache, addr, type->id, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to emit log message") + FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_expunge_entry() */ @@ -674,6 +748,8 @@ H5AC_flush(H5F_t *f, hid_t dxpl_id) char trace[128] = ""; FILE * trace_file_ptr = NULL; #endif /* H5AC__TRACE_FILE_ENABLED */ + hbool_t log_enabled; /* TRUE if logging was set up */ + hbool_t curr_logging; /* TRUE if currently logging */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -683,6 +759,10 @@ H5AC_flush(H5F_t *f, hid_t dxpl_id) HDassert(f->shared); HDassert(f->shared->cache); + /* Check if log messages are being emitted */ + if(H5C_get_logging_status(f->shared->cache, &log_enabled, &curr_logging) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to get logging status") + #if H5AC__TRACE_FILE_ENABLED /* For the flush, only the flags are really necessary in the trace file. * Write the result to catch occult errors. @@ -693,7 +773,7 @@ H5AC_flush(H5F_t *f, hid_t dxpl_id) #ifdef H5_HAVE_PARALLEL /* flushing the cache, so clear all collective entries */ - if(H5C_clear_coll_entries(f->shared->cache, 0) < 0) + if(H5C_clear_coll_entries(f->shared->cache, FALSE) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5C_clear_coll_entries() failed.") /* Attempt to flush all entries from rank 0 & Bcast clean list to other ranks */ @@ -712,6 +792,11 @@ done: HDfprintf(trace_file_ptr, "%s %d\n", trace, (int)ret_value); #endif /* H5AC__TRACE_FILE_ENABLED */ + /* If currently logging, generate a message */ + if(curr_logging) + if(H5AC__write_flush_cache_log_msg(f->shared->cache, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to emit log message") + FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_flush() */ @@ -805,6 +890,8 @@ H5AC_insert_entry(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t add size_t trace_entry_size = 0; FILE * trace_file_ptr = NULL; #endif /* H5AC__TRACE_FILE_ENABLED */ + hbool_t log_enabled; /* TRUE if logging was set up */ + hbool_t curr_logging; /* TRUE if currently logging */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -818,6 +905,10 @@ H5AC_insert_entry(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t add HDassert(H5F_addr_defined(addr)); HDassert(thing); + /* Check if log messages are being emitted */ + if(H5C_get_logging_status(f->shared->cache, &log_enabled, &curr_logging) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to get logging status") + /* Check for invalid access request */ if(0 == (H5F_INTENT(f) & H5F_ACC_RDWR)) HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "no write intent on file") @@ -836,7 +927,7 @@ H5AC_insert_entry(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t add #endif /* H5AC__TRACE_FILE_ENABLED */ #if H5AC_DO_TAGGING_SANITY_CHECKS - if(!H5C_get_ignore_tags(f->shared->cache) && (H5AC__verify_tag(dxpl_id, type) < 0)) + if(!H5C_get_ignore_tags(f->shared->cache) && H5AC__verify_tag(dxpl_id, type) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "Bad tag value") #endif /* H5AC_DO_TAGGING_SANITY_CHECKS */ @@ -872,6 +963,10 @@ done: if(trace_file_ptr != NULL) HDfprintf(trace_file_ptr, "%s %d %d\n", trace, (int)trace_entry_size, (int)ret_value); #endif /* H5AC__TRACE_FILE_ENABLED */ + /* If currently logging, generate a message */ + if(curr_logging) + if(H5AC__write_insert_entry_log_msg(f->shared->cache, addr, type->id, flags, ((H5C_cache_entry_t *)thing)->size, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_insert_entry() */ @@ -897,6 +992,10 @@ H5AC_mark_entry_dirty(void *thing) char trace[128] = ""; FILE * trace_file_ptr = NULL; #endif /* H5AC__TRACE_FILE_ENABLED */ + hbool_t log_enabled; /* TRUE if logging was set up */ + hbool_t curr_logging; /* TRUE if currently logging */ + H5AC_info_t *entry_ptr = NULL; /* Pointer to the cache entry */ + H5C_t *cache_ptr = NULL; /* Pointer to the entry's associated metadata cache */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -904,20 +1003,25 @@ H5AC_mark_entry_dirty(void *thing) /* Sanity check */ HDassert(thing); + /* Set up entry & cache pointers */ + entry_ptr = (H5AC_info_t *)thing; + cache_ptr = entry_ptr->cache_ptr; + #if H5AC__TRACE_FILE_ENABLED /* For the mark pinned or protected entry dirty call, only the addr * is really necessary in the trace file. Write the result to catch * occult errors. */ if(NULL != (trace_file_ptr = H5C_get_trace_file_ptr_from_entry(thing))) - sprintf(trace, "%s 0x%lx", FUNC, - (unsigned long)(((H5C_cache_entry_t *)thing)->addr)); + sprintf(trace, "%s 0x%lx", FUNC, (unsigned long)(entry_ptr->addr)); #endif /* H5AC__TRACE_FILE_ENABLED */ + /* Check if log messages are being emitted */ + if(H5C_get_logging_status(cache_ptr, &log_enabled, &curr_logging) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to get logging status") + #ifdef H5_HAVE_PARALLEL { - H5AC_info_t *entry_ptr = (H5AC_info_t *)thing; - H5C_t *cache_ptr = entry_ptr->cache_ptr; H5AC_aux_t *aux_ptr; aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); @@ -937,11 +1041,94 @@ done: HDfprintf(trace_file_ptr, "%s %d\n", trace, (int)ret_value); #endif /* H5AC__TRACE_FILE_ENABLED */ + /* If currently logging, generate a message */ + if(curr_logging) + if(H5AC__write_mark_dirty_entry_log_msg(cache_ptr, entry_ptr, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to emit log message") + FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_mark_entry_dirty() */ /*------------------------------------------------------------------------- + * Function: H5AC_mark_entry_clean + * + * Purpose: Mark a pinned entry as dirty. The target + * entry MUST be pinned. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * 7/23/16 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_mark_entry_clean(void *thing) +{ +#if H5AC__TRACE_FILE_ENABLED + char trace[128] = ""; + FILE * trace_file_ptr = NULL; +#endif /* H5AC__TRACE_FILE_ENABLED */ + hbool_t log_enabled; /* TRUE if logging was set up */ + hbool_t curr_logging; /* TRUE if currently logging */ + H5AC_info_t *entry_ptr = NULL; /* Pointer to the cache entry */ + H5C_t *cache_ptr = NULL; /* Pointer to the entry's associated metadata cache */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(thing); + +#if H5AC__TRACE_FILE_ENABLED + /* For the mark pinned or protected entry clean call, only the addr + * is really necessary in the trace file. Write the result to catch + * occult errors. + */ + if(NULL != (trace_file_ptr = H5C_get_trace_file_ptr_from_entry(thing))) + sprintf(trace, "%s 0x%lx", FUNC, + (unsigned long)(((H5C_cache_entry_t *)thing)->addr)); +#endif /* H5AC__TRACE_FILE_ENABLED */ + + entry_ptr = (H5AC_info_t *)thing; + cache_ptr = entry_ptr->cache_ptr; + + /* Check if log messages are being emitted */ + if(H5C_get_logging_status(cache_ptr, &log_enabled, &curr_logging) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to get logging status") + +#ifdef H5_HAVE_PARALLEL +{ + H5AC_aux_t *aux_ptr; + + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + if((!entry_ptr->is_dirty) && (!entry_ptr->is_protected) && + (entry_ptr->is_pinned) && (NULL != aux_ptr)) + if(H5AC__log_cleaned_entry(entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKCLEAN, FAIL, "can't log cleaned entry") +} +#endif /* H5_HAVE_PARALLEL */ + + if(H5C_mark_entry_clean(thing) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKCLEAN, FAIL, "can't mark pinned or protected entry clean") + +done: +#if H5AC__TRACE_FILE_ENABLED + if(trace_file_ptr) + HDfprintf(trace_file_ptr, "%s %d\n", trace, (int)ret_value); +#endif /* H5AC__TRACE_FILE_ENABLED */ + + /* If currently logging, generate a message */ + if(curr_logging) + if(H5AC__write_mark_clean_entry_log_msg(cache_ptr, entry_ptr, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to emit log message") + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_mark_entry_clean() */ + + +/*------------------------------------------------------------------------- * Function: H5AC_move_entry * * Purpose: Use this function to notify the cache that an object's @@ -970,6 +1157,8 @@ H5_ATTR_UNUSED #ifdef H5_HAVE_PARALLEL H5AC_aux_t *aux_ptr; #endif /* H5_HAVE_PARALLEL */ + hbool_t log_enabled; /* TRUE if logging was set up */ + hbool_t curr_logging; /* TRUE if currently logging */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -982,6 +1171,10 @@ H5_ATTR_UNUSED HDassert(H5F_addr_defined(new_addr)); HDassert(H5F_addr_ne(old_addr, new_addr)); + /* Check if log messages are being emitted */ + if(H5C_get_logging_status(f->shared->cache, &log_enabled, &curr_logging) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to get logging status") + #if H5AC__TRACE_FILE_ENABLED /* For the move call, only the old addr and new addr are really * necessary in the trace file. Include the type id so we don't have to @@ -1015,6 +1208,11 @@ done: HDfprintf(trace_file_ptr, "%s %d\n", trace, (int)ret_value); #endif /* H5AC__TRACE_FILE_ENABLED */ + /* If currently logging, generate a message */ + if(curr_logging) + if(H5AC__write_move_entry_log_msg(f->shared->cache, old_addr, new_addr, type->id, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to emit log message") + FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_move_entry() */ @@ -1039,6 +1237,10 @@ H5AC_pin_protected_entry(void *thing) char trace[128] = ""; FILE * trace_file_ptr = NULL; #endif /* H5AC__TRACE_FILE_ENABLED */ + hbool_t log_enabled; /* TRUE if logging was set up */ + hbool_t curr_logging; /* TRUE if currently logging */ + H5AC_info_t *entry_ptr = NULL; /* Pointer to the cache entry */ + H5C_t *cache_ptr = NULL; /* Pointer to the entry's associated metadata cache */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -1055,6 +1257,14 @@ H5AC_pin_protected_entry(void *thing) (unsigned long)(((H5C_cache_entry_t *)thing)->addr)); #endif /* H5AC__TRACE_FILE_ENABLED */ + entry_ptr = (H5AC_info_t *)thing; + cache_ptr = entry_ptr->cache_ptr; + HDassert(cache_ptr); + + /* Check if log messages are being emitted */ + if(H5C_get_logging_status(cache_ptr, &log_enabled, &curr_logging) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to get logging status") + /* Pin entry */ if(H5C_pin_protected_entry(thing) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTPIN, FAIL, "can't pin entry") @@ -1065,6 +1275,11 @@ done: HDfprintf(trace_file_ptr, "%s %d\n", trace, (int)ret_value); #endif /* H5AC__TRACE_FILE_ENABLED */ + /* If currently logging, generate a message */ + if(curr_logging) + if(H5AC__write_pin_entry_log_msg(cache_ptr, entry_ptr, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to emit log message") + FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_pin_protected_entry() */ @@ -1089,6 +1304,10 @@ H5AC_create_flush_dependency(void * parent_thing, void * child_thing) char trace[128] = ""; FILE * trace_file_ptr = NULL; #endif /* H5AC__TRACE_FILE_ENABLED */ + hbool_t log_enabled; /* TRUE if logging was set up */ + hbool_t curr_logging; /* TRUE if currently logging */ + H5AC_info_t *entry_ptr = NULL; /* Pointer to the cache entry */ + H5C_t *cache_ptr = NULL; /* Pointer to the entry's associated metadata cache */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -1104,6 +1323,14 @@ H5AC_create_flush_dependency(void * parent_thing, void * child_thing) (unsigned long)(((H5C_cache_entry_t *)child_thing)->addr)); #endif /* H5AC__TRACE_FILE_ENABLED */ + entry_ptr = (H5AC_info_t *)parent_thing; + cache_ptr = entry_ptr->cache_ptr; + HDassert(cache_ptr); + + /* Check if log messages are being emitted */ + if(H5C_get_logging_status(cache_ptr, &log_enabled, &curr_logging) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to get logging status") + /* Create the flush dependency */ if(H5C_create_flush_dependency(parent_thing, child_thing) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTDEPEND, FAIL, "H5C_create_flush_dependency() failed.") @@ -1114,6 +1341,11 @@ done: HDfprintf(trace_file_ptr, "%s %d\n", trace, (int)ret_value); #endif /* H5AC__TRACE_FILE_ENABLED */ + /* If currently logging, generate a message */ + if(curr_logging) + if(H5AC__write_create_fd_log_msg(cache_ptr, (H5AC_info_t *)parent_thing, (H5AC_info_t *)child_thing, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to emit log message") + FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_create_flush_dependency() */ @@ -1152,6 +1384,8 @@ H5AC_protect(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, FILE * trace_file_ptr = NULL; #endif /* H5AC__TRACE_FILE_ENABLED */ void * thing = NULL; /* Pointer to native data structure for entry */ + hbool_t log_enabled; /* TRUE if logging was set up */ + hbool_t curr_logging; /* TRUE if currently logging */ void * ret_value = NULL; /* Return value */ FUNC_ENTER_NOAPI(NULL) @@ -1164,6 +1398,10 @@ H5AC_protect(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, HDassert(type->serialize); HDassert(H5F_addr_defined(addr)); + /* Check if log messages are being emitted */ + if(H5C_get_logging_status(f->shared->cache, &log_enabled, &curr_logging) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, NULL, "unable to get logging status") + /* Check for unexpected flags -- H5C__FLUSH_COLLECTIVELY_FLAG * only permitted in the parallel case. */ @@ -1177,7 +1415,7 @@ H5AC_protect(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, #endif /* H5_HAVE_PARALLEL */ /* Check for invalid access request */ - if((0 == (H5F_INTENT(f) & H5F_ACC_RDWR)) && (0 == (flags & H5C__READ_ONLY_FLAG))) + if((0 == (H5F_INTENT(f) & H5F_ACC_RDWR)) && (0 == (flags & H5C__READ_ONLY_FLAG))) HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "no write intent on file") #if H5AC__TRACE_FILE_ENABLED @@ -1191,7 +1429,7 @@ H5AC_protect(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, #endif /* H5AC__TRACE_FILE_ENABLED */ #if H5AC_DO_TAGGING_SANITY_CHECKS - if(!H5C_get_ignore_tags(f->shared->cache) && (H5AC__verify_tag(dxpl_id, type) < 0)) + if(!H5C_get_ignore_tags(f->shared->cache) && H5AC__verify_tag(dxpl_id, type) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, NULL, "Bad tag value") #endif /* H5AC_DO_TAGGING_SANITY_CHECKS */ @@ -1213,6 +1451,14 @@ done: HDfprintf(trace_file_ptr, "%s %d %d\n", trace, (int)trace_entry_size, (int)(ret_value != NULL)); #endif /* H5AC__TRACE_FILE_ENABLED */ + /* If currently logging, generate a message */ + if(curr_logging) { + herr_t fake_ret_value = (NULL == ret_value) ? FAIL : SUCCEED; + + if(H5AC__write_protect_entry_log_msg(f->shared->cache, (H5AC_info_t *)thing, flags, fake_ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGFAIL, NULL, "unable to emit log message") + } /* end if */ + FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_protect() */ @@ -1236,6 +1482,10 @@ H5AC_resize_entry(void *thing, size_t new_size) char trace[128] = ""; FILE * trace_file_ptr = NULL; #endif /* H5AC__TRACE_FILE_ENABLED */ + hbool_t log_enabled; /* TRUE if logging was set up */ + hbool_t curr_logging; /* TRUE if currently logging */ + H5AC_info_t *entry_ptr = NULL; /* Pointer to the cache entry */ + H5C_t *cache_ptr = NULL; /* Pointer to the entry's associated metadata cache */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -1254,14 +1504,20 @@ H5AC_resize_entry(void *thing, size_t new_size) (int)new_size); #endif /* H5AC__TRACE_FILE_ENABLED */ + entry_ptr = (H5AC_info_t *)thing; + cache_ptr = entry_ptr->cache_ptr; + HDassert(cache_ptr); + + /* Check if log messages are being emitted */ + if(H5C_get_logging_status(cache_ptr, &log_enabled, &curr_logging) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to get logging status") + /* Resize the entry */ if(H5C_resize_entry(thing, new_size) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTRESIZE, FAIL, "can't resize entry") #ifdef H5_HAVE_PARALLEL { - H5AC_info_t * entry_ptr = (H5AC_info_t *)thing; - H5C_t *cache_ptr = entry_ptr->cache_ptr; H5AC_aux_t *aux_ptr; aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); @@ -1277,6 +1533,11 @@ done: HDfprintf(trace_file_ptr, "%s %d\n", trace, (int)ret_value); #endif /* H5AC__TRACE_FILE_ENABLED */ + /* If currently logging, generate a message */ + if(curr_logging) + if(H5AC__write_resize_entry_log_msg(cache_ptr, entry_ptr, new_size, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to emit log message") + FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_resize_entry() */ @@ -1301,6 +1562,10 @@ H5AC_unpin_entry(void *thing) char trace[128] = ""; FILE * trace_file_ptr = NULL; #endif /* H5AC__TRACE_FILE_ENABLED */ + hbool_t log_enabled; /* TRUE if logging was set up */ + hbool_t curr_logging; /* TRUE if currently logging */ + H5AC_info_t *entry_ptr = NULL; /* Pointer to the cache entry */ + H5C_t *cache_ptr = NULL; /* Pointer to the entry's associated metadata cache */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -1317,6 +1582,14 @@ H5AC_unpin_entry(void *thing) (unsigned long)(((H5C_cache_entry_t *)thing)->addr)); #endif /* H5AC__TRACE_FILE_ENABLED */ + entry_ptr = (H5AC_info_t *)thing; + cache_ptr = entry_ptr->cache_ptr; + HDassert(cache_ptr); + + /* Check if log messages are being emitted */ + if(H5C_get_logging_status(cache_ptr, &log_enabled, &curr_logging) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to get logging status") + /* Unpin the entry */ if(H5C_unpin_entry(thing) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPIN, FAIL, "can't unpin entry") @@ -1327,6 +1600,11 @@ done: HDfprintf(trace_file_ptr, "%s %d\n", trace, (int)ret_value); #endif /* H5AC__TRACE_FILE_ENABLED */ + /* If currently logging, generate a message */ + if(curr_logging) + if(H5AC__write_unpin_entry_log_msg(cache_ptr, entry_ptr, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to emit log message") + FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_unpin_entry() */ @@ -1350,6 +1628,10 @@ H5AC_destroy_flush_dependency(void * parent_thing, void * child_thing) char trace[128] = ""; FILE * trace_file_ptr = NULL; #endif /* H5AC__TRACE_FILE_ENABLED */ + hbool_t log_enabled; /* TRUE if logging was set up */ + hbool_t curr_logging; /* TRUE if currently logging */ + H5AC_info_t *entry_ptr = NULL; /* Pointer to the cache entry */ + H5C_t *cache_ptr = NULL; /* Pointer to the entry's associated metadata cache */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -1365,6 +1647,14 @@ H5AC_destroy_flush_dependency(void * parent_thing, void * child_thing) (unsigned long long)(((H5C_cache_entry_t *)child_thing)->addr)); #endif /* H5AC__TRACE_FILE_ENABLED */ + entry_ptr = (H5AC_info_t *)parent_thing; + cache_ptr = entry_ptr->cache_ptr; + HDassert(cache_ptr); + + /* Check if log messages are being emitted */ + if(H5C_get_logging_status(cache_ptr, &log_enabled, &curr_logging) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to get logging status") + /* Destroy the flush dependency */ if(H5C_destroy_flush_dependency(parent_thing, child_thing) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTUNDEPEND, FAIL, "H5C_destroy_flush_dependency() failed.") @@ -1375,6 +1665,11 @@ done: HDfprintf(trace_file_ptr, "%s %d\n", trace, (int)ret_value); #endif /* H5AC__TRACE_FILE_ENABLED */ + /* If currently logging, generate a message */ + if(curr_logging) + if(H5AC__write_destroy_fd_log_msg(cache_ptr, (H5AC_info_t *)parent_thing, (H5AC_info_t *)child_thing, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to emit log message") + FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_destroy_flush_dependency() */ @@ -1430,6 +1725,8 @@ H5AC_unprotect(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, #ifdef H5_HAVE_PARALLEL H5AC_aux_t * aux_ptr = NULL; #endif /* H5_HAVE_PARALLEL */ + hbool_t log_enabled; /* TRUE if logging was set up */ + hbool_t curr_logging; /* TRUE if currently logging */ herr_t ret_value=SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -1446,6 +1743,10 @@ H5AC_unprotect(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, HDassert( ((H5AC_info_t *)thing)->addr == addr ); HDassert( ((H5AC_info_t *)thing)->type == type ); + /* Check if log messages are being emitted */ + if(H5C_get_logging_status(f->shared->cache, &log_enabled, &curr_logging) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to get logging status") + #if H5AC__TRACE_FILE_ENABLED /* For the unprotect call, only the addr, type id, flags, and possible * new size are really necessary in the trace file. Write the return @@ -1463,11 +1764,9 @@ H5AC_unprotect(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, * the entry. */ if(dirtied && !deleted) { - hbool_t curr_compressed = FALSE; /* dummy for call */ size_t curr_size = 0; - size_t curr_compressed_size = 0; /* dummy for call */ - if((type->image_len)(thing, &curr_size, &curr_compressed, &curr_compressed_size) < 0) + if((type->image_len)(thing, &curr_size) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTGETSIZE, FAIL, "Can't get size of thing") if(((H5AC_info_t *)thing)->size != curr_size) @@ -1502,72 +1801,13 @@ done: HDfprintf(trace_file_ptr, "%s 0x%x %d\n", trace, (unsigned)flags, (int)ret_value); #endif /* H5AC__TRACE_FILE_ENABLED */ - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC_unprotect() */ - -#ifndef NDEBUG /* debugging functions */ - -/*------------------------------------------------------------------------- - * Function: H5AC_stats - * - * Purpose: Prints statistics about the cache. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Robb Matzke - * Thursday, October 30, 1997 - * - *------------------------------------------------------------------------- - */ -herr_t -H5AC_stats(const H5F_t *f) -{ - FUNC_ENTER_NOAPI_NOINIT_NOERR - - /* Sanity checks */ - HDassert(f); - HDassert(f->shared); - HDassert(f->shared->cache); - - /* at present, this can't fail */ - (void)H5C_stats(f->shared->cache, H5F_OPEN_NAME(f), FALSE); - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5AC_stats() */ - - -/*------------------------------------------------------------------------- - * Function: H5AC_dump_cache - * - * Purpose: Dumps a summary of the contents of the metadata cache - * to stdout. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: John Mainzer - * Sunday, October 10, 2010 - * - *------------------------------------------------------------------------- - */ -herr_t -H5AC_dump_cache(const H5F_t *f) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) + /* If currently logging, generate a message */ + if(curr_logging) + if(H5AC__write_unprotect_entry_log_msg(f->shared->cache, (H5AC_info_t *)thing, type->id, flags, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to emit log message") - /* Sanity checks */ - HDassert(f); - HDassert(f->shared); - HDassert(f->shared->cache); - - if(H5C_dump_cache(f->shared->cache, H5F_OPEN_NAME(f)) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_dump_cache() failed.") - -done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC_dump_cache() */ -#endif /* NDEBUG */ +} /* H5AC_unprotect() */ /*------------------------------------------------------------------------- @@ -1767,6 +2007,8 @@ H5AC_set_cache_auto_resize_config(H5AC_t *cache_ptr, H5AC_cache_config_t *config H5AC_cache_config_t trace_config = H5AC__DEFAULT_CACHE_CONFIG; FILE * trace_file_ptr = NULL; #endif /* H5AC__TRACE_FILE_ENABLED */ + hbool_t log_enabled; /* TRUE if logging was set up */ + hbool_t curr_logging; /* TRUE if currently logging */ H5C_auto_size_ctl_t internal_config; herr_t ret_value = SUCCEED; /* Return value */ @@ -1775,6 +2017,10 @@ H5AC_set_cache_auto_resize_config(H5AC_t *cache_ptr, H5AC_cache_config_t *config /* Sanity checks */ HDassert(cache_ptr); + /* Check if log messages are being emitted */ + if(H5C_get_logging_status(cache_ptr, &log_enabled, &curr_logging) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to get logging status") + #if H5AC__TRACE_FILE_ENABLED /* Make note of the new configuration. Don't look up the trace file * pointer, as that may change before we use it. @@ -1883,6 +2129,11 @@ done: (int)ret_value); #endif /* H5AC__TRACE_FILE_ENABLED */ + /* If currently logging, generate a message */ + if(curr_logging) + if(H5AC__write_set_cache_config_log_msg(cache_ptr, config_ptr, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to emit log message") + FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_set_cache_auto_resize_config() */ @@ -1964,121 +2215,6 @@ done: } /* H5AC_validate_config() */ -/*------------------------------------------------------------------------- - * Function: H5AC__close_trace_file() - * - * Purpose: If a trace file is open, stop logging calls to the cache, - * and close the file. - * - * Note that the function does nothing if there is no trace - * file. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: John Mainzer - * 6/2/06 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5AC__close_trace_file(H5AC_t *cache_ptr) -{ - FILE * trace_file_ptr = NULL; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - if(cache_ptr == NULL) - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "NULL cache_ptr on entry.") - - if(NULL == (trace_file_ptr = H5C_get_trace_file_ptr(cache_ptr))) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_get_trace_file_ptr() failed.") - - if(trace_file_ptr != NULL) { - if(H5C_set_trace_file_ptr(cache_ptr, NULL) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_set_trace_file_ptr() failed.") - - if(HDfclose(trace_file_ptr) != 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "can't close metadata cache trace file") - } /* end if */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__close_trace_file() */ - - -/*------------------------------------------------------------------------- - * Function: H5AC__open_trace_file() - * - * Purpose: Open a trace file, and start logging calls to the cache. - * - * This logging is done at the H5C level, and will only take - * place if H5C_TRACE_FILE_ENABLED (defined in H5Cprivate.h) - * is TRUE. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: John Mainzer - * 6/1/06 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5AC__open_trace_file(H5AC_t *cache_ptr, const char *trace_file_name) -{ - char file_name[H5AC__MAX_TRACE_FILE_NAME_LEN + H5C__PREFIX_LEN + 2]; - FILE * file_ptr = NULL; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - HDassert(cache_ptr); - - /* Check args */ - if(cache_ptr == NULL) - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "cache_ptr NULL on entry.") - if(trace_file_name == NULL) - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "NULL trace_file_name on entry.") - if(HDstrlen(trace_file_name) > H5AC__MAX_TRACE_FILE_NAME_LEN) - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "trace file name too long.") - if(NULL != (file_ptr = H5C_get_trace_file_ptr(cache_ptr))) - HGOTO_ERROR(H5E_CACHE, H5E_FILEOPEN, FAIL, "trace file already open.") - -#ifdef H5_HAVE_PARALLEL -{ - H5AC_aux_t * aux_ptr; - - aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); - if(aux_ptr == NULL) - sprintf(file_name, "%s", trace_file_name); - else { - if(aux_ptr->magic != H5AC__H5AC_AUX_T_MAGIC) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Bad aux_ptr->magic.") - - sprintf(file_name, "%s.%d", trace_file_name, aux_ptr->mpi_rank); - } /* end else */ - - if(HDstrlen(file_name) > H5AC__MAX_TRACE_FILE_NAME_LEN + H5C__PREFIX_LEN + 1) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "cooked trace file name too long.") -} -#else /* H5_HAVE_PARALLEL */ - HDsnprintf(file_name, (size_t)(H5AC__MAX_TRACE_FILE_NAME_LEN + H5C__PREFIX_LEN + 1), - "%s", trace_file_name); -#endif /* H5_HAVE_PARALLEL */ - - if((file_ptr = HDfopen(file_name, "w")) == NULL) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "trace file open failed.") - - HDfprintf(file_ptr, "### HDF5 metadata cache trace file version 1 ###\n"); - - if(H5C_set_trace_file_ptr(cache_ptr, file_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_set_trace_file_ptr() failed.") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__open_trace_file() */ - - /*************************************************************************/ /**************************** Private Functions: *************************/ /*************************************************************************/ @@ -2256,7 +2392,6 @@ done: herr_t H5AC_tag(hid_t dxpl_id, haddr_t metadata_tag, haddr_t *prev_tag) { - H5C_tag_t tag; /* Tag structure */ H5P_genplist_t *dxpl; /* Dataset transfer property list */ herr_t ret_value = SUCCEED; /* Return value */ @@ -2268,32 +2403,16 @@ H5AC_tag(hid_t dxpl_id, haddr_t metadata_tag, haddr_t *prev_tag) /* Get the current tag value and return that (if prev_tag is NOT null) */ if(prev_tag) { - if((H5P_get(dxpl, "H5C_tag", &tag)) < 0) + haddr_t tag; /* Tag value */ + + if((H5P_get(dxpl, H5AC_TAG_NAME, &tag)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to query dxpl") - *prev_tag = tag.value; + *prev_tag = tag; } /* end if */ - /* Add metadata_tag to tag structure */ - tag.value = metadata_tag; - - /* Determine globality of tag */ - switch(metadata_tag) { - case H5AC__SUPERBLOCK_TAG: - case H5AC__SOHM_TAG: - case H5AC__GLOBALHEAP_TAG: - tag.globality = H5C_GLOBALITY_MAJOR; - break; - case H5AC__FREESPACE_TAG: - tag.globality = H5C_GLOBALITY_MINOR; - break; - default: - tag.globality = H5C_GLOBALITY_NONE; - break; - } /* end switch */ - /* Set the provided tag in the dxpl_id. */ - if(H5P_set(dxpl, "H5C_tag", &tag) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTSET, FAIL, "can't set property in dxpl") + if(H5P_set(dxpl, H5AC_TAG_NAME, &metadata_tag) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTSET, FAIL, "can't set tag in dxpl") done: FUNC_LEAVE_NOAPI(ret_value) @@ -2383,7 +2502,7 @@ done: *------------------------------------------------------------------------------ */ herr_t -H5AC_evict_tagged_metadata(H5F_t * f, haddr_t metadata_tag, hid_t dxpl_id) +H5AC_evict_tagged_metadata(H5F_t * f, haddr_t metadata_tag, hbool_t match_global, hid_t dxpl_id) { /* Variable Declarations */ herr_t ret_value = SUCCEED; @@ -2396,7 +2515,7 @@ H5AC_evict_tagged_metadata(H5F_t * f, haddr_t metadata_tag, hid_t dxpl_id) HDassert(f->shared); /* Call cache level function to evict metadata entries with specified tag */ - if(H5C_evict_tagged_entries(f, dxpl_id, metadata_tag) < 0) + if(H5C_evict_tagged_entries(f, dxpl_id, metadata_tag, match_global) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Cannot evict metadata") done: @@ -2435,10 +2554,43 @@ H5AC_expunge_tag_type_metadata(H5F_t *f, hid_t dxpl_id, haddr_t tag, int type_id done: FUNC_LEAVE_NOAPI(ret_value) - } /* H5AC_expunge_tag_type_metadata*/ +/*------------------------------------------------------------------------------ + * Function: H5AC_get_tag() + * + * Purpose: Get the tag for a metadata cache entry. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Fall 2016 + * + *------------------------------------------------------------------------------ + */ +herr_t +H5AC_get_tag(const void *thing, haddr_t *tag) +{ + /* Variable Declarations */ + herr_t ret_value = SUCCEED; + + /* Function Enter Macro */ + FUNC_ENTER_NOAPI(FAIL) + + /* Assertions */ + HDassert(thing); + HDassert(tag); + + /* Call cache level function to get the tag */ + if(H5C_get_tag(thing, tag) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "Cannot get tag for metadata cache entry") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5AC_get_tag() */ + + /*------------------------------------------------------------------------- * Function: H5AC_cork * @@ -2494,7 +2646,7 @@ static herr_t H5AC__verify_tag(hid_t dxpl_id, const H5AC_class_t *type) { H5P_genplist_t *dxpl; /* DXPL for operation */ - H5C_tag_t tag; /* Entry tag to validate */ + haddr_t tag; /* Entry tag to validate */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -2504,11 +2656,11 @@ H5AC__verify_tag(hid_t dxpl_id, const H5AC_class_t *type) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") /* Get the tag from the DXPL */ - if((H5P_get(dxpl, "H5C_tag", &tag)) < 0) + if((H5P_get(dxpl, H5AC_TAG_NAME, &tag)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to query property value") /* Verify legal tag value */ - if(H5C_verify_tag(type->id, tag.value, tag.globality) < 0) + if(H5C_verify_tag(type->id, tag) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "tag verification failed") done: @@ -2626,3 +2778,67 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5AC_reset_ring() */ + +/*------------------------------------------------------------------------- + * Function: H5AC_remove_entry() + * + * Purpose: Remove an entry from the cache. Must be not protected, pinned, + * dirty, involved in flush dependencies, etc. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * September 17, 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_remove_entry(void *_entry) +{ + H5AC_info_t *entry = (H5AC_info_t *)_entry; /* Entry to remove */ + H5C_t *cache = NULL; /* Pointer to the entry's associated metadata cache */ +#if H5AC__TRACE_FILE_ENABLED + char trace[128] = ""; + FILE * trace_file_ptr = NULL; +#endif /* H5AC__TRACE_FILE_ENABLED */ + hbool_t log_enabled; /* TRUE if logging was set up */ + hbool_t curr_logging; /* TRUE if currently logging */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(entry); + cache = entry->cache_ptr; + HDassert(cache); + +#if H5AC__TRACE_FILE_ENABLED + /* For the remove entry call, only the addr is really necessary + * in the trace file. Also write the result to catch occult errors. + */ + if(NULL != (trace_file_ptr = H5C_get_trace_file_ptr_from_entry(entry))) + sprintf(trace, "%s 0x%lx", FUNC, (unsigned long)(entry->addr)); +#endif /* H5AC__TRACE_FILE_ENABLED */ + + /* Check if log messages are being emitted */ + if(H5C_get_logging_status(cache, &log_enabled, &curr_logging) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "unable to get logging status") + + /* Remove the entry from the cache*/ + if(H5C_remove_entry(entry) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove entry") + +done: +#if H5AC__TRACE_FILE_ENABLED + if(trace_file_ptr) + HDfprintf(trace_file_ptr, "%s %d\n", trace, (int)ret_value); +#endif /* H5AC__TRACE_FILE_ENABLED */ + + /* If currently logging, generate a message */ + if(curr_logging) + if(H5AC__write_remove_entry_log_msg(cache, entry, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to emit log message") + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_remove_entry() */ + diff --git a/src/H5ACdbg.c b/src/H5ACdbg.c new file mode 100644 index 0000000..6120242 --- /dev/null +++ b/src/H5ACdbg.c @@ -0,0 +1,251 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5ACdbg.c + * + * Purpose: Functions for debugging the metadata cache + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ +#include "H5ACmodule.h" /* This source code file is part of the H5AC module */ +#define H5F_FRIEND /* Suppress error about including H5Fpkg */ + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5ACpkg.h" /* Metadata cache */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* Files */ + + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + + + +/*------------------------------------------------------------------------- + * Function: H5AC_stats + * + * Purpose: Prints statistics about the cache. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Robb Matzke + * Thursday, October 30, 1997 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_stats(const H5F_t *f) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->cache); + + /* at present, this can't fail */ + (void)H5C_stats(f->shared->cache, H5F_OPEN_NAME(f), FALSE); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5AC_stats() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC_dump_cache + * + * Purpose: Dumps a summary of the contents of the metadata cache + * to stdout. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * Sunday, October 10, 2010 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_dump_cache(const H5F_t *f) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->cache); + + if(H5C_dump_cache(f->shared->cache, H5F_OPEN_NAME(f)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_dump_cache() failed.") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_dump_cache() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__close_trace_file() + * + * Purpose: If a trace file is open, stop logging calls to the cache, + * and close the file. + * + * Note that the function does nothing if there is no trace + * file. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 6/2/06 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__close_trace_file(H5AC_t *cache_ptr) +{ + FILE * trace_file_ptr; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + if(cache_ptr == NULL) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "NULL cache_ptr on entry.") + + if(NULL == (trace_file_ptr = H5C_get_trace_file_ptr(cache_ptr))) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_get_trace_file_ptr() failed.") + + if(trace_file_ptr != NULL) { + if(H5C_set_trace_file_ptr(cache_ptr, NULL) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_set_trace_file_ptr() failed.") + + if(HDfclose(trace_file_ptr) != 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "can't close metadata cache trace file") + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__close_trace_file() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__open_trace_file() + * + * Purpose: Open a trace file, and start logging calls to the cache. + * + * This logging is done at the H5C level, and will only take + * place if H5C_TRACE_FILE_ENABLED (defined in H5Cprivate.h) + * is TRUE. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 6/1/06 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__open_trace_file(H5AC_t *cache_ptr, const char *trace_file_name) +{ + char file_name[H5AC__MAX_TRACE_FILE_NAME_LEN + H5C__PREFIX_LEN + 2]; + FILE * file_ptr; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + HDassert(cache_ptr); + + /* Check args */ + if(cache_ptr == NULL) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "cache_ptr NULL on entry.") + if(trace_file_name == NULL) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "NULL trace_file_name on entry.") + if(HDstrlen(trace_file_name) > H5AC__MAX_TRACE_FILE_NAME_LEN) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "trace file name too long.") + if(NULL != (file_ptr = H5C_get_trace_file_ptr(cache_ptr))) + HGOTO_ERROR(H5E_CACHE, H5E_FILEOPEN, FAIL, "trace file already open.") + +#ifdef H5_HAVE_PARALLEL +{ + H5AC_aux_t * aux_ptr; + + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + if(aux_ptr == NULL) + sprintf(file_name, "%s", trace_file_name); + else { + if(aux_ptr->magic != H5AC__H5AC_AUX_T_MAGIC) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Bad aux_ptr->magic.") + + sprintf(file_name, "%s.%d", trace_file_name, aux_ptr->mpi_rank); + } /* end else */ + + if(HDstrlen(file_name) > H5AC__MAX_TRACE_FILE_NAME_LEN + H5C__PREFIX_LEN + 1) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "cooked trace file name too long.") +} +#else /* H5_HAVE_PARALLEL */ + HDsnprintf(file_name, (size_t)(H5AC__MAX_TRACE_FILE_NAME_LEN + H5C__PREFIX_LEN + 1), + "%s", trace_file_name); +#endif /* H5_HAVE_PARALLEL */ + + if((file_ptr = HDfopen(file_name, "w")) == NULL) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "trace file open failed.") + + HDfprintf(file_ptr, "### HDF5 metadata cache trace file version 1 ###\n"); + + if(H5C_set_trace_file_ptr(cache_ptr, file_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_set_trace_file_ptr() failed.") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__open_trace_file() */ + diff --git a/src/H5AClog.c b/src/H5AClog.c new file mode 100644 index 0000000..1cdaa00 --- /dev/null +++ b/src/H5AClog.c @@ -0,0 +1,1012 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5AClog.c + * + * Purpose: Functions for metadata cache logging in JSON format + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ +#include "H5ACmodule.h" /* This source code file is part of the H5AC module */ + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5ACpkg.h" /* Metadata cache */ +#include "H5Cprivate.h" /* Cache */ +#include "H5Eprivate.h" /* Error handling */ + + +/****************/ +/* Local Macros */ +/****************/ + +#define MSG_SIZE 128 + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + + + +/*------------------------------------------------------------------------- + * Function: H5AC__write_create_cache_log_msg + * + * Purpose: Write a log message for cache creation. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__write_create_cache_log_msg(H5AC_t *cache) +{ + char msg[MSG_SIZE]; + hbool_t log_enabled; /* TRUE if logging was set up */ + hbool_t curr_logging; /* TRUE if currently logging */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache); + + /* Check if log messages are being emitted */ + if(H5C_get_logging_status(cache, &log_enabled, &curr_logging) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to get logging status") + + /* Since we're about to override the current logging flag, + * check the "log enabled" flag to see if we didn't get here + * by mistake. + */ + if(!log_enabled) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "attempt to write opening log message when logging is disabled") + + /* Create the log message string */ + HDsnprintf(msg, MSG_SIZE, +"\ +{\n\ +\"create_time\":%lld,\n\ +\"messages\":\n\ +[\n\ +" + , (long long)HDtime(NULL)); + + /* Have to temporarily enable logging, if it isn't currently */ + if(!curr_logging) + if(H5C_start_logging(cache) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to start mdc logging") + + /* Write the log message to the file */ + if(H5C_write_log_message(cache, msg) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + + /* Stop logging, if it wasn't started originally */ + if(!curr_logging) + if(H5C_stop_logging(cache) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to stop mdc logging") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__write_create_cache_log_msg() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__write_destroy_cache_log_msg + * + * Purpose: Write a log message for cache destruction. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__write_destroy_cache_log_msg(H5AC_t *cache) +{ + char msg[MSG_SIZE]; + hbool_t log_enabled; /* TRUE if logging was set up */ + hbool_t curr_logging; /* TRUE if currently logging */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache); + + /* Check if log messages are being emitted */ + if(H5C_get_logging_status(cache, &log_enabled, &curr_logging) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to get logging status") + + /* Since we're about to override the current logging flag, + * check the "log enabled" flag to see if we didn't get here + * by mistake. + */ + if(!log_enabled) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "attempt to write closing log message when logging is disabled") + + /* Create the log message string */ + HDsnprintf(msg, MSG_SIZE, +"\ +],\n\ +\"close_time\":%lld,\n\ +}\n\ +" + , (long long)HDtime(NULL)); + + /* Have to temporarily enable logging, if it isn't currently */ + if(!curr_logging) + if(H5C_start_logging(cache) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to start mdc logging") + + /* Write the log message to the file */ + if(H5C_write_log_message(cache, msg) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + + /* Stop logging, if it wasn't started originally */ + if(!curr_logging) + if(H5C_stop_logging(cache) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_LOGFAIL, FAIL, "unable to stop mdc logging") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__write_destroy_cache_log_msg() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__write_evict_cache_log_msg + * + * Purpose: Write a log message for eviction of cache entries. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__write_evict_cache_log_msg(const H5AC_t *cache, + herr_t fxn_ret_value) +{ + char msg[MSG_SIZE]; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache); + + /* Create the log message string */ + HDsnprintf(msg, MSG_SIZE, +"\ +{\ +\"timestamp\":%lld,\ +\"action\":\"evict\",\ +\"returned\":%d\ +},\n\ +" + , (long long)HDtime(NULL), (int)fxn_ret_value); + + /* Write the log message to the file */ + if(H5C_write_log_message(cache, msg) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__write_evict_cache_log_msg() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__write_expunge_entry_log_msg + * + * Purpose: Write a log message for expunge of cache entries. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__write_expunge_entry_log_msg(const H5AC_t *cache, + haddr_t address, + int type_id, + herr_t fxn_ret_value) +{ + char msg[MSG_SIZE]; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache); + + /* Create the log message string */ + HDsnprintf(msg, MSG_SIZE, +"\ +{\ +\"timestamp\":%lld,\ +\"action\":\"expunge\",\ +\"address\":0x%lx,\ +\"type_id\":%d,\ +\"returned\":%d\ +},\n\ +" + , (long long)HDtime(NULL), (unsigned long)address, (int)type_id, (int)fxn_ret_value); + + + /* Write the log message to the file */ + if(H5C_write_log_message(cache, msg) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__write_expunge_entry_log_msg() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__write_flush_cache_log_msg + * + * Purpose: Write a log message for cache flushes. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__write_flush_cache_log_msg(const H5AC_t *cache, + herr_t fxn_ret_value) +{ + char msg[MSG_SIZE]; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache); + + /* Create the log message string */ + HDsnprintf(msg, MSG_SIZE, +"\ +{\ +\"timestamp\":%lld,\ +\"action\":\"flush\",\ +\"returned\":%d\ +},\n\ +" + , (long long)HDtime(NULL), (int)fxn_ret_value); + + /* Write the log message to the file */ + if(H5C_write_log_message(cache, msg) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__write_flush_cache_log_msg() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__write_insert_entry_log_msg + * + * Purpose: Write a log message for insertion of cache entries. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__write_insert_entry_log_msg(const H5AC_t *cache, + haddr_t address, + int type_id, + unsigned flags, + size_t size, + herr_t fxn_ret_value) +{ + char msg[MSG_SIZE]; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache); + + + /* Create the log message string */ + HDsnprintf(msg, MSG_SIZE, +"\ +{\ +\"timestamp\":%lld,\ +\"action\":\"insert\",\ +\"address\":0x%lx,\ +\"flags\":0x%x,\ +\"type_id\":%d,\ +\"size\":%d,\ +\"returned\":%d\ +},\n\ +" + , (long long)HDtime(NULL), (unsigned long)address, flags, type_id, + (int)size, (int)fxn_ret_value); + + /* Write the log message to the file */ + if(H5C_write_log_message(cache, msg) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__write_insert_entry_log_msg() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__write_mark_dirty_entry_log_msg + * + * Purpose: Write a log message for marking cache entries as dirty. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__write_mark_dirty_entry_log_msg(const H5AC_t *cache, + const H5AC_info_t *entry, + herr_t fxn_ret_value) +{ + char msg[MSG_SIZE]; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache); + HDassert(entry); + + /* Create the log message string */ + HDsnprintf(msg, MSG_SIZE, +"\ +{\ +\"timestamp\":%lld,\ +\"action\":\"dirty\",\ +\"address\":0x%lx,\ +\"returned\":%d\ +},\n\ +" + , (long long)HDtime(NULL), (unsigned long)entry->addr, (int)fxn_ret_value); + + /* Write the log message to the file */ + if(H5C_write_log_message(cache, msg) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__write_mark_dirty_entry_log_msg() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__write_mark_clean_entry_log_msg + * + * Purpose: Write a log message for marking cache entries as clean. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * Saturday, July 23, 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__write_mark_clean_entry_log_msg(const H5AC_t *cache, const H5AC_info_t *entry, + herr_t fxn_ret_value) +{ + char msg[MSG_SIZE]; /* Log message buffer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache); + HDassert(entry); + + /* Create the log message string */ + HDsnprintf(msg, MSG_SIZE, +"\ +{\ +\"timestamp\":%lld,\ +\"action\":\"clean\",\ +\"address\":0x%lx,\ +\"returned\":%d\ +},\n\ +" + , (long long)HDtime(NULL), (unsigned long)entry->addr, (int)fxn_ret_value); + + /* Write the log message to the file */ + if(H5C_write_log_message(cache, msg) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__write_mark_clean_entry_log_msg() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__write_move_entry_log_msg + * + * Purpose: Write a log message for moving a cache entry. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__write_move_entry_log_msg(const H5AC_t *cache, + haddr_t old_addr, + haddr_t new_addr, + int type_id, + herr_t fxn_ret_value) +{ + char msg[MSG_SIZE]; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache); + + /* Create the log message string */ + HDsnprintf(msg, MSG_SIZE, +"\ +{\ +\"timestamp\":%lld,\ +\"action\":\"move\",\ +\"old_address\":0x%lx,\ +\"new_address\":0x%lx,\ +\"type_id\":%d,\ +\"returned\":%d\ +},\n\ +" + , (long long)HDtime(NULL), (unsigned long)old_addr, + (unsigned long)new_addr, type_id, (int)fxn_ret_value); + + /* Write the log message to the file */ + if(H5C_write_log_message(cache, msg) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__write_move_entry_log_msg() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__write_pin_entry_log_msg + * + * Purpose: Write a log message for pinning a cache entry. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__write_pin_entry_log_msg(const H5AC_t *cache, + const H5AC_info_t *entry, + herr_t fxn_ret_value) +{ + char msg[MSG_SIZE]; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache); + HDassert(entry); + + /* Create the log message string */ + HDsnprintf(msg, MSG_SIZE, +"\ +{\ +\"timestamp\":%lld,\ +\"action\":\"pin\",\ +\"address\":0x%lx,\ +\"returned\":%d\ +},\n\ +" + , (long long)HDtime(NULL), (unsigned long)entry->addr, + (int)fxn_ret_value); + + /* Write the log message to the file */ + if(H5C_write_log_message(cache, msg) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__write_pin_entry_log_msg() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__write_create_fd_log_msg + * + * Purpose: Write a log message for creating a flush dependency between + * two cache entries. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__write_create_fd_log_msg(const H5AC_t *cache, + const H5AC_info_t *parent, + const H5AC_info_t *child, + herr_t fxn_ret_value) +{ + char msg[MSG_SIZE]; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache); + HDassert(parent); + HDassert(child); + + /* Create the log message string */ + HDsnprintf(msg, MSG_SIZE, +"\ +{\ +\"timestamp\":%lld,\ +\"action\":\"create_fd\",\ +\"parent_addr\":0x%lx,\ +\"child_addr\":0x%lx,\ +\"returned\":%d\ +},\n\ +" + , (long long)HDtime(NULL), (unsigned long)parent->addr, + (unsigned long)child->addr, (int)fxn_ret_value); + + /* Write the log message to the file */ + if(H5C_write_log_message(cache, msg) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__write_create_fd_log_msg() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__write_protect_entry_log_msg + * + * Purpose: Write a log message for protecting a cache entry. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__write_protect_entry_log_msg(const H5AC_t *cache, + const H5AC_info_t *entry, + unsigned flags, + herr_t fxn_ret_value) +{ + char msg[MSG_SIZE]; + char rw_s[16]; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache); + HDassert(entry); + + if(H5AC__READ_ONLY_FLAG == flags) + HDstrcpy(rw_s, "READ"); + else + HDstrcpy(rw_s, "WRITE"); + + /* Create the log message string */ + HDsnprintf(msg, MSG_SIZE, +"\ +{\ +\"timestamp\":%lld,\ +\"action\":\"protect\",\ +\"address\":0x%lx,\ +\"readwrite\":\"%s\",\ +\"size\":%d,\ +\"returned\":%d\ +},\n\ +" + , (long long)HDtime(NULL), (unsigned long)entry->addr, + rw_s, (int)entry->size, (int)fxn_ret_value); + + /* Write the log message to the file */ + if(H5C_write_log_message(cache, msg) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__write_protect_entry_log_msg() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__write_resize_entry_log_msg + * + * Purpose: Write a log message for resizing a cache entry. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__write_resize_entry_log_msg(const H5AC_t *cache, + const H5AC_info_t *entry, + size_t new_size, + herr_t fxn_ret_value) +{ + char msg[MSG_SIZE]; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache); + HDassert(entry); + + /* Create the log message string */ + HDsnprintf(msg, MSG_SIZE, +"\ +{\ +\"timestamp\":%lld,\ +\"action\":\"resize\",\ +\"address\":0x%lx,\ +\"new_size\":%d,\ +\"returned\":%d\ +},\n\ +" + , (long long)HDtime(NULL), (unsigned long)entry->addr, + (int)new_size, (int)fxn_ret_value); + + /* Write the log message to the file */ + if(H5C_write_log_message(cache, msg) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__write_resize_entry_log_msg() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__write_unpin_entry_log_msg + * + * Purpose: Write a log message for unpinning a cache entry. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__write_unpin_entry_log_msg(const H5AC_t *cache, + const H5AC_info_t *entry, + herr_t fxn_ret_value) +{ + char msg[MSG_SIZE]; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache); + HDassert(entry); + + /* Create the log message string */ + HDsnprintf(msg, MSG_SIZE, +"\ +{\ +\"timestamp\":%lld,\ +\"action\":\"unpin\",\ +\"address\":0x%lx,\ +\"returned\":%d\ +},\n\ +" + , (long long)HDtime(NULL), (unsigned long)entry->addr, + (int)fxn_ret_value); + + /* Write the log message to the file */ + if(H5C_write_log_message(cache, msg) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__write_unpin_entry_log_msg() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__write_destroy_fd_log_msg + * + * Purpose: Write a log message for destroying a flush dependency + * between two cache entries. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__write_destroy_fd_log_msg(const H5AC_t *cache, + const H5AC_info_t *parent, + const H5AC_info_t *child, + herr_t fxn_ret_value) +{ + char msg[MSG_SIZE]; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache); + HDassert(parent); + HDassert(child); + + /* Create the log message string */ + HDsnprintf(msg, MSG_SIZE, +"\ +{\ +\"timestamp\":%lld,\ +\"action\":\"destroy_fd\",\ +\"parent_addr\":0x%lx,\ +\"child_addr\":0x%lx,\ +\"returned\":%d\ +},\n\ +" + , (long long)HDtime(NULL), (unsigned long)parent->addr, + (unsigned long)child->addr, (int)fxn_ret_value); + + /* Write the log message to the file */ + if(H5C_write_log_message(cache, msg) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__write_destroy_fd_log_msg() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__write_unprotect_entry_log_msg + * + * Purpose: Write a log message for unprotecting a cache entry. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__write_unprotect_entry_log_msg(const H5AC_t *cache, + const H5AC_info_t *entry, + int type_id, + unsigned flags, + herr_t fxn_ret_value) +{ + char msg[MSG_SIZE]; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache); + HDassert(entry); + + /* Create the log message string */ + HDsnprintf(msg, MSG_SIZE, +"\ +{\ +\"timestamp\":%lld,\ +\"action\":\"unprotect\",\ +\"address\":0x%lx,\ +\"id\":%d,\ +\"flags\":%x,\ +\"returned\":%d\ +},\n\ +" + , (long long)HDtime(NULL), (unsigned long)entry->addr, + type_id, flags, (int)fxn_ret_value); + + HDsnprintf(msg, MSG_SIZE, " "); + + /* Write the log message to the file */ + if(H5C_write_log_message(cache, msg) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__write_unprotect_entry_log_msg() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__write_set_cache_config_log_msg + * + * Purpose: Write a log message for setting the cache configuration. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__write_set_cache_config_log_msg(const H5AC_t *cache, + const H5AC_cache_config_t *config, + herr_t fxn_ret_value) +{ + char msg[MSG_SIZE]; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache); + HDassert(config); + + /* Create the log message string */ + HDsnprintf(msg, MSG_SIZE, +"\ +{\ +\"timestamp\":%lld,\ +\"action\":\"set_config\",\ +\"returned\":%d\ +},\n\ +" + , (long long)HDtime(NULL), (int)fxn_ret_value); + + + /* Write the log message to the file */ + if(H5C_write_log_message(cache, msg) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__write_set_cache_config_log_msg() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__write_remove_entry_log_msg + * + * Purpose: Write a log message for removing a cache entry. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * September 17, 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__write_remove_entry_log_msg(const H5AC_t *cache, const H5AC_info_t *entry, + herr_t fxn_ret_value) +{ + char msg[MSG_SIZE]; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache); + HDassert(entry); + + /* Create the log message string */ + HDsnprintf(msg, MSG_SIZE, +"\ +{\ +\"timestamp\":%lld,\ +\"action\":\"remove\",\ +\"address\":0x%lx,\ +\"returned\":%d\ +},\n\ +" + , (long long)HDtime(NULL), (unsigned long)entry->addr, + (int)fxn_ret_value); + + /* Write the log message to the file */ + if(H5C_write_log_message(cache, msg) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unable to emit log message") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__write_remove_entry_log_msg() */ + diff --git a/src/H5ACmpio.c b/src/H5ACmpio.c index e33fc8e..570783a 100644 --- a/src/H5ACmpio.c +++ b/src/H5ACmpio.c @@ -730,12 +730,10 @@ H5AC__log_deleted_entry(const H5AC_info_t *entry_ptr) * * If mpi_rank is 0, we must first check to see if the entry * appears in the dirty entries slist. If it is, do nothing. - * If it isn't, add the size to th dirty_bytes count, add the + * If it isn't, add the size to the dirty_bytes count, add the * entry to the dirty entries slist, and remove it from the * cleaned list (if it is present there). * - * Return SUCCEED on success, and FAIL on failure. - * * Return: Non-negative on success/Negative on failure. * * Programmer: John Mainzer, 6/29/05 @@ -807,6 +805,63 @@ done: /*------------------------------------------------------------------------- * + * Function: H5AC__log_cleaned_entry() + * + * Purpose: Treat this operation as a 'clear' and remove the entry + * from both the cleaned and dirtied lists if it is present. + * Reduces the dirty_bytes count by the size of the entry. + * + * Return: Non-negative on success/Negative on failure. + * + * Programmer: Quincey Koziol + * 7/23/16 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__log_cleaned_entry(const H5AC_info_t *entry_ptr) +{ + H5AC_t * cache_ptr; + H5AC_aux_t * aux_ptr; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity check */ + HDassert(entry_ptr); + HDassert(entry_ptr->is_dirty == FALSE); + cache_ptr = entry_ptr->cache_ptr; + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + + if(aux_ptr->mpi_rank == 0) { + H5AC_slist_entry_t *slist_entry_ptr; + haddr_t addr = entry_ptr->addr; + + /* Sanity checks */ + HDassert(aux_ptr->d_slist_ptr != NULL); + HDassert(aux_ptr->c_slist_ptr != NULL); + + /* Remove it from both the cleaned list and the dirtied list. */ + if(NULL != (slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->c_slist_ptr, (void *)(&addr)))) + slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); + if(NULL != (slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->d_slist_ptr, (void *)(&addr)))) + slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); + + } /* end if */ + + /* Decrement the dirty byte count */ + aux_ptr->dirty_bytes -= entry_ptr->size; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__log_cleaned_entry() */ + + +/*------------------------------------------------------------------------- + * * Function: H5AC__log_flushed_entry() * * Purpose: Update the clean entry slist for the flush of an entry -- diff --git a/src/H5ACpkg.h b/src/H5ACpkg.h index 6b964d6..3900ff1 100644 --- a/src/H5ACpkg.h +++ b/src/H5ACpkg.h @@ -42,7 +42,6 @@ /* Get needed headers */ #include "H5Cprivate.h" /* Cache */ #include "H5FLprivate.h" /* Free Lists */ -#include "H5SLprivate.h" /* Skip lists */ /*****************************/ /* Package Private Variables */ @@ -413,6 +412,7 @@ typedef struct H5AC_aux_t /* Parallel I/O routines */ H5_DLL herr_t H5AC__log_deleted_entry(const H5AC_info_t *entry_ptr); H5_DLL herr_t H5AC__log_dirtied_entry(const H5AC_info_t *entry_ptr); +H5_DLL herr_t H5AC__log_cleaned_entry(const H5AC_info_t *entry_ptr); H5_DLL herr_t H5AC__log_flushed_entry(H5C_t *cache_ptr, haddr_t addr, hbool_t was_dirty, unsigned flags); H5_DLL herr_t H5AC__log_inserted_entry(const H5AC_info_t *entry_ptr); @@ -426,5 +426,70 @@ H5_DLL herr_t H5AC__set_write_done_callback(H5C_t * cache_ptr, void (* write_done)(void)); #endif /* H5_HAVE_PARALLEL */ +/* Trace file routines */ +H5_DLL herr_t H5AC__close_trace_file(H5AC_t *cache_ptr); +H5_DLL herr_t H5AC__open_trace_file(H5AC_t *cache_ptr, const char *trace_file_name); + +/* Cache logging routines */ +H5_DLL herr_t H5AC__write_create_cache_log_msg(H5AC_t *cache); +H5_DLL herr_t H5AC__write_destroy_cache_log_msg(H5AC_t *cache); +H5_DLL herr_t H5AC__write_evict_cache_log_msg(const H5AC_t *cache, + herr_t fxn_ret_value); +H5_DLL herr_t H5AC__write_expunge_entry_log_msg(const H5AC_t *cache, + haddr_t address, + int type_id, + herr_t fxn_ret_value); +H5_DLL herr_t H5AC__write_flush_cache_log_msg(const H5AC_t *cache, + herr_t fxn_ret_value); +H5_DLL herr_t H5AC__write_insert_entry_log_msg(const H5AC_t *cache, + haddr_t address, + int type_id, + unsigned flags, + size_t size, + herr_t fxn_ret_value); +H5_DLL herr_t H5AC__write_mark_dirty_entry_log_msg(const H5AC_t *cache, + const H5AC_info_t *entry, + herr_t fxn_ret_value); +H5_DLL herr_t H5AC__write_mark_clean_entry_log_msg(const H5AC_t *cache, + const H5AC_info_t *entry, herr_t fxn_ret_value); +H5_DLL herr_t H5AC__write_move_entry_log_msg(const H5AC_t *cache, + haddr_t old_addr, + haddr_t new_addr, + int type_id, + herr_t fxn_ret_value); +H5_DLL herr_t H5AC__write_pin_entry_log_msg(const H5AC_t *cache, + const H5AC_info_t *entry, + herr_t fxn_ret_value); +H5_DLL herr_t H5AC__write_create_fd_log_msg(const H5AC_t *cache, + const H5AC_info_t *parent, + const H5AC_info_t *child, + herr_t fxn_ret_value); +H5_DLL herr_t H5AC__write_protect_entry_log_msg(const H5AC_t *cache, + const H5AC_info_t *entry, + unsigned flags, + herr_t fxn_ret_value); +H5_DLL herr_t H5AC__write_resize_entry_log_msg(const H5AC_t *cache, + const H5AC_info_t *entry, + size_t new_size, + herr_t fxn_ret_value); +H5_DLL herr_t H5AC__write_unpin_entry_log_msg(const H5AC_t *cache, + const H5AC_info_t *entry, + herr_t fxn_ret_value); +H5_DLL herr_t H5AC__write_destroy_fd_log_msg(const H5AC_t *cache, + const H5AC_info_t *parent, + const H5AC_info_t *child, + herr_t fxn_ret_value); +H5_DLL herr_t H5AC__write_unprotect_entry_log_msg(const H5AC_t *cache, + const H5AC_info_t *entry, + int type_id, + unsigned flags, + herr_t fxn_ret_value); +H5_DLL herr_t H5AC__write_set_cache_config_log_msg(const H5AC_t *cache, + const H5AC_cache_config_t *config, + herr_t fxn_ret_value); +H5_DLL herr_t H5AC__write_remove_entry_log_msg(const H5AC_t *cache, + const H5AC_info_t *entry, + herr_t fxn_ret_value); + #endif /* _H5ACpkg_H */ diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index d1d41c9..48b7c6b 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -35,6 +35,7 @@ #include "H5Cprivate.h" /* Cache */ #include "H5Fprivate.h" /* File access */ #include "H5Pprivate.h" /* Property lists */ +#include "H5SLprivate.h" /* Skip lists */ #ifdef H5_METADATA_TRACE_FILE #define H5AC__TRACE_FILE_ENABLED 1 @@ -45,43 +46,49 @@ /* Global metadata tag values */ #define H5AC__INVALID_TAG (haddr_t)0 #define H5AC__IGNORE_TAG (haddr_t)1 -#define H5AC__SUPERBLOCK_TAG (haddr_t)2 -#define H5AC__FREESPACE_TAG (haddr_t)3 -#define H5AC__SOHM_TAG (haddr_t)4 -#define H5AC__GLOBALHEAP_TAG (haddr_t)5 -#define H5AC__COPIED_TAG (haddr_t)6 +#define H5AC__COPIED_TAG (haddr_t)2 +#define H5AC__SUPERBLOCK_TAG (haddr_t)3 +#define H5AC__FREESPACE_TAG (haddr_t)4 +#define H5AC__SOHM_TAG (haddr_t)5 +#define H5AC__GLOBALHEAP_TAG (haddr_t)6 + +/* Definitions for cache "tag" property */ +#define H5AC_TAG_NAME "H5AC_tag" +#define H5AC_TAG_SIZE sizeof(haddr_t) +#define H5AC_TAG_DEF (H5AC__INVALID_TAG) /* Types of metadata objects cached */ typedef enum { - H5AC_BT_ID = 0, /* ( 0) B-tree nodes */ - H5AC_SNODE_ID, /* ( 1) symbol table nodes */ - H5AC_LHEAP_PRFX_ID, /* ( 2) local heap prefix */ - H5AC_LHEAP_DBLK_ID, /* ( 3) local heap data block */ - H5AC_GHEAP_ID, /* ( 4) global heap */ - H5AC_OHDR_ID, /* ( 5) object header */ - H5AC_OHDR_CHK_ID, /* ( 6) object header chunk */ - H5AC_BT2_HDR_ID, /* ( 7) v2 B-tree header */ - H5AC_BT2_INT_ID, /* ( 8) v2 B-tree internal node */ - H5AC_BT2_LEAF_ID, /* ( 9) v2 B-tree leaf node */ - H5AC_FHEAP_HDR_ID, /* (10) fractal heap header */ - H5AC_FHEAP_DBLOCK_ID, /* (11) fractal heap direct block */ - H5AC_FHEAP_IBLOCK_ID, /* (12) fractal heap indirect block */ - H5AC_FSPACE_HDR_ID, /* (13) free space header */ - H5AC_FSPACE_SINFO_ID, /* (14) free space sections */ - H5AC_SOHM_TABLE_ID, /* (15) shared object header message master table */ - H5AC_SOHM_LIST_ID, /* (16) shared message index stored as a list */ - H5AC_EARRAY_HDR_ID, /* (17) extensible array header */ - H5AC_EARRAY_IBLOCK_ID, /* (18) extensible array index block */ - H5AC_EARRAY_SBLOCK_ID, /* (19) extensible array super block */ - H5AC_EARRAY_DBLOCK_ID, /* (20) extensible array data block */ - H5AC_EARRAY_DBLK_PAGE_ID, /* (21) extensible array data block page */ - H5AC_FARRAY_HDR_ID, /* (22) fixed array header */ - H5AC_FARRAY_DBLOCK_ID, /* (23) fixed array data block */ - H5AC_FARRAY_DBLK_PAGE_ID, /* (24) fixed array data block page */ - H5AC_SUPERBLOCK_ID, /* (25) file superblock */ - H5AC_DRVRINFO_ID, /* (26) driver info block (supplements superblock) */ - H5AC_TEST_ID, /* (27) test entry -- not used for actual files */ - H5AC_NTYPES /* Number of types, must be last */ + H5AC_BT_ID = 0, /* ( 0) B-tree nodes */ + H5AC_SNODE_ID, /* ( 1) symbol table nodes */ + H5AC_LHEAP_PRFX_ID, /* ( 2) local heap prefix */ + H5AC_LHEAP_DBLK_ID, /* ( 3) local heap data block */ + H5AC_GHEAP_ID, /* ( 4) global heap */ + H5AC_OHDR_ID, /* ( 5) object header */ + H5AC_OHDR_CHK_ID, /* ( 6) object header chunk */ + H5AC_BT2_HDR_ID, /* ( 7) v2 B-tree header */ + H5AC_BT2_INT_ID, /* ( 8) v2 B-tree internal node */ + H5AC_BT2_LEAF_ID, /* ( 9) v2 B-tree leaf node */ + H5AC_FHEAP_HDR_ID, /* (10) fractal heap header */ + H5AC_FHEAP_DBLOCK_ID, /* (11) fractal heap direct block */ + H5AC_FHEAP_IBLOCK_ID, /* (12) fractal heap indirect block */ + H5AC_FSPACE_HDR_ID, /* (13) free space header */ + H5AC_FSPACE_SINFO_ID, /* (14) free space sections */ + H5AC_SOHM_TABLE_ID, /* (15) shared object header message master table */ + H5AC_SOHM_LIST_ID, /* (16) shared message index stored as a list */ + H5AC_EARRAY_HDR_ID, /* (17) extensible array header */ + H5AC_EARRAY_IBLOCK_ID, /* (18) extensible array index block */ + H5AC_EARRAY_SBLOCK_ID, /* (19) extensible array super block */ + H5AC_EARRAY_DBLOCK_ID, /* (20) extensible array data block */ + H5AC_EARRAY_DBLK_PAGE_ID, /* (21) extensible array data block page */ + H5AC_FARRAY_HDR_ID, /* (22) fixed array header */ + H5AC_FARRAY_DBLOCK_ID, /* (23) fixed array data block */ + H5AC_FARRAY_DBLK_PAGE_ID, /* (24) fixed array data block page */ + H5AC_SUPERBLOCK_ID, /* (25) file superblock */ + H5AC_DRVRINFO_ID, /* (26) driver info block (supplements superblock)*/ + H5AC_PROXY_ENTRY_ID, /* (27) cache entry proxy */ + H5AC_TEST_ID, /* (28) test entry -- not used for actual files */ + H5AC_NTYPES /* Number of types, must be last */ } H5AC_type_t; /* H5AC_DUMP_STATS_ON_CLOSE should always be FALSE when @@ -136,7 +143,6 @@ typedef enum { #define H5AC__SERIALIZE_RESIZED_FLAG H5C__SERIALIZE_RESIZED_FLAG #define H5AC__SERIALIZE_MOVED_FLAG H5C__SERIALIZE_MOVED_FLAG -#define H5AC__SERIALIZE_COMPRESSED_FLAG H5C__SERIALIZE_COMPRESSED_FLAG /* Cork actions: cork/uncork/get cork status of an object */ #define H5AC__SET_CORK H5C__SET_CORK @@ -158,17 +164,21 @@ typedef H5C_notify_action_t H5AC_notify_action_t; #define H5AC_NOTIFY_ACTION_AFTER_LOAD H5C_NOTIFY_ACTION_AFTER_LOAD #define H5AC_NOTIFY_ACTION_AFTER_FLUSH H5C_NOTIFY_ACTION_AFTER_FLUSH #define H5AC_NOTIFY_ACTION_BEFORE_EVICT H5C_NOTIFY_ACTION_BEFORE_EVICT +#define H5AC_NOTIFY_ACTION_ENTRY_DIRTIED H5C_NOTIFY_ACTION_ENTRY_DIRTIED +#define H5AC_NOTIFY_ACTION_ENTRY_CLEANED H5C_NOTIFY_ACTION_ENTRY_CLEANED +#define H5AC_NOTIFY_ACTION_CHILD_DIRTIED H5C_NOTIFY_ACTION_CHILD_DIRTIED +#define H5AC_NOTIFY_ACTION_CHILD_CLEANED H5C_NOTIFY_ACTION_CHILD_CLEANED #define H5AC__CLASS_NO_FLAGS_SET H5C__CLASS_NO_FLAGS_SET #define H5AC__CLASS_SPECULATIVE_LOAD_FLAG H5C__CLASS_SPECULATIVE_LOAD_FLAG -#define H5AC__CLASS_COMPRESSED_FLAG H5C__CLASS_COMPRESSED_FLAG /* The following flags should only appear in test code */ -#define H5AC__CLASS_NO_IO_FLAG H5C__CLASS_NO_IO_FLAG #define H5AC__CLASS_SKIP_READS H5C__CLASS_SKIP_READS #define H5AC__CLASS_SKIP_WRITES H5C__CLASS_SKIP_WRITES -typedef H5C_get_load_size_func_t H5AC_get_load_size_func_t; +typedef H5C_get_initial_load_size_func_t H5AC_get_initial_load_size_func_t; +typedef H5C_get_final_load_size_func_t H5AC_get_final_load_size_func_t; +typedef H5C_verify_chksum_func_t H5AC_verify_chksum_func_t; typedef H5C_deserialize_func_t H5AC_deserialize_func_t; typedef H5C_image_len_func_t H5AC_image_len_func_t; @@ -180,7 +190,6 @@ typedef H5C_pre_serialize_func_t H5AC_pre_serialize_func_t; typedef H5C_serialize_func_t H5AC_serialize_func_t; typedef H5C_notify_func_t H5AC_notify_func_t; typedef H5C_free_icr_func_t H5AC_free_icr_func_t; -typedef H5C_clear_func_t H5AC_clear_func_t; typedef H5C_get_fsf_size_t H5AC_get_fsf_size_t; typedef H5C_class_t H5AC_class_t; @@ -188,10 +197,28 @@ typedef H5C_class_t H5AC_class_t; /* Cache entry info */ typedef H5C_cache_entry_t H5AC_info_t; - /* Typedef for metadata cache (defined in H5Cpkg.h) */ typedef H5C_t H5AC_t; +/* Metadata cache proxy entry type */ +typedef struct H5AC_proxy_entry_t { + H5AC_info_t cache_info; /* Information for H5AC cache functions */ + /* (MUST be first field in structure) */ + + /* General fields */ + haddr_t addr; /* Address of the entry in the file */ + /* (Should be in 'temporary' address space) */ + + /* Parent fields */ + H5SL_t *parents; /* Skip list to track parent addresses */ + + /* Child fields */ + size_t nchildren; /* Number of children */ + size_t ndirty_children; /* Number of dirty children */ + /* (Note that this currently duplicates some cache functionality) */ +} H5AC_proxy_entry_t; + + #define H5AC_RING_NAME "H5AC_ring_type" /* Dataset transfer property lists for metadata calls */ @@ -346,11 +373,14 @@ H5_DLL herr_t H5AC_unprotect(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, void *thing, unsigned flags); H5_DLL herr_t H5AC_flush(H5F_t *f, hid_t dxpl_id); H5_DLL herr_t H5AC_mark_entry_dirty(void *thing); +H5_DLL herr_t H5AC_mark_entry_clean(void *thing); H5_DLL herr_t H5AC_move_entry(H5F_t *f, const H5AC_class_t *type, haddr_t old_addr, haddr_t new_addr, hid_t dxpl_id); H5_DLL herr_t H5AC_dest(H5F_t *f, hid_t dxpl_id); +H5_DLL herr_t H5AC_evict(H5F_t *f, hid_t dxpl_id); H5_DLL herr_t H5AC_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, unsigned flags); +H5_DLL herr_t H5AC_remove_entry(void *entry); H5_DLL herr_t H5AC_get_cache_auto_resize_config(const H5AC_t * cache_ptr, H5AC_cache_config_t *config_ptr); H5_DLL herr_t H5AC_get_cache_size(H5AC_t *cache_ptr, size_t *max_size_ptr, @@ -364,7 +394,7 @@ H5_DLL herr_t H5AC_validate_config(H5AC_cache_config_t *config_ptr); /* Tag & Ring routines */ H5_DLL herr_t H5AC_tag(hid_t dxpl_id, haddr_t metadata_tag, haddr_t *prev_tag); H5_DLL herr_t H5AC_flush_tagged_metadata(H5F_t * f, haddr_t metadata_tag, hid_t dxpl_id); -H5_DLL herr_t H5AC_evict_tagged_metadata(H5F_t * f, haddr_t metadata_tag, hid_t dxpl_id); +H5_DLL herr_t H5AC_evict_tagged_metadata(H5F_t * f, haddr_t metadata_tag, hbool_t match_global, hid_t dxpl_id); H5_DLL herr_t H5AC_retag_copied_metadata(const H5F_t *f, haddr_t metadata_tag); H5_DLL herr_t H5AC_ignore_tags(const H5F_t *f); H5_DLL herr_t H5AC_cork(H5F_t *f, haddr_t obj_addr, unsigned action, hbool_t *corked); @@ -373,15 +403,24 @@ H5_DLL herr_t H5AC_set_ring(hid_t dxpl_id, H5AC_ring_t ring, H5P_genplist_t **dx H5AC_ring_t *orig_ring); H5_DLL herr_t H5AC_reset_ring(H5P_genplist_t *dxpl, H5AC_ring_t orig_ring); H5_DLL herr_t H5AC_expunge_tag_type_metadata(H5F_t *f, hid_t dxpl_id, haddr_t tag, int type_id, unsigned flags); +H5_DLL herr_t H5AC_get_tag(const void *thing, /*OUT*/ haddr_t *tag); + +/* Virtual entry routines */ +H5_DLL H5AC_proxy_entry_t *H5AC_proxy_entry_create(void); +H5_DLL herr_t H5AC_proxy_entry_add_parent(H5AC_proxy_entry_t *pentry, void *parent); +H5_DLL herr_t H5AC_proxy_entry_remove_parent(H5AC_proxy_entry_t *pentry, void *parent); +H5_DLL herr_t H5AC_proxy_entry_add_child(H5AC_proxy_entry_t *pentry, H5F_t *f, + hid_t dxpl_id, void *child); +H5_DLL herr_t H5AC_proxy_entry_remove_child(H5AC_proxy_entry_t *pentry, void *child); +H5_DLL herr_t H5AC_proxy_entry_dest(H5AC_proxy_entry_t *pentry); #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5AC_add_candidate(H5AC_t * cache_ptr, haddr_t addr); #endif /* H5_HAVE_PARALLEL */ -#ifndef NDEBUG /* debugging functions */ +/* Debugging functions */ H5_DLL herr_t H5AC_stats(const H5F_t *f); H5_DLL herr_t H5AC_dump_cache(const H5F_t *f); -#endif /* NDEBUG */ /* end debugging functions */ #endif /* !_H5ACprivate_H */ diff --git a/src/H5ACproxy_entry.c b/src/H5ACproxy_entry.c new file mode 100644 index 0000000..66aacb3 --- /dev/null +++ b/src/H5ACproxy_entry.c @@ -0,0 +1,634 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5ACproxy_entry.c + * + * Purpose: Functions and a cache client for a "proxy" cache entry. + * A proxy cache entry is used as a placeholder for entire + * data structures to attach flush dependencies, etc. + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ +#include "H5ACmodule.h" /* This source code file is part of the H5AC module */ + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5ACpkg.h" /* Metadata cache */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5MFprivate.h" /* File memory management */ + + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + +/* Metadata cache (H5AC) callbacks */ +static herr_t H5AC__proxy_entry_image_len(const void *thing, size_t *image_len); +static herr_t H5AC__proxy_entry_serialize(const H5F_t *f, void *image_ptr, + size_t len, void *thing); +static herr_t H5AC__proxy_entry_notify(H5AC_notify_action_t action, void *thing); +static herr_t H5AC__proxy_entry_free_icr(void *thing); + +/*********************/ +/* Package Variables */ +/*********************/ + +/* H5AC proxy entries inherit cache-like properties from H5AC */ +const H5AC_class_t H5AC_PROXY_ENTRY[1] = {{ + H5AC_PROXY_ENTRY_ID, /* Metadata client ID */ + "Proxy entry", /* Metadata client name (for debugging) */ + H5FD_MEM_SUPER, /* File space memory type for client */ + 0, /* Client class behavior flags */ + NULL, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + NULL, /* 'verify_chksum' callback */ + NULL, /* 'deserialize' callback */ + H5AC__proxy_entry_image_len, /* 'image_len' callback */ + NULL, /* 'pre_serialize' callback */ + H5AC__proxy_entry_serialize, /* 'serialize' callback */ + H5AC__proxy_entry_notify, /* 'notify' callback */ + H5AC__proxy_entry_free_icr, /* 'free_icr' callback */ + NULL, /* 'fsf_size' callback */ +}}; + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Declare a free list to manage H5AC_proxy_entry_t objects */ +H5FL_DEFINE_STATIC(H5AC_proxy_entry_t); + + + +/*------------------------------------------------------------------------- + * Function: H5AC_proxy_entry_create + * + * Purpose: Create a new proxy entry + * + * Return: Success: Pointer to the new proxy entry object. + * Failure: NULL + * + * Programmer: Quincey Koziol + * September 17, 2016 + * + *------------------------------------------------------------------------- + */ +H5AC_proxy_entry_t * +H5AC_proxy_entry_create(void) +{ + H5AC_proxy_entry_t *pentry = NULL; /* Pointer to new proxy entry */ + H5AC_proxy_entry_t *ret_value = NULL; /* Return value */ + + FUNC_ENTER_NOAPI(NULL) + + /* Allocate new proxy entry */ + if(NULL == (pentry = H5FL_CALLOC(H5AC_proxy_entry_t))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, NULL, "can't allocate proxy entry") + + /* Set non-zero fields */ + pentry->addr = HADDR_UNDEF; + + /* Set return value */ + ret_value = pentry; + +done: + /* Release resources on error */ + if(!ret_value) + if(pentry) + pentry = H5FL_FREE(H5AC_proxy_entry_t, pentry); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5AC_proxy_entry_create() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC_proxy_entry_add_parent + * + * Purpose: Add a parent to a proxy entry + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * September 17, 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_proxy_entry_add_parent(H5AC_proxy_entry_t *pentry, void *_parent) +{ + H5AC_info_t *parent = (H5AC_info_t *)_parent; /* Parent entry's cache info */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(parent); + HDassert(pentry); + + /* Add parent to the list of parents */ + if(NULL == pentry->parents) + if(NULL == (pentry->parents = H5SL_create(H5SL_TYPE_HADDR, NULL))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, FAIL, "unable to create skip list for parents of proxy entry") + + /* Insert parent address into skip list */ + if(H5SL_insert(pentry->parents, parent, &parent->addr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "unable to insert parent into proxy's skip list") + + /* Add flush dependency on parent */ + if(pentry->nchildren > 0) { + /* Sanity check */ + HDassert(H5F_addr_defined(pentry->addr)); + + if(H5AC_create_flush_dependency(parent, pentry) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTDEPEND, FAIL, "unable to set flush dependency on proxy entry") + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5AC_proxy_entry_add_parent() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC_proxy_entry_remove_parent + * + * Purpose: Removes a parent from a proxy entry + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * September 17, 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_proxy_entry_remove_parent(H5AC_proxy_entry_t *pentry, void *_parent) +{ + H5AC_info_t *parent = (H5AC_info_t *)_parent; /* Pointer to the parent entry */ + H5AC_info_t *rem_parent; /* Pointer to the removed parent entry */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(pentry); + HDassert(pentry->parents); + HDassert(parent); + + /* Remove parent from skip list */ + if(NULL == (rem_parent = (H5AC_info_t *)H5SL_remove(pentry->parents, &parent->addr))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "unable to remove proxy entry parent from skip list") + if(!H5F_addr_eq(rem_parent->addr, parent->addr)) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "removed proxy entry parent not the same as real parent") + + /* Shut down the skip list, if this is the last parent */ + if(0 == H5SL_count(pentry->parents)) { + /* Sanity check */ + HDassert(0 == pentry->nchildren); + + if(H5SL_close(pentry->parents) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CLOSEERROR, FAIL, "can't close proxy parent skip list") + pentry->parents = NULL; + } /* end if */ + + /* Remove flush dependency between the proxy entry and a parent */ + if(pentry->nchildren > 0) + if(H5AC_destroy_flush_dependency(parent, pentry) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTUNDEPEND, FAIL, "unable to remove flush dependency on proxy entry") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5AC_proxy_entry_remove_parent() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__proxy_entry_add_child_cb + * + * Purpose: Callback routine for adding an entry as a flush dependency for + * a proxy entry. + * + * Return: Success: Non-negative on success + * Failure: Negative + * + * Programmer: Quincey Koziol + * Thursday, September 22, 2016 + * + *------------------------------------------------------------------------- + */ +static int +H5AC__proxy_entry_add_child_cb(void *_item, void H5_ATTR_UNUSED *_key, void *_udata) +{ + H5AC_info_t *parent = (H5AC_info_t *)_item; /* Pointer to the parent entry */ + H5AC_proxy_entry_t *pentry = (H5AC_proxy_entry_t *)_udata; /* Pointer to the proxy entry */ + int ret_value = H5_ITER_CONT; /* Callback return value */ + + FUNC_ENTER_STATIC + + /* Add flush dependency on parent for proxy entry */ + if(H5AC_create_flush_dependency(parent, pentry) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTDEPEND, H5_ITER_ERROR, "unable to set flush dependency for virtual entry") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5AC__proxy_entry_add_child_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC_proxy_entry_add_child + * + * Purpose: Add a child a proxy entry + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * September 17, 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_proxy_entry_add_child(H5AC_proxy_entry_t *pentry, H5F_t *f, hid_t dxpl_id, + void *child) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(pentry); + HDassert(child); + + /* Check for first child */ + if(0 == pentry->nchildren) { + /* Get an address, if the proxy doesn't already have one */ + if(!H5F_addr_defined(pentry->addr)) + if(HADDR_UNDEF == (pentry->addr = H5MF_alloc_tmp(f, 1))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "temporary file space allocation failed for proxy entry") + + /* Insert the proxy entry into the cache */ + if(H5AC_insert_entry(f, dxpl_id, H5AC_PROXY_ENTRY, pentry->addr, pentry, H5AC__PIN_ENTRY_FLAG) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "unable to cache proxy entry") + + /* Proxies start out clean (insertions are automatically marked dirty) */ + if(H5AC_mark_entry_clean(pentry) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTCLEAN, FAIL, "can't mark proxy entry clean") + + /* If there are currently parents, iterate over the list of parents, creating flush dependency on them */ + if(pentry->parents) + if(H5SL_iterate(pentry->parents, H5AC__proxy_entry_add_child_cb, pentry) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_BADITER, FAIL, "can't visit parents") + } /* end if */ + + /* Add flush dependency on proxy entry */ + if(H5AC_create_flush_dependency(pentry, child) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTDEPEND, FAIL, "unable to set flush dependency on proxy entry") + + /* Increment count of children */ + pentry->nchildren++; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5AC_proxy_entry_add_child() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__proxy_entry_remove_child_cb + * + * Purpose: Callback routine for removing an entry as a flush dependency for + * proxy entry. + * + * Return: Success: Non-negative on success + * Failure: Negative + * + * Programmer: Quincey Koziol + * Thursday, September 22, 2016 + * + *------------------------------------------------------------------------- + */ +static int +H5AC__proxy_entry_remove_child_cb(void *_item, void H5_ATTR_UNUSED *_key, void *_udata) +{ + H5AC_info_t *parent = (H5AC_info_t *)_item; /* Pointer to the parent entry */ + H5AC_proxy_entry_t *pentry = (H5AC_proxy_entry_t *)_udata; /* Pointer to the proxy entry */ + int ret_value = H5_ITER_CONT; /* Callback return value */ + + FUNC_ENTER_STATIC + + /* Remove flush dependency on parent for proxy entry */ + if(H5AC_destroy_flush_dependency(parent, pentry) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTUNDEPEND, H5_ITER_ERROR, "unable to remove flush dependency for proxy entry") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5AC__proxy_entry_remove_child_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC_proxy_entry_remove_child + * + * Purpose: Remove a child a proxy entry + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * September 17, 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_proxy_entry_remove_child(H5AC_proxy_entry_t *pentry, void *child) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(pentry); + HDassert(child); + + /* Remove flush dependency on proxy entry */ + if(H5AC_destroy_flush_dependency(pentry, child) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTUNDEPEND, FAIL, "unable to remove flush dependency on proxy entry") + + /* Decrement count of children */ + pentry->nchildren--; + + /* Check for last child */ + if(0 == pentry->nchildren) { + /* Check for flush dependencies on proxy's parents */ + if(pentry->parents) + /* Iterate over the list of parents, removing flush dependency on them */ + if(H5SL_iterate(pentry->parents, H5AC__proxy_entry_remove_child_cb, pentry) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_BADITER, FAIL, "can't visit parents") + + /* Unpin proxy */ + if(H5AC_unpin_entry(pentry) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPIN, FAIL, "can't unpin proxy entry") + + /* Remove proxy entry from cache */ + if(H5AC_remove_entry(pentry) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "unable to remove proxy entry") + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5AC_proxy_entry_remove_child() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC_proxy_entry_dest + * + * Purpose: Destroys a proxy entry in memory. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * September 17, 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_proxy_entry_dest(H5AC_proxy_entry_t *pentry) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(pentry); + HDassert(NULL == pentry->parents); + HDassert(0 == pentry->nchildren); + HDassert(0 == pentry->ndirty_children); + + /* Free the proxy entry object */ + pentry = H5FL_FREE(H5AC_proxy_entry_t, pentry); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5AC_proxy_entry_dest() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__proxy_entry_image_len + * + * Purpose: Compute the size of the data structure on disk. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * September 17, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__proxy_entry_image_len(const void H5_ATTR_UNUSED *thing, size_t *image_len) +{ + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image_len); + + /* Set the image length size to 1 byte */ + *image_len = 1; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5AC__proxy_entry_image_len() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__proxy_entry_serialize + * + * Purpose: Serializes a data structure for writing to disk. + * + * Note: Should never be invoked. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * September 17, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__proxy_entry_serialize(const H5F_t H5_ATTR_UNUSED *f, void H5_ATTR_UNUSED *image, + size_t H5_ATTR_UNUSED len, void H5_ATTR_UNUSED *thing) +{ + FUNC_ENTER_STATIC_NOERR /* Yes, even though this pushes an error on the stack */ + + /* Should never be invoked */ + HDassert(0 && "Invalid callback?!?"); + + HERROR(H5E_CACHE, H5E_CANTSERIALIZE, "called unreachable fcn."); + + FUNC_LEAVE_NOAPI(FAIL) +} /* end H5AC__proxy_entry_serialize() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__proxy_entry_notify + * + * Purpose: Handle cache action notifications + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * September 17, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__proxy_entry_notify(H5AC_notify_action_t action, void *_thing) +{ + H5AC_proxy_entry_t *pentry = (H5AC_proxy_entry_t *)_thing; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(pentry); + + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + break; + + case H5AC_NOTIFY_ACTION_AFTER_LOAD: +#ifdef NDEBUG + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "invalid notify action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Invalid action?!?"); +#endif /* NDEBUG */ + break; + + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: +#ifdef NDEBUG + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "invalid notify action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Invalid action?!?"); +#endif /* NDEBUG */ + break; + + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + /* Sanity checks */ + HDassert(0 == pentry->ndirty_children); + + /* No action */ + break; + + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + /* Sanity checks */ + HDassert(pentry->ndirty_children > 0); + + /* No action */ + break; + + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + /* Sanity checks */ + HDassert(0 == pentry->ndirty_children); + + /* No action */ + break; + + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + /* Increment # of dirty children */ + pentry->ndirty_children++; + + /* Check for first dirty child */ + if(1 == pentry->ndirty_children) + if(H5AC_mark_entry_dirty(pentry) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTDIRTY, FAIL, "can't mark proxy entry dirty") + break; + + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + /* Sanity check */ + HDassert(pentry->ndirty_children > 0); + + /* Decrement # of dirty children */ + pentry->ndirty_children--; + + /* Check for last dirty child */ + if(0 == pentry->ndirty_children) + if(H5AC_mark_entry_clean(pentry) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTCLEAN, FAIL, "can't mark proxy entry clean") + break; + + default: +#ifdef NDEBUG + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "unknown notify action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Unknown action?!?"); +#endif /* NDEBUG */ + } /* end switch */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5AC__proxy_entry_notify() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__proxy_entry_free_icr + * + * Purpose: Destroy/release an "in core representation" of a data + * structure + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * September 17, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__proxy_entry_free_icr(void *_thing) +{ + H5AC_proxy_entry_t *pentry = (H5AC_proxy_entry_t *)_thing; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Destroy the proxy entry */ + if(H5AC_proxy_entry_dest(pentry) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "unable to destroy proxy entry") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__proxy_entry_free_icr() */ + diff --git a/src/H5ACpublic.h b/src/H5ACpublic.h index 4e5502d..dd16764 100644 --- a/src/H5ACpublic.h +++ b/src/H5ACpublic.h @@ -78,6 +78,8 @@ extern "C" { * open_trace_file: Boolean field indicating whether the trace_file_name * field should be used to open a trace file for the cache. * + * *** DEPRECATED *** Use H5Fstart/stop logging functions instead + * * The trace file is a debuging feature that allow the capture of * top level metadata cache requests for purposes of debugging and/or * optimization. This field should normally be set to FALSE, as @@ -91,6 +93,8 @@ extern "C" { * close_trace_file: Boolean field indicating whether the current trace * file (if any) should be closed. * + * *** DEPRECATED *** Use H5Fstart/stop logging functions instead + * * See the above comments on the open_trace_file field. This field * should be set to FALSE unless there is an open trace file on the * cache that you wish to close. @@ -98,6 +102,8 @@ extern "C" { * trace_file_name: Full path of the trace file to be opened if the * open_trace_file field is TRUE. * + * *** DEPRECATED *** Use H5Fstart/stop logging functions instead + * * In the parallel case, an ascii representation of the mpi rank of * the process will be appended to the file name to yield a unique * trace file name for each process. diff --git a/src/H5Aint.c b/src/H5Aint.c index 66af0ff..e31cd83 100644 --- a/src/H5Aint.c +++ b/src/H5Aint.c @@ -1050,7 +1050,7 @@ H5A_close(H5A_t *attr) HDassert(attr->shared); /* Close the object's symbol-table entry */ - if(attr->obj_opened && (H5O_close(&(attr->oloc)) < 0)) + if(attr->obj_opened && (H5O_close(&(attr->oloc), NULL) < 0)) HGOTO_ERROR(H5E_ATTR, H5E_CANTRELEASE, FAIL, "can't release object header info") /* Reference count can be 0. It only happens when H5A_create fails. */ @@ -290,7 +290,7 @@ H5B2_insert(H5B2_t *bt2, hid_t dxpl_id, void *udata) hdr = bt2->hdr; /* Insert the record */ - if(H5B2__insert_hdr(hdr, dxpl_id, udata) < 0) + if(H5B2__insert(hdr, dxpl_id, udata) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, FAIL, "unable to insert record into B-tree") done: @@ -336,27 +336,37 @@ H5B2_update(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_modify_t op, void *op_ /* Check if the root node is allocated yet */ if(!H5F_addr_defined(hdr->root.addr)) { /* Create root node as leaf node in B-tree */ - if(H5B2__create_leaf(hdr, dxpl_id, &(hdr->root)) < 0) + if(H5B2__create_leaf(hdr, dxpl_id, hdr, &(hdr->root)) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, FAIL, "unable to create root node") } /* end if */ /* Attempt to insert record into B-tree */ if(hdr->depth > 0) { - if(H5B2__update_internal(hdr, dxpl_id, hdr->depth, NULL, &hdr->root, &status, H5B2_POS_ROOT, udata, op, op_data) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, FAIL, "unable to update record in B-tree internal node") + if(H5B2__update_internal(hdr, dxpl_id, hdr->depth, NULL, &hdr->root, &status, H5B2_POS_ROOT, hdr, udata, op, op_data) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update record in B-tree internal node") } /* end if */ else { - if(H5B2__update_leaf(hdr, dxpl_id, &hdr->root, &status, H5B2_POS_ROOT, udata, op, op_data) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, FAIL, "unable to update record in B-tree leaf node") + if(H5B2__update_leaf(hdr, dxpl_id, &hdr->root, &status, H5B2_POS_ROOT, hdr, udata, op, op_data) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update record in B-tree leaf node") } /* end else */ /* Sanity check */ HDassert(H5B2_UPDATE_UNKNOWN != status); /* Use insert algorithm if nodes to leaf full */ - if(H5B2_UPDATE_INSERT_CHILD_FULL == status) - if(H5B2__insert_hdr(hdr, dxpl_id, udata) < 0) + if(H5B2_UPDATE_INSERT_CHILD_FULL == status) { + if(H5B2__insert(hdr, dxpl_id, udata) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, FAIL, "unable to insert record into B-tree") + } /* end if */ + else if(H5B2_UPDATE_SHADOW_DONE == status || H5B2_UPDATE_INSERT_DONE == status) { + /* Mark B-tree header as dirty */ + if(H5B2__hdr_dirty(hdr) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTMARKDIRTY, FAIL, "unable to mark B-tree header dirty") + } /* end else-if */ + else { + /* Sanity check */ + HDassert(H5B2_UPDATE_MODIFY_DONE == status); + } /* end else */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -432,7 +442,7 @@ H5B2_iterate(H5B2_t *bt2, hid_t dxpl_id, H5B2_operator_t op, void *op_data) /* Iterate through records */ if(hdr->root.node_nrec > 0) { /* Iterate through nodes */ - if((ret_value = H5B2__iterate_node(hdr, dxpl_id, hdr->depth, &hdr->root, op, op_data)) < 0) + if((ret_value = H5B2__iterate_node(hdr, dxpl_id, hdr->depth, &hdr->root, hdr, op, op_data)) < 0) HERROR(H5E_BTREE, H5E_CANTLIST, "node iteration failed"); } /* end if */ @@ -469,6 +479,7 @@ H5B2_find(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_found_t op, { H5B2_hdr_t *hdr; /* Pointer to the B-tree header */ H5B2_node_ptr_t curr_node_ptr; /* Node pointer info for current node */ + void *parent = NULL; /* Parent of current node */ uint16_t depth; /* Current depth of the tree */ int cmp; /* Comparison value of records */ unsigned idx; /* Location of record which matches key */ @@ -522,6 +533,10 @@ H5B2_find(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_found_t op, /* Current depth of the tree */ depth = hdr->depth; + /* Set initial parent, if doing swmr writes */ + if(hdr->swmr_write) + parent = hdr; + /* Walk down B-tree to find record or leaf node where record is located */ cmp = -1; curr_pos = H5B2_POS_ROOT; @@ -530,12 +545,18 @@ H5B2_find(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_found_t op, H5B2_node_ptr_t next_node_ptr; /* Node pointer info for next node */ /* Lock B-tree current node */ - if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, curr_node_ptr.addr, curr_node_ptr.node_nrec, depth, H5AC__READ_ONLY_FLAG))) + if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, parent, &curr_node_ptr, depth, FALSE, H5AC__READ_ONLY_FLAG))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to load B-tree internal node") + /* Unpin parent if necessary */ + if(parent) { + if(parent != hdr && H5AC_unpin_entry(parent) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPIN, FAIL, "unable to unpin parent entry") + parent = NULL; + } /* end if */ + /* Locate node pointer for child */ - if(H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, - udata, &idx, &cmp) < 0) { + if(H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx, &cmp) < 0) { /* Unlock current node before failing */ H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr.addr, internal, H5AC__NO_FLAGS_SET); HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") @@ -566,9 +587,13 @@ H5B2_find(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_found_t op, } /* end if */ /* Unlock current node */ - if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr.addr, internal, H5AC__NO_FLAGS_SET) < 0) + if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr.addr, internal, (unsigned)(hdr->swmr_write ? H5AC__PIN_ENTRY_FLAG : H5AC__NO_FLAGS_SET)) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree node") + /* Keep track of parent if necessary */ + if(hdr->swmr_write) + parent = internal; + /* Set pointer to next node to load */ curr_node_ptr = next_node_ptr; } /* end if */ @@ -598,16 +623,22 @@ H5B2_find(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_found_t op, H5B2_leaf_t *leaf; /* Pointer to leaf node in B-tree */ /* Lock B-tree leaf node */ - if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, curr_node_ptr.addr, curr_node_ptr.node_nrec, H5AC__READ_ONLY_FLAG))) + if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, parent, &curr_node_ptr, FALSE, H5AC__READ_ONLY_FLAG))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") + /* Unpin parent if necessary */ + if(parent) { + if(parent != hdr && H5AC_unpin_entry(parent) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPIN, FAIL, "unable to unpin parent entry") + parent = NULL; + } /* end if */ + /* Locate record */ - if(H5B2__locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, - udata, &idx, &cmp) < 0) { - /* unlock current node before failing */ + if(H5B2__locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx, &cmp) < 0) { + /* Unlock current node before failing */ H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_LEAF, curr_node_ptr.addr, leaf, H5AC__NO_FLAGS_SET); HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") - } + } /* end if */ if(cmp != 0) { /* Unlock leaf node */ @@ -655,6 +686,12 @@ H5B2_find(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_found_t op, } /* end block */ done: + if(parent) { + HDassert(ret_value < 0); + if(parent != hdr && H5AC_unpin_entry(parent) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPIN, FAIL, "unable to unpin parent entry") + } /* end if */ + FUNC_LEAVE_NOAPI(ret_value) } /* H5B2_find() */ @@ -683,6 +720,7 @@ H5B2_index(H5B2_t *bt2, hid_t dxpl_id, H5_iter_order_t order, hsize_t idx, { H5B2_hdr_t *hdr; /* Pointer to the B-tree header */ H5B2_node_ptr_t curr_node_ptr; /* Node pointer info for current node */ + void *parent = NULL; /* Parent of current node */ uint16_t depth; /* Current depth of the tree */ herr_t ret_value = SUCCEED; /* Return value */ @@ -712,6 +750,10 @@ H5B2_index(H5B2_t *bt2, hid_t dxpl_id, H5_iter_order_t order, hsize_t idx, /* Current depth of the tree */ depth = hdr->depth; + /* Set initial parent, if doing swmr writes */ + if(hdr->swmr_write) + parent = hdr; + /* Check for reverse indexing and map requested index to appropriate forward index */ if(order == H5_ITER_DEC) idx = curr_node_ptr.all_nrec - (idx + 1); @@ -723,9 +765,16 @@ H5B2_index(H5B2_t *bt2, hid_t dxpl_id, H5_iter_order_t order, hsize_t idx, unsigned u; /* Local index variable */ /* Lock B-tree current node */ - if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, curr_node_ptr.addr, curr_node_ptr.node_nrec, depth, H5AC__READ_ONLY_FLAG))) + if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, parent, &curr_node_ptr, depth, FALSE, H5AC__READ_ONLY_FLAG))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to load B-tree internal node") + /* Unpin parent if necessary */ + if(parent) { + if(parent != hdr && H5AC_unpin_entry(parent) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPIN, FAIL, "unable to unpin parent entry") + parent = NULL; + } /* end if */ + /* Search for record with correct index */ for(u = 0; u < internal->nrec; u++) { /* Check if record is in child node */ @@ -734,9 +783,13 @@ H5B2_index(H5B2_t *bt2, hid_t dxpl_id, H5_iter_order_t order, hsize_t idx, next_node_ptr = internal->node_ptrs[u]; /* Unlock current node */ - if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr.addr, internal, H5AC__NO_FLAGS_SET) < 0) + if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr.addr, internal, (unsigned)(hdr->swmr_write ? H5AC__PIN_ENTRY_FLAG : H5AC__NO_FLAGS_SET)) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree node") + /* Keep track of parent if necessary */ + if(hdr->swmr_write) + parent = internal; + /* Set pointer to next node to load */ curr_node_ptr = next_node_ptr; @@ -776,9 +829,13 @@ H5B2_index(H5B2_t *bt2, hid_t dxpl_id, H5_iter_order_t order, hsize_t idx, next_node_ptr = internal->node_ptrs[u]; /* Unlock current node */ - if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr.addr, internal, H5AC__NO_FLAGS_SET) < 0) + if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr.addr, internal, (unsigned)(hdr->swmr_write ? H5AC__PIN_ENTRY_FLAG : H5AC__NO_FLAGS_SET)) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree node") + /* Keep track of parent if necessary */ + if(hdr->swmr_write) + parent = internal; + /* Set pointer to next node to load */ curr_node_ptr = next_node_ptr; } /* end if */ @@ -795,9 +852,16 @@ H5B2_index(H5B2_t *bt2, hid_t dxpl_id, H5_iter_order_t order, hsize_t idx, H5B2_leaf_t *leaf; /* Pointer to leaf node in B-tree */ /* Lock B-tree leaf node */ - if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, curr_node_ptr.addr, curr_node_ptr.node_nrec, H5AC__READ_ONLY_FLAG))) + if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, parent, &curr_node_ptr, FALSE, H5AC__READ_ONLY_FLAG))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") + /* Unpin parent if necessary */ + if(parent) { + if(parent != hdr && H5AC_unpin_entry(parent) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPIN, FAIL, "unable to unpin parent entry") + parent = NULL; + } /* end if */ + /* Sanity check index */ HDassert(idx < leaf->nrec); @@ -816,6 +880,12 @@ H5B2_index(H5B2_t *bt2, hid_t dxpl_id, H5_iter_order_t order, hsize_t idx, } /* end block */ done: + if(parent) { + HDassert(ret_value < 0); + if(parent != hdr && H5AC_unpin_entry(parent) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPIN, FAIL, "unable to unpin parent entry") + } /* end if */ + FUNC_LEAVE_NOAPI(ret_value) } /* H5B2_index() */ @@ -859,8 +929,9 @@ H5B2_remove(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_remove_t op, if(hdr->depth > 0) { hbool_t depth_decreased = FALSE; /* Flag to indicate whether the depth of the B-tree decreased */ - if(H5B2__remove_internal(hdr, dxpl_id, &depth_decreased, NULL, hdr->depth, - &(hdr->cache_info), NULL, H5B2_POS_ROOT, &hdr->root, udata, op, op_data) < 0) + if(H5B2__remove_internal(hdr, dxpl_id, &depth_decreased, NULL, NULL, + hdr->depth, &(hdr->cache_info), NULL, H5B2_POS_ROOT, &hdr->root, + udata, op, op_data) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTDELETE, FAIL, "unable to remove record from B-tree internal node") /* Check for decreasing the depth of the B-tree */ @@ -878,7 +949,7 @@ H5B2_remove(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_remove_t op, } /* end for */ } /* end if */ else { - if(H5B2__remove_leaf(hdr, dxpl_id, &hdr->root, H5B2_POS_ROOT, udata, op, op_data) < 0) + if(H5B2__remove_leaf(hdr, dxpl_id, &hdr->root, H5B2_POS_ROOT, hdr, udata, op, op_data) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTDELETE, FAIL, "unable to remove record from B-tree leaf node") } /* end else */ @@ -941,8 +1012,9 @@ H5B2_remove_by_idx(H5B2_t *bt2, hid_t dxpl_id, H5_iter_order_t order, if(hdr->depth > 0) { hbool_t depth_decreased = FALSE; /* Flag to indicate whether the depth of the B-tree decreased */ - if(H5B2__remove_internal_by_idx(hdr, dxpl_id, &depth_decreased, NULL, hdr->depth, - &(hdr->cache_info), NULL, &hdr->root, H5B2_POS_ROOT, idx, op, op_data) < 0) + if(H5B2__remove_internal_by_idx(hdr, dxpl_id, &depth_decreased, NULL, NULL, + hdr->depth, &(hdr->cache_info), NULL, &hdr->root, H5B2_POS_ROOT, + idx, op, op_data) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTDELETE, FAIL, "unable to remove record from B-tree internal node") /* Check for decreasing the depth of the B-tree */ @@ -960,7 +1032,7 @@ H5B2_remove_by_idx(H5B2_t *bt2, hid_t dxpl_id, H5_iter_order_t order, } /* end for */ } /* end if */ else { - if(H5B2__remove_leaf_by_idx(hdr, dxpl_id, &hdr->root, H5B2_POS_ROOT, (unsigned)idx, op, op_data) < 0) + if(H5B2__remove_leaf_by_idx(hdr, dxpl_id, &hdr->root, H5B2_POS_ROOT, hdr, (unsigned)idx, op, op_data) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTDELETE, FAIL, "unable to remove record from B-tree leaf node") } /* end else */ @@ -1055,11 +1127,11 @@ H5B2_neighbor(H5B2_t *bt2, hid_t dxpl_id, H5B2_compare_t range, void *udata, /* Attempt to find neighbor record in B-tree */ if(hdr->depth > 0) { - if(H5B2__neighbor_internal(hdr, dxpl_id, hdr->depth, &hdr->root, NULL, range, udata, op, op_data) < 0) + if(H5B2__neighbor_internal(hdr, dxpl_id, hdr->depth, &hdr->root, NULL, range, hdr, udata, op, op_data) < 0) HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, FAIL, "unable to find neighbor record in B-tree internal node") } /* end if */ else { - if(H5B2__neighbor_leaf(hdr, dxpl_id, &hdr->root, NULL, range, udata, op, op_data) < 0) + if(H5B2__neighbor_leaf(hdr, dxpl_id, &hdr->root, NULL, range, hdr, udata, op, op_data) < 0) HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, FAIL, "unable to find neighbor record in B-tree leaf node") } /* end else */ @@ -1094,6 +1166,7 @@ H5B2_modify(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_modify_t op, { H5B2_hdr_t *hdr; /* Pointer to the B-tree header */ H5B2_node_ptr_t curr_node_ptr; /* Node pointer info for current node */ + void *parent = NULL; /* Parent of current node */ H5B2_nodepos_t curr_pos; /* Position of current node */ uint16_t depth; /* Current depth of the tree */ int cmp; /* Comparison value of records */ @@ -1122,6 +1195,10 @@ H5B2_modify(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_modify_t op, /* Current depth of the tree */ depth = hdr->depth; + /* Set initial parent, if doing swmr writes */ + if(hdr->swmr_write) + parent = hdr; + /* Walk down B-tree to find record or leaf node where record is located */ cmp = -1; curr_pos = H5B2_POS_ROOT; @@ -1131,16 +1208,22 @@ H5B2_modify(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_modify_t op, H5B2_node_ptr_t next_node_ptr; /* Node pointer info for next node */ /* Lock B-tree current node */ - if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, curr_node_ptr.addr, curr_node_ptr.node_nrec, depth, H5AC__NO_FLAGS_SET))) + if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, parent, &curr_node_ptr, depth, FALSE, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to load B-tree internal node") + /* Unpin parent if necessary */ + if(parent) { + if(parent != hdr && H5AC_unpin_entry(parent) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPIN, FAIL, "unable to unpin parent entry") + parent = NULL; + } /* end if */ + /* Locate node pointer for child */ - if(H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, - udata, &idx, &cmp) < 0) { - /* Unlock current node */ + if(H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx, &cmp) < 0) { + /* Unlock current node before failing */ H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr.addr, internal, H5AC__NO_FLAGS_SET); HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") - } + } /* end if */ if(cmp > 0) idx++; @@ -1168,9 +1251,13 @@ H5B2_modify(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_modify_t op, } /* end if */ /* Unlock current node */ - if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr.addr, internal, H5AC__NO_FLAGS_SET) < 0) + if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr.addr, internal, (unsigned)(hdr->swmr_write ? H5AC__PIN_ENTRY_FLAG : H5AC__NO_FLAGS_SET)) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree node") + /* Keep track of parent if necessary */ + if(hdr->swmr_write) + parent = internal; + /* Set pointer to next node to load */ curr_node_ptr = next_node_ptr; } /* end if */ @@ -1209,15 +1296,22 @@ H5B2_modify(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_modify_t op, hbool_t changed = FALSE;/* Whether the 'modify' callback changed the record */ /* Lock B-tree leaf node */ - if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, curr_node_ptr.addr, curr_node_ptr.node_nrec, H5AC__NO_FLAGS_SET))) + if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, parent, &curr_node_ptr, FALSE, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") + /* Unpin parent if necessary */ + if(parent) { + if(parent != hdr && H5AC_unpin_entry(parent) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPIN, FAIL, "unable to unpin parent entry") + parent = NULL; + } /* end if */ + /* Locate record */ - if(H5B2__locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, - udata, &idx, &cmp) < 0) { + if(H5B2__locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx, &cmp) < 0) { + /* Unlock current node before failing */ H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_LEAF, curr_node_ptr.addr, leaf, H5AC__NO_FLAGS_SET); HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") - } + } /* end if */ if(cmp != 0) { /* Unlock leaf node */ @@ -1278,6 +1372,12 @@ H5B2_modify(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_modify_t op, } /* end block */ done: + if(parent) { + HDassert(ret_value < 0); + if(parent != hdr && H5AC_unpin_entry(parent) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPIN, FAIL, "unable to unpin parent entry") + } /* end if */ + FUNC_LEAVE_NOAPI(ret_value) } /* H5B2_modify() */ @@ -1451,6 +1551,59 @@ done: /*------------------------------------------------------------------------- + * Function: H5B2_depend + * + * Purpose: Make a child flush dependency between the v2 B-tree's + * header and another piece of metadata in the file. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Fall 2012 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2_depend(H5B2_t *bt2, hid_t dxpl_id, H5AC_proxy_entry_t *parent) +{ + /* Local variables */ + H5B2_hdr_t *hdr = bt2->hdr; /* Header for B-tree */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(SUCCEED) + + /* + * Check arguments. + */ + HDassert(bt2); + HDassert(hdr); + HDassert(parent); + HDassert(hdr->parent == NULL || hdr->parent == parent); + + /* + * Check to see if the flush dependency between the parent + * and the v2 B-tree header has already been setup. If it hasn't, then + * set it up. + */ + if(NULL == hdr->parent) { + /* Sanity check */ + HDassert(hdr->top_proxy); + + /* Set the shared v2 B-tree header's file context for this operation */ + hdr->f = bt2->f; + + /* Add the v2 B-tree as a child of the parent (proxy) */ + if(H5AC_proxy_entry_add_child(parent, hdr->f, dxpl_id, hdr->top_proxy) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSET, FAIL, "unable to add v2 B-tree as child of proxy") + hdr->parent = parent; + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B2_depend() */ + + +/*------------------------------------------------------------------------- * Function: H5B2_patch_file * * Purpose: Patch the top-level file pointer contained in bt2 diff --git a/src/H5B2cache.c b/src/H5B2cache.c index b899f6a..6954e6c 100644 --- a/src/H5B2cache.c +++ b/src/H5B2cache.c @@ -65,31 +65,34 @@ /********************/ /* Metadata cache callbacks */ -static herr_t H5B2__cache_hdr_get_load_size(const void *udata, size_t *image_len); +static herr_t H5B2__cache_hdr_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5B2__cache_hdr_verify_chksum(const void *image_ptr, size_t len, void *udata); static void *H5B2__cache_hdr_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5B2__cache_hdr_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5B2__cache_hdr_image_len(const void *thing, size_t *image_len); static herr_t H5B2__cache_hdr_serialize(const H5F_t *f, void *image, size_t len, void *thing); +static herr_t H5B2__cache_hdr_notify(H5AC_notify_action_t action, void *thing); static herr_t H5B2__cache_hdr_free_icr(void *thing); -static herr_t H5B2__cache_int_get_load_size(const void *udata, size_t *image_len); +static herr_t H5B2__cache_int_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5B2__cache_int_verify_chksum(const void *image_ptr, size_t len, void *udata); static void *H5B2__cache_int_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5B2__cache_int_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5B2__cache_int_image_len(const void *thing, size_t *image_len); static herr_t H5B2__cache_int_serialize(const H5F_t *f, void *image, size_t len, void *thing); +static herr_t H5B2__cache_int_notify(H5AC_notify_action_t action, void *thing); static herr_t H5B2__cache_int_free_icr(void *thing); -static herr_t H5B2__cache_leaf_get_load_size(const void *udata, size_t *image_len); +static herr_t H5B2__cache_leaf_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5B2__cache_leaf_verify_chksum(const void *image_ptr, size_t len, void *udata); static void *H5B2__cache_leaf_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5B2__cache_leaf_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5B2__cache_leaf_image_len(const void *thing, size_t *image_len); static herr_t H5B2__cache_leaf_serialize(const H5F_t *f, void *image, size_t len, void *thing); +static herr_t H5B2__cache_leaf_notify(H5AC_notify_action_t action, void *thing); static herr_t H5B2__cache_leaf_free_icr(void *thing); /*********************/ @@ -102,14 +105,15 @@ const H5AC_class_t H5AC_BT2_HDR[1] = {{ "v2 B-tree header", /* Metadata client name (for debugging) */ H5FD_MEM_BTREE, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5B2__cache_hdr_get_load_size, /* 'get_load_size' callback */ + H5B2__cache_hdr_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5B2__cache_hdr_verify_chksum, /* 'verify_chksum' callback */ H5B2__cache_hdr_deserialize, /* 'deserialize' callback */ H5B2__cache_hdr_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5B2__cache_hdr_serialize, /* 'serialize' callback */ - NULL, /* 'notify' callback */ + H5B2__cache_hdr_notify, /* 'notify' callback */ H5B2__cache_hdr_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -119,14 +123,15 @@ const H5AC_class_t H5AC_BT2_INT[1] = {{ "v2 B-tree internal node", /* Metadata client name (for debugging) */ H5FD_MEM_BTREE, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5B2__cache_int_get_load_size, /* 'get_load_size' callback */ + H5B2__cache_int_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5B2__cache_int_verify_chksum, /* 'verify_chksum' callback */ H5B2__cache_int_deserialize, /* 'deserialize' callback */ H5B2__cache_int_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5B2__cache_int_serialize, /* 'serialize' callback */ - NULL, /* 'notify' callback */ + H5B2__cache_int_notify, /* 'notify' callback */ H5B2__cache_int_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -136,14 +141,15 @@ const H5AC_class_t H5AC_BT2_LEAF[1] = {{ "v2 B-tree leaf node", /* Metadata client name (for debugging) */ H5FD_MEM_BTREE, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5B2__cache_leaf_get_load_size, /* 'get_load_size' callback */ + H5B2__cache_leaf_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5B2__cache_leaf_verify_chksum, /* 'verify_chksum' callback */ H5B2__cache_leaf_deserialize, /* 'deserialize' callback */ H5B2__cache_leaf_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5B2__cache_leaf_serialize, /* 'serialize' callback */ - NULL, /* 'notify' callback */ + H5B2__cache_leaf_notify, /* 'notify' callback */ H5B2__cache_leaf_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -160,7 +166,7 @@ const H5AC_class_t H5AC_BT2_LEAF[1] = {{ /*------------------------------------------------------------------------- - * Function: H5B2__cache_hdr_get_load_size + * Function: H5B2__cache_hdr_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -173,21 +179,58 @@ const H5AC_class_t H5AC_BT2_LEAF[1] = {{ *------------------------------------------------------------------------- */ static herr_t -H5B2__cache_hdr_get_load_size(const void *_udata, size_t *image_len) +H5B2__cache_hdr_get_initial_load_size(void *_udata, size_t *image_len) { - const H5B2_hdr_cache_ud_t *udata = (const H5B2_hdr_cache_ud_t *)_udata; /* User data for callback */ + H5B2_hdr_cache_ud_t *udata = (H5B2_hdr_cache_ud_t *)_udata; /* User data for callback */ FUNC_ENTER_STATIC_NOERR /* Check arguments */ HDassert(udata); + HDassert(udata->f); HDassert(image_len); /* Set the image length size */ *image_len = H5B2_HEADER_SIZE_FILE(udata->f); FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5B2__cache_hdr_get_load_size() */ +} /* end H5B2__cache_hdr_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__cache_hdr_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5B2__cache_hdr_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B2__cache_hdr_verify_chksum() */ /*------------------------------------------------------------------------- @@ -214,7 +257,6 @@ H5B2__cache_hdr_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, H5B2_subid_t id; /* ID of B-tree class, as found in file */ uint16_t depth; /* Depth of B-tree */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ H5B2_hdr_t *ret_value = NULL; /* Return value */ @@ -260,19 +302,14 @@ H5B2__cache_hdr_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, UINT16DECODE(image, hdr->root.node_nrec); H5F_DECODE_LENGTH(udata->f, image, hdr->root.all_nrec); + /* checksum verification already done in verify_chksum cb */ + /* Metadata checksum */ UINT32DECODE(image, stored_chksum); /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == hdr->hdr_size); - /* Compute checksum on entire header */ - computed_chksum = H5_checksum_metadata(_image, (hdr->hdr_size - H5B2_SIZEOF_CHKSUM), 0); - - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "incorrect metadata checksum for v2 B-tree header") - /* Initialize B-tree header info */ cparam.cls = H5B2_client_class_g[id]; if(H5B2__hdr_init(hdr, &cparam, udata->ctx_udata, depth) < 0) @@ -310,8 +347,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B2__cache_hdr_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5B2__cache_hdr_image_len(const void *_thing, size_t *image_len) { const H5B2_hdr_t *hdr = (const H5B2_hdr_t *)_thing; /* Pointer to the B-tree header */ @@ -400,6 +436,91 @@ H5B2__cache_hdr_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED le /*------------------------------------------------------------------------- + * Function: H5B2__cache_hdr_notify + * + * Purpose: Handle cache action notifications + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * nfortne2@hdfgroup.org + * Apr 24 2012 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5B2__cache_hdr_notify(H5AC_notify_action_t action, void *_thing) +{ + H5B2_hdr_t *hdr = (H5B2_hdr_t *)_thing; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* + * Check arguments. + */ + HDassert(hdr); + + /* Check if the file was opened with SWMR-write access */ + if(hdr->swmr_write) { + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + case H5AC_NOTIFY_ACTION_AFTER_LOAD: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + /* Increment the shadow epoch, forcing new modifications to + * internal and leaf nodes to create new shadow copies */ + hdr->shadow_epoch++; + break; + + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + /* If hdr->parent != NULL, hdr->parent is used to destroy + * the flush dependency before the header is evicted. + */ + if(hdr->parent) { + /* Sanity check */ + HDassert(hdr->top_proxy); + + /* Destroy flush dependency on object header proxy */ + if(H5AC_proxy_entry_remove_child((H5AC_proxy_entry_t *)hdr->parent, (void *)hdr->top_proxy) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNDEPEND, FAIL, "unable to destroy flush dependency between v2 B-tree and proxy") + hdr->parent = NULL; + } /* end if */ + + /* Detach from 'top' proxy for extensible array */ + if(hdr->top_proxy) { + if(H5AC_proxy_entry_remove_child(hdr->top_proxy, hdr) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNDEPEND, FAIL, "unable to destroy flush dependency between header and v2 B-tree 'top' proxy") + /* Don't reset hdr->top_proxy here, it's destroyed when the header is freed -QAK */ + } /* end if */ + break; + + default: +#ifdef NDEBUG + HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, FAIL, "unknown action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Unknown action?!?"); +#endif /* NDEBUG */ + } /* end switch */ + } /* end if */ + else + HDassert(NULL == hdr->parent); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B2__cache_hdr_notify() */ + + +/*------------------------------------------------------------------------- * Function: H5B2__cache_hdr_free_icr * * Purpose: Destroy/release an "in core representation" of a data @@ -433,7 +554,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5B2__cache_int_get_load_size + * Function: H5B2__cache_int_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -446,9 +567,9 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B2__cache_int_get_load_size(const void *_udata, size_t *image_len) +H5B2__cache_int_get_initial_load_size(void *_udata, size_t *image_len) { - const H5B2_internal_cache_ud_t *udata = (const H5B2_internal_cache_ud_t *)_udata; /* User data for callback */ + H5B2_internal_cache_ud_t *udata = (H5B2_internal_cache_ud_t *)_udata; /* User data for callback */ FUNC_ENTER_STATIC_NOERR @@ -461,7 +582,49 @@ H5B2__cache_int_get_load_size(const void *_udata, size_t *image_len) *image_len = udata->hdr->node_size; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5B2__cache_int_get_load_size() */ +} /* end H5B2__cache_int_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__cache_int_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5B2__cache_int_verify_chksum(const void *_image, size_t H5_ATTR_UNUSED len, void *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5B2_internal_cache_ud_t *udata = (H5B2_internal_cache_ud_t *)_udata; /* Pointer to user data */ + size_t chk_size; /* Exact size of the node with checksum at the end */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + HDassert(udata); + + /* Internal node prefix header + records + child pointer triplets: size with checksum at the end */ + chk_size = H5B2_INT_PREFIX_SIZE + (udata->nrec * udata->hdr->rrec_size) + ((size_t)(udata->nrec + 1) * H5B2_INT_POINTER_SIZE(udata->hdr, udata->depth)); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, chk_size, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B2__cache_int_verify_chksum() */ /*------------------------------------------------------------------------- @@ -488,7 +651,6 @@ H5B2__cache_int_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, uint8_t *native; /* Pointer to native record info */ H5B2_node_ptr_t *int_node_ptr; /* Pointer to node pointer info */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ unsigned u; /* Local index variable */ H5B2_internal_t *ret_value = NULL; /* Return value */ @@ -499,9 +661,8 @@ H5B2__cache_int_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, HDassert(udata); /* Allocate new internal node and reset cache info */ - if(NULL == (internal = H5FL_MALLOC(H5B2_internal_t))) + if(NULL == (internal = H5FL_CALLOC(H5B2_internal_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - HDmemset(&internal->cache_info, 0, sizeof(H5AC_info_t)); /* Increment ref. count on B-tree header */ if(H5B2__hdr_incr(udata->hdr) < 0) @@ -509,6 +670,8 @@ H5B2__cache_int_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, /* Share B-tree information */ internal->hdr = udata->hdr; + internal->parent = udata->parent; + internal->shadow_epoch = udata->hdr->shadow_epoch; /* Magic number */ if(HDmemcmp(image, H5B2_INT_MAGIC, (size_t)H5_SIZEOF_MAGIC)) @@ -562,8 +725,7 @@ H5B2__cache_int_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, int_node_ptr++; } /* end for */ - /* Compute checksum on internal node */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -571,10 +733,6 @@ H5B2__cache_int_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, /* Sanity check parsing */ HDassert((size_t)(image - (const uint8_t *)_image) <= len); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "incorrect metadata checksum for v2 internal node") - /* Set return value */ ret_value = internal; @@ -601,8 +759,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B2__cache_int_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5B2__cache_int_image_len(const void *_thing, size_t *image_len) { const H5B2_internal_t *internal = (const H5B2_internal_t *)_thing; /* Pointer to the B-tree internal node */ @@ -707,6 +864,80 @@ done: /*------------------------------------------------------------------------- + * Function: H5B2__cache_int_notify + * + * Purpose: Handle cache action notifications + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * nfortne2@hdfgroup.org + * Apr 25 2012 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5B2__cache_int_notify(H5AC_notify_action_t action, void *_thing) +{ + H5B2_internal_t *internal = (H5B2_internal_t *)_thing; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + /* + * Check arguments. + */ + HDassert(internal); + HDassert(internal->hdr); + + /* Check if the file was opened with SWMR-write access */ + if(internal->hdr->swmr_write) { + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + case H5AC_NOTIFY_ACTION_AFTER_LOAD: + /* Create flush dependency on parent */ + if(H5B2__create_flush_depend((H5AC_info_t *)internal->parent, (H5AC_info_t *)internal) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTDEPEND, FAIL, "unable to create flush dependency") + break; + + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + /* Destroy flush dependency on parent */ + if(H5B2__destroy_flush_depend((H5AC_info_t *)internal->parent, (H5AC_info_t *)internal) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNDEPEND, FAIL, "unable to destroy flush dependency") + + /* Detach from 'top' proxy for v2 B-tree */ + if(internal->top_proxy) { + if(H5AC_proxy_entry_remove_child(internal->top_proxy, internal) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNDEPEND, FAIL, "unable to destroy flush dependency between internal node and v2 B-tree 'top' proxy") + internal->top_proxy = NULL; + } /* end if */ + break; + + default: +#ifdef NDEBUG + HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, FAIL, "unknown action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Unknown action?!?"); +#endif /* NDEBUG */ + } /* end switch */ + } /* end if */ + else + HDassert(NULL == internal->top_proxy); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B2__cache_int_notify() */ + + +/*------------------------------------------------------------------------- * Function: H5B2__cache_int_free_icr * * Purpose: Destroy/release an "in core representation" of a data @@ -721,17 +952,18 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B2__cache_int_free_icr(void *thing) +H5B2__cache_int_free_icr(void *_thing) { + H5B2_internal_t *internal = (H5B2_internal_t *)_thing; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC /* Check arguments */ - HDassert(thing); + HDassert(internal); /* Release v2 B-tree internal node */ - if(H5B2__internal_free((H5B2_internal_t *)thing) < 0) + if(H5B2__internal_free(internal) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTFREE, FAIL, "unable to release v2 B-tree internal node") done: @@ -740,7 +972,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5B2__cache_leaf_get_load_size + * Function: H5B2__cache_leaf_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -753,9 +985,9 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B2__cache_leaf_get_load_size(const void *_udata, size_t *image_len) +H5B2__cache_leaf_get_initial_load_size(void *_udata, size_t *image_len) { - const H5B2_leaf_cache_ud_t *udata = (const H5B2_leaf_cache_ud_t *)_udata; /* User data for callback */ + H5B2_leaf_cache_ud_t *udata = (H5B2_leaf_cache_ud_t *)_udata; /* User data for callback */ FUNC_ENTER_STATIC_NOERR @@ -768,7 +1000,49 @@ H5B2__cache_leaf_get_load_size(const void *_udata, size_t *image_len) *image_len = udata->hdr->node_size; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5B2__cache_leaf_get_load_size() */ +} /* end H5B2__cache_leaf_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__cache_leaf_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5B2__cache_leaf_verify_chksum(const void *_image, size_t H5_ATTR_UNUSED len, void *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5B2_internal_cache_ud_t *udata = (H5B2_internal_cache_ud_t *)_udata; /* Pointer to user data */ + size_t chk_size; /* Exact size of the node with checksum at the end */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + HDassert(udata); + + /* Leaf node prefix header + records: size with checksum at the end */ + chk_size = H5B2_LEAF_PREFIX_SIZE + (udata->nrec * udata->hdr->rrec_size); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, chk_size, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B2__cache_leaf_verify_chksum() */ /*------------------------------------------------------------------------- @@ -794,7 +1068,6 @@ H5B2__cache_leaf_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint8_t *native; /* Pointer to native keys */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ unsigned u; /* Local index variable */ H5B2_leaf_t *ret_value = NULL; /* Return value */ @@ -805,9 +1078,8 @@ H5B2__cache_leaf_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, HDassert(udata); /* Allocate new leaf node and reset cache info */ - if(NULL == (leaf = H5FL_MALLOC(H5B2_leaf_t))) + if(NULL == (leaf = H5FL_CALLOC(H5B2_leaf_t))) HGOTO_ERROR(H5E_BTREE, H5E_CANTALLOC, NULL, "memory allocation failed") - HDmemset(&leaf->cache_info, 0, sizeof(H5AC_info_t)); /* Increment ref. count on B-tree header */ if(H5B2__hdr_incr(udata->hdr) < 0) @@ -815,6 +1087,8 @@ H5B2__cache_leaf_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, /* Share B-tree header information */ leaf->hdr = udata->hdr; + leaf->parent = udata->parent; + leaf->shadow_epoch = udata->hdr->shadow_epoch; /* Magic number */ if(HDmemcmp(image, H5B2_LEAF_MAGIC, (size_t)H5_SIZEOF_MAGIC)) @@ -848,8 +1122,7 @@ H5B2__cache_leaf_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, native += udata->hdr->cls->nrec_size; } /* end for */ - /* Compute checksum on leaf node */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -857,10 +1130,6 @@ H5B2__cache_leaf_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, /* Sanity check parsing */ HDassert((size_t)(image - (const uint8_t *)_image) <= udata->hdr->node_size); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "incorrect metadata checksum for v2 leaf node") - /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) <= len); @@ -890,8 +1159,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B2__cache_leaf_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5B2__cache_leaf_image_len(const void *_thing, size_t *image_len) { const H5B2_leaf_t *leaf = (const H5B2_leaf_t *)_thing; /* Pointer to the B-tree leaf node */ @@ -923,7 +1191,7 @@ H5B2__cache_leaf_image_len(const void *_thing, size_t *image_len, *------------------------------------------------------------------------- */ static herr_t -H5B2__cache_leaf_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED len, +H5B2__cache_leaf_serialize(const H5F_t H5_ATTR_UNUSED *f, void *_image, size_t H5_ATTR_UNUSED len, void *_thing) { H5B2_leaf_t *leaf = (H5B2_leaf_t *)_thing; /* Pointer to the B-tree leaf node */ @@ -982,6 +1250,80 @@ done: /*------------------------------------------------------------------------- + * Function: H5B2__cache_leaf_notify + * + * Purpose: Handle cache action notifications + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * nfortne2@hdfgroup.org + * Apr 25 2012 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5B2__cache_leaf_notify(H5AC_notify_action_t action, void *_thing) +{ + H5B2_leaf_t *leaf = (H5B2_leaf_t *)_thing; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + /* + * Check arguments. + */ + HDassert(leaf); + HDassert(leaf->hdr); + + /* Check if the file was opened with SWMR-write access */ + if(leaf->hdr->swmr_write) { + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + case H5AC_NOTIFY_ACTION_AFTER_LOAD: + /* Create flush dependency on parent */ + if(H5B2__create_flush_depend((H5AC_info_t *)leaf->parent, (H5AC_info_t *)leaf) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTDEPEND, FAIL, "unable to create flush dependency") + break; + + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + /* Destroy flush dependency on parent */ + if(H5B2__destroy_flush_depend((H5AC_info_t *)leaf->parent, (H5AC_info_t *)leaf) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNDEPEND, FAIL, "unable to destroy flush dependency") + + /* Detach from 'top' proxy for v2 B-tree */ + if(leaf->top_proxy) { + if(H5AC_proxy_entry_remove_child(leaf->top_proxy, leaf) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNDEPEND, FAIL, "unable to destroy flush dependency between leaf node and v2 B-tree 'top' proxy") + leaf->top_proxy = NULL; + } /* end if */ + break; + + default: +#ifdef NDEBUG + HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, FAIL, "unknown action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Unknown action?!?"); +#endif /* NDEBUG */ + } /* end switch */ + } /* end if */ + else + HDassert(NULL == leaf->top_proxy); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B2__cache_leaf_notify() */ + + +/*------------------------------------------------------------------------- * Function: H5B2__cache_leaf_free_icr * * Purpose: Destroy/release an "in core representation" of a data @@ -996,17 +1338,18 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B2__cache_leaf_free_icr(void *thing) +H5B2__cache_leaf_free_icr(void *_thing) { + H5B2_leaf_t *leaf = (H5B2_leaf_t *)_thing; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC /* Check arguments */ - HDassert(thing); + HDassert(leaf); /* Destroy v2 B-tree leaf node */ - if(H5B2__leaf_free((H5B2_leaf_t *)thing) < 0) + if(H5B2__leaf_free(leaf) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTFREE, FAIL, "unable to destroy B-tree leaf node") done: diff --git a/src/H5B2dbg.c b/src/H5B2dbg.c index ddff9e9..19ca89a 100644 --- a/src/H5B2dbg.c +++ b/src/H5B2dbg.c @@ -188,6 +188,7 @@ H5B2__int_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE *stream, int indent, { H5B2_hdr_t *hdr = NULL; /* B-tree header */ H5B2_internal_t *internal = NULL; /* B-tree internal node */ + H5B2_node_ptr_t node_ptr; /* Fake node pointer for protect */ unsigned u; /* Local index variable */ char temp_str[128]; /* Temporary string, for formatting */ herr_t ret_value=SUCCEED; /* Return value */ @@ -217,9 +218,10 @@ H5B2__int_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE *stream, int indent, /* * Load the B-tree internal node */ - H5_CHECK_OVERFLOW(nrec, unsigned, uint16_t); H5_CHECK_OVERFLOW(depth, unsigned, uint16_t); - if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, addr, (uint16_t)nrec, (uint16_t)depth, H5AC__READ_ONLY_FLAG))) + node_ptr.addr = addr; + H5_CHECKED_ASSIGN(node_ptr.node_nrec, unsigned, nrec, uint16_t) + if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, NULL, &node_ptr, (uint16_t)depth, FALSE, H5AC__READ_ONLY_FLAG))) HGOTO_ERROR(H5E_BTREE, H5E_CANTLOAD, FAIL, "unable to load B-tree internal node") /* Print opening message */ @@ -298,6 +300,7 @@ H5B2__leaf_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE *stream, int indent { H5B2_hdr_t *hdr = NULL; /* B-tree header */ H5B2_leaf_t *leaf = NULL; /* B-tree leaf node */ + H5B2_node_ptr_t node_ptr; /* Fake node pointer for protect */ unsigned u; /* Local index variable */ char temp_str[128]; /* Temporary string, for formatting */ herr_t ret_value = SUCCEED; /* Return value */ @@ -328,7 +331,9 @@ H5B2__leaf_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE *stream, int indent * Load the B-tree leaf node */ H5_CHECK_OVERFLOW(nrec, unsigned, uint16_t); - if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, addr, (uint16_t)nrec, H5AC__READ_ONLY_FLAG))) + node_ptr.addr = addr; + H5_CHECKED_ASSIGN(node_ptr.node_nrec, unsigned, nrec, uint16_t) + if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, NULL, &node_ptr, FALSE, H5AC__READ_ONLY_FLAG))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") /* Print opening message */ diff --git a/src/H5B2hdr.c b/src/H5B2hdr.c index 49ffb5b..251a33d 100644 --- a/src/H5B2hdr.c +++ b/src/H5B2hdr.c @@ -133,10 +133,6 @@ H5B2__hdr_init(H5B2_hdr_t *hdr, const H5B2_create_t *cparam, void *ctx_udata, HDassert(cparam->split_percent > 0 && cparam->split_percent <= 100); HDassert(cparam->merge_percent < (cparam->split_percent / 2)); - /* Initialize basic information */ - hdr->rc = 0; - hdr->pending_delete = FALSE; - /* Assign dynamic information */ hdr->depth = depth; @@ -207,6 +203,13 @@ H5B2__hdr_init(H5B2_hdr_t *hdr, const H5B2_create_t *cparam, void *ctx_udata, } /* end for */ } /* end if */ + /* Determine if we are doing SWMR writes. Only enable for data chunks for now. */ + hdr->swmr_write = (H5F_INTENT(hdr->f) & H5F_ACC_SWMR_WRITE) > 0 + && (hdr->cls->id == H5B2_CDSET_ID || hdr->cls->id == H5B2_CDSET_FILT_ID); + + /* Reset the shadow epoch */ + hdr->shadow_epoch = 0; + /* Create the callback context, if the callback exists */ if(hdr->cls->crt_context) if(NULL == (hdr->cb_ctx = (*hdr->cls->crt_context)(ctx_udata))) @@ -285,6 +288,7 @@ H5B2__hdr_create(H5F_t *f, hid_t dxpl_id, const H5B2_create_t *cparam, void *ctx_udata) { H5B2_hdr_t *hdr = NULL; /* The new v2 B-tree header information */ + hbool_t inserted = FALSE; /* Whether the header was inserted into cache */ haddr_t ret_value = HADDR_UNDEF; /* Return value */ FUNC_ENTER_PACKAGE @@ -307,17 +311,40 @@ H5B2__hdr_create(H5F_t *f, hid_t dxpl_id, const H5B2_create_t *cparam, if(HADDR_UNDEF == (hdr->addr = H5MF_alloc(f, H5FD_MEM_BTREE, dxpl_id, (hsize_t)hdr->hdr_size))) HGOTO_ERROR(H5E_BTREE, H5E_CANTALLOC, HADDR_UNDEF, "file allocation failed for B-tree header") + /* Create 'top' proxy for extensible array entries */ + if(hdr->swmr_write) + if(NULL == (hdr->top_proxy = H5AC_proxy_entry_create())) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCREATE, HADDR_UNDEF, "can't create v2 B-tree proxy") + /* Cache the new B-tree node */ if(H5AC_insert_entry(f, dxpl_id, H5AC_BT2_HDR, hdr->addr, hdr, H5AC__NO_FLAGS_SET) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, HADDR_UNDEF, "can't add B-tree header to cache") + inserted = TRUE; + + /* Add header as child of 'top' proxy */ + if(hdr->top_proxy) + if(H5AC_proxy_entry_add_child(hdr->top_proxy, f, dxpl_id, hdr) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSET, HADDR_UNDEF, "unable to add v2 B-tree header as child of array proxy") /* Set address of v2 B-tree header to return */ ret_value = hdr->addr; done: - if(!H5F_addr_defined(ret_value) && hdr) - if(H5B2__hdr_free(hdr) < 0) - HDONE_ERROR(H5E_BTREE, H5E_CANTRELEASE, HADDR_UNDEF, "unable to release v2 B-tree header") + if(!H5F_addr_defined(ret_value)) + if(hdr) { + /* Remove from cache, if inserted */ + if(inserted) + if(H5AC_remove_entry(hdr) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTREMOVE, HADDR_UNDEF, "unable to remove v2 B-tree header from cache") + + /* Release header's disk space */ + if(H5F_addr_defined(hdr->addr) && H5MF_xfree(f, H5FD_MEM_BTREE, dxpl_id, hdr->addr, (hsize_t)hdr->hdr_size) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTFREE, HADDR_UNDEF, "unable to free v2 B-tree header") + + /* Destroy header */ + if(H5B2__hdr_free(hdr) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTRELEASE, HADDR_UNDEF, "unable to release v2 B-tree header") + } /* end if */ FUNC_LEAVE_NOAPI(ret_value) } /* end H5B2__hdr_create() */ @@ -503,6 +530,7 @@ H5B2__hdr_protect(H5F_t *f, hid_t dxpl_id, haddr_t hdr_addr, void *ctx_udata, unsigned flags) { H5B2_hdr_cache_ud_t udata; /* User data for cache callbacks */ + H5B2_hdr_t *hdr = NULL; /* v2 B-tree header */ H5B2_hdr_t *ret_value = NULL; /* Return value */ FUNC_ENTER_PACKAGE @@ -520,11 +548,32 @@ H5B2__hdr_protect(H5F_t *f, hid_t dxpl_id, haddr_t hdr_addr, void *ctx_udata, udata.ctx_udata = ctx_udata; /* Protect the header */ - if(NULL == (ret_value = (H5B2_hdr_t *)H5AC_protect(f, dxpl_id, H5AC_BT2_HDR, hdr_addr, &udata, flags))) + if(NULL == (hdr = (H5B2_hdr_t *)H5AC_protect(f, dxpl_id, H5AC_BT2_HDR, hdr_addr, &udata, flags))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, NULL, "unable to load v2 B-tree header, address = %llu", (unsigned long long)hdr_addr) - ret_value->f = f; /* (Must be set again here, in case the header was already in the cache -QAK) */ + hdr->f = f; /* (Must be set again here, in case the header was already in the cache -QAK) */ + + /* Create top proxy, if it doesn't exist */ + if(hdr->swmr_write && NULL == hdr->top_proxy) { + /* Create 'top' proxy for v2 B-tree entries */ + if(NULL == (hdr->top_proxy = H5AC_proxy_entry_create())) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCREATE, NULL, "can't create v2 B-tree proxy") + + /* Add header as child of 'top' proxy */ + if(H5AC_proxy_entry_add_child(hdr->top_proxy, f, dxpl_id, hdr) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSET, NULL, "unable to add v2 B-tree header as child of proxy") + } /* end if */ + + /* Set return value */ + ret_value = hdr; done: + /* Clean up on error */ + if(!ret_value) { + /* Release the header, if it was protected */ + if(hdr && H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_HDR, hdr_addr, hdr, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, NULL, "unable to unprotect v2 B-tree header, address = %llu", (unsigned long long)hdr_addr) + } /* end if */ + FUNC_LEAVE_NOAPI(ret_value) } /* end H5B2__hdr_protect() */ @@ -623,6 +672,13 @@ H5B2__hdr_free(H5B2_hdr_t *hdr) if(hdr->max_native_rec) hdr->max_native_rec = H5MM_xfree(hdr->max_native_rec); + /* Destroy the 'top' proxy */ + if(hdr->top_proxy) { + if(H5AC_proxy_entry_dest(hdr->top_proxy) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTRELEASE, FAIL, "unable to destroy v2 B-tree 'top' proxy") + hdr->top_proxy = NULL; + } /* end if */ + /* Free B-tree header info */ hdr = H5FL_FREE(H5B2_hdr_t, hdr); @@ -671,7 +727,7 @@ H5B2__hdr_delete(H5B2_hdr_t *hdr, hid_t dxpl_id) /* Delete all nodes in B-tree */ if(H5F_addr_defined(hdr->root.addr)) - if(H5B2__delete_node(hdr, dxpl_id, hdr->depth, &hdr->root, hdr->remove_op, hdr->remove_op_data) < 0) + if(H5B2__delete_node(hdr, dxpl_id, hdr->depth, &hdr->root, hdr, hdr->remove_op, hdr->remove_op_data) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTDELETE, FAIL, "unable to delete B-tree nodes") /* Indicate that the heap header should be deleted & file space freed */ diff --git a/src/H5B2int.c b/src/H5B2int.c index 3bd788c..47100fd 100644 --- a/src/H5B2int.c +++ b/src/H5B2int.c @@ -37,16 +37,13 @@ #include "H5private.h" /* Generic Functions */ #include "H5B2pkg.h" /* v2 B-trees */ #include "H5Eprivate.h" /* Error handling */ -#include "H5MFprivate.h" /* File memory management */ -#include "H5MMprivate.h" /* Memory management */ #include "H5VMprivate.h" /* Vectors and arrays */ + /****************/ /* Local Macros */ /****************/ -/* Uncomment this macro to enable extra sanity checking */ -/* #define H5B2_DEBUG */ /******************/ /* Local Typedefs */ @@ -61,44 +58,15 @@ /********************/ /* Local Prototypes */ /********************/ +static herr_t H5B2__update_child_flush_depends(H5B2_hdr_t *hdr, hid_t dxpl_id, + unsigned depth, const H5B2_node_ptr_t *node_ptrs, unsigned start_idx, + unsigned end_idx, void *old_parent, void *new_parent); -/* Helper functions */ -static herr_t H5B2__split1(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, - H5B2_node_ptr_t *curr_node_ptr, unsigned *parent_cache_info_flags_ptr, - H5B2_internal_t *internal, unsigned *internal_flags_ptr, unsigned idx); -static herr_t H5B2__redistribute2(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, - H5B2_internal_t *internal, unsigned idx); -static herr_t H5B2__redistribute3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, - H5B2_internal_t *internal, unsigned *internal_flags_ptr, unsigned idx); -static herr_t H5B2__merge2(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, - H5B2_node_ptr_t *curr_node_ptr, unsigned *parent_cache_info_flags_ptr, - H5B2_internal_t *internal, unsigned *internal_flags_ptr, unsigned idx); -static herr_t H5B2__merge3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, - H5B2_node_ptr_t *curr_node_ptr, unsigned *parent_cache_info_flags_ptr, - H5B2_internal_t *internal, unsigned *internal_flags_ptr, unsigned idx); -static herr_t H5B2__swap_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, - H5B2_internal_t *internal, unsigned *internal_flags_ptr, unsigned idx, - void *swap_loc); -static herr_t H5B2__create_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, - H5B2_node_ptr_t *node_ptr, uint16_t depth); -#ifdef H5B2_DEBUG -/* Don't label these with H5_ATTR_PURE or you'll get even more warnings... */ -static herr_t H5B2__assert_leaf(const H5B2_hdr_t *hdr, const H5B2_leaf_t *leaf); -static herr_t H5B2__assert_leaf2(const H5B2_hdr_t *hdr, const H5B2_leaf_t *leaf, const H5B2_leaf_t *leaf2); -static herr_t H5B2__assert_internal(hsize_t parent_all_nrec, const H5B2_hdr_t *hdr, const H5B2_internal_t *internal); -static herr_t H5B2__assert_internal2(hsize_t parent_all_nrec, const H5B2_hdr_t *hdr, const H5B2_internal_t *internal, const H5B2_internal_t *internal2); -#endif /* H5B2_DEBUG */ /*********************/ /* Package Variables */ /*********************/ -/* Declare a free list to manage the H5B2_internal_t struct */ -H5FL_DEFINE(H5B2_internal_t); - -/* Declare a free list to manage the H5B2_leaf_t struct */ -H5FL_DEFINE(H5B2_leaf_t); - /*****************************/ /* Library Private Variables */ @@ -137,7 +105,7 @@ H5FL_SEQ_EXTERN(H5B2_node_info_t); */ herr_t H5B2__locate_record(const H5B2_class_t *type, unsigned nrec, size_t *rec_off, - const uint8_t *native, const void *udata, unsigned *idx, int *cmp) + const uint8_t *native, const void *udata, unsigned *idx, int *cmp) { unsigned lo = 0, hi; /* Low & high index values */ unsigned my_idx = 0; /* Final index value */ @@ -179,7 +147,7 @@ done: * *------------------------------------------------------------------------- */ -static herr_t +herr_t H5B2__split1(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, H5B2_node_ptr_t *curr_node_ptr, unsigned *parent_cache_info_flags_ptr, H5B2_internal_t *internal, unsigned *internal_flags_ptr, unsigned idx) @@ -195,7 +163,7 @@ H5B2__split1(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, unsigned left_child_flags = H5AC__NO_FLAGS_SET, right_child_flags = H5AC__NO_FLAGS_SET; /* Flags for unprotecting child nodes */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC + FUNC_ENTER_PACKAGE /* Check arguments. */ HDassert(hdr); @@ -214,19 +182,20 @@ H5B2__split1(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Create new internal node */ internal->node_ptrs[idx + 1].all_nrec = internal->node_ptrs[idx + 1].node_nrec = 0; - if(H5B2__create_internal(hdr, dxpl_id, &(internal->node_ptrs[idx + 1]), (uint16_t)(depth - 1)) < 0) + if(H5B2__create_internal(hdr, dxpl_id, internal, &(internal->node_ptrs[idx + 1]), (uint16_t)(depth - 1)) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, FAIL, "unable to create new internal node") /* Setup information for unlocking child nodes */ child_class = H5AC_BT2_INT; - left_addr = internal->node_ptrs[idx].addr; - right_addr = internal->node_ptrs[idx + 1].addr; /* Protect both leaves */ - if(NULL == (left_int = H5B2__protect_internal(hdr, dxpl_id, left_addr, internal->node_ptrs[idx].node_nrec, (uint16_t)(depth - 1), H5AC__NO_FLAGS_SET))) + /* (Shadow left node if doing SWMR writes) */ + if(NULL == (left_int = H5B2__protect_internal(hdr, dxpl_id, internal, &internal->node_ptrs[idx], (uint16_t)(depth - 1), hdr->swmr_write, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") - if(NULL == (right_int = H5B2__protect_internal(hdr, dxpl_id, right_addr, internal->node_ptrs[idx + 1].node_nrec, (uint16_t)(depth - 1), H5AC__NO_FLAGS_SET))) + left_addr = internal->node_ptrs[idx].addr; + if(NULL == (right_int = H5B2__protect_internal(hdr, dxpl_id, internal, &internal->node_ptrs[idx + 1], (uint16_t)(depth - 1), FALSE, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") + right_addr = internal->node_ptrs[idx + 1].addr; /* More setup for child nodes */ left_child = left_int; @@ -243,19 +212,20 @@ H5B2__split1(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Create new leaf node */ internal->node_ptrs[idx + 1].all_nrec = internal->node_ptrs[idx + 1].node_nrec = 0; - if(H5B2__create_leaf(hdr, dxpl_id, &(internal->node_ptrs[idx + 1])) < 0) + if(H5B2__create_leaf(hdr, dxpl_id, internal, &(internal->node_ptrs[idx + 1])) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, FAIL, "unable to create new leaf node") /* Setup information for unlocking child nodes */ child_class = H5AC_BT2_LEAF; - left_addr = internal->node_ptrs[idx].addr; - right_addr = internal->node_ptrs[idx + 1].addr; /* Protect both leaves */ - if(NULL == (left_leaf = H5B2__protect_leaf(hdr, dxpl_id, left_addr, internal->node_ptrs[idx].node_nrec, H5AC__NO_FLAGS_SET))) + /* (Shadow the left node if doing SWMR writes) */ + if(NULL == (left_leaf = H5B2__protect_leaf(hdr, dxpl_id, internal, &internal->node_ptrs[idx], hdr->swmr_write, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") - if(NULL == (right_leaf = H5B2__protect_leaf(hdr, dxpl_id, right_addr, internal->node_ptrs[idx + 1].node_nrec, H5AC__NO_FLAGS_SET))) + left_addr = internal->node_ptrs[idx].addr; + if(NULL == (right_leaf = H5B2__protect_leaf(hdr, dxpl_id, internal, &internal->node_ptrs[idx + 1], FALSE, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") + right_addr = internal->node_ptrs[idx + 1].addr; /* More setup for child nodes */ left_child = left_leaf; @@ -295,7 +265,7 @@ H5B2__split1(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Determine total number of records in new child nodes */ if(depth > 1) { - unsigned u; /* Local index variable */ + unsigned u; /* Local index variable */ hsize_t new_left_all_nrec; /* New total number of records in left child */ hsize_t new_right_all_nrec; /* New total number of records in right child */ @@ -329,6 +299,12 @@ H5B2__split1(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, if(parent_cache_info_flags_ptr) *parent_cache_info_flags_ptr |= H5AC__DIRTIED_FLAG; + /* Update flush dependencies for grandchildren, if using SWMR */ + if(hdr->swmr_write && depth > 1) + if(H5B2__update_child_flush_depends(hdr, dxpl_id, depth, right_node_ptrs, + 0, (unsigned)(*right_nrec + 1), left_child, right_child) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update child nodes to new parent") + #ifdef H5B2_DEBUG H5B2__assert_internal((hsize_t)0, hdr, internal); if(depth > 1) { @@ -407,11 +383,11 @@ H5B2__split_root(H5B2_hdr_t *hdr, hid_t dxpl_id) /* Create new internal node to use as root */ hdr->root.node_nrec = 0; - if(H5B2__create_internal(hdr, dxpl_id, &(hdr->root), hdr->depth) < 0) + if(H5B2__create_internal(hdr, dxpl_id, hdr, &(hdr->root), hdr->depth) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, FAIL, "unable to create new internal node") /* Protect new root node */ - if(NULL == (new_root = H5B2__protect_internal(hdr, dxpl_id, hdr->root.addr, hdr->root.node_nrec, hdr->depth, H5AC__NO_FLAGS_SET))) + if(NULL == (new_root = H5B2__protect_internal(hdr, dxpl_id, hdr, &hdr->root, hdr->depth, FALSE, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") /* Set first node pointer in root node to old root node pointer info */ @@ -444,7 +420,7 @@ done: * *------------------------------------------------------------------------- */ -static herr_t +herr_t H5B2__redistribute2(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, H5B2_internal_t *internal, unsigned idx) { @@ -458,7 +434,7 @@ H5B2__redistribute2(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, unsigned left_child_flags = H5AC__NO_FLAGS_SET, right_child_flags = H5AC__NO_FLAGS_SET; /* Flags for unprotecting child nodes */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC + FUNC_ENTER_PACKAGE /* Check arguments. */ HDassert(hdr); @@ -471,14 +447,15 @@ H5B2__redistribute2(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Setup information for unlocking child nodes */ child_class = H5AC_BT2_INT; - left_addr = internal->node_ptrs[idx].addr; - right_addr = internal->node_ptrs[idx + 1].addr; /* Lock left & right B-tree child nodes */ - if(NULL == (left_internal = H5B2__protect_internal(hdr, dxpl_id, left_addr, internal->node_ptrs[idx].node_nrec, (uint16_t)(depth - 1), H5AC__NO_FLAGS_SET))) + /* (Shadow both nodes if doing SWMR writes) */ + if(NULL == (left_internal = H5B2__protect_internal(hdr, dxpl_id, internal, &internal->node_ptrs[idx], (uint16_t)(depth - 1), hdr->swmr_write, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") - if(NULL == (right_internal = H5B2__protect_internal(hdr, dxpl_id, right_addr, internal->node_ptrs[idx + 1].node_nrec, (uint16_t)(depth - 1), H5AC__NO_FLAGS_SET))) + left_addr = internal->node_ptrs[idx].addr; + if(NULL == (right_internal = H5B2__protect_internal(hdr, dxpl_id, internal, &internal->node_ptrs[idx + 1], (uint16_t)(depth - 1), hdr->swmr_write, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") + right_addr = internal->node_ptrs[idx + 1].addr; /* More setup for child nodes */ left_child = left_internal; @@ -496,14 +473,15 @@ H5B2__redistribute2(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Setup information for unlocking child nodes */ child_class = H5AC_BT2_LEAF; - left_addr = internal->node_ptrs[idx].addr; - right_addr = internal->node_ptrs[idx + 1].addr; /* Lock left & right B-tree child nodes */ - if(NULL == (left_leaf = H5B2__protect_leaf(hdr, dxpl_id, left_addr, internal->node_ptrs[idx].node_nrec, H5AC__NO_FLAGS_SET))) + /* (Shadow both nodes if doing SWMR writes) */ + if(NULL == (left_leaf = H5B2__protect_leaf(hdr, dxpl_id, internal, &internal->node_ptrs[idx], hdr->swmr_write, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") - if(NULL == (right_leaf = H5B2__protect_leaf(hdr, dxpl_id, right_addr, internal->node_ptrs[idx + 1].node_nrec, H5AC__NO_FLAGS_SET))) + left_addr = internal->node_ptrs[idx].addr; + if(NULL == (right_leaf = H5B2__protect_leaf(hdr, dxpl_id, internal, &internal->node_ptrs[idx + 1], hdr->swmr_write, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") + right_addr = internal->node_ptrs[idx + 1].addr; /* More setup for child nodes */ left_child = left_leaf; @@ -549,7 +527,7 @@ H5B2__redistribute2(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Handle node pointers, if we have an internal node */ if(depth > 1) { hsize_t moved_nrec = move_nrec; /* Total number of records moved, for internal redistrib */ - unsigned u; /* Local index variable */ + unsigned u; /* Local index variable */ /* Count the number of records being moved */ for(u = 0; u < move_nrec; u++) @@ -564,6 +542,12 @@ H5B2__redistribute2(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, HDmemmove(&(right_node_ptrs[0]), &(right_node_ptrs[move_nrec]), sizeof(H5B2_node_ptr_t) * (new_right_nrec + (unsigned)1)); } /* end if */ + /* Update flush dependencies for grandchildren, if using SWMR */ + if(hdr->swmr_write && depth > 1) + if(H5B2__update_child_flush_depends(hdr, dxpl_id, depth, left_node_ptrs, + (unsigned)(*left_nrec + 1), (unsigned)(*left_nrec + move_nrec + 1), right_child, left_child) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update child nodes to new parent") + /* Update number of records in child nodes */ *left_nrec = (uint16_t)(*left_nrec + move_nrec); *right_nrec = new_right_nrec; @@ -599,7 +583,7 @@ H5B2__redistribute2(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Handle node pointers, if we have an internal node */ if(depth > 1) { hsize_t moved_nrec = move_nrec; /* Total number of records moved, for internal redistrib */ - unsigned u; /* Local index variable */ + unsigned u; /* Local index variable */ /* Slide node pointers in right node up */ HDmemmove(&(right_node_ptrs[move_nrec]), &(right_node_ptrs[0]), sizeof(H5B2_node_ptr_t) * (size_t)(*right_nrec + 1)); @@ -614,6 +598,12 @@ H5B2__redistribute2(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, H5_CHECKED_ASSIGN(right_moved_nrec, hssize_t, moved_nrec, hsize_t) } /* end if */ + /* Update flush dependencies for grandchildren, if using SWMR */ + if(hdr->swmr_write && depth > 1) + if(H5B2__update_child_flush_depends(hdr, dxpl_id, depth, right_node_ptrs, + 0, (unsigned)move_nrec, left_child, right_child) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update child nodes to new parent") + /* Update number of records in child nodes */ *left_nrec = new_left_nrec; *right_nrec = (uint16_t)(*right_nrec + move_nrec); @@ -674,7 +664,7 @@ done: * *------------------------------------------------------------------------- */ -static herr_t +herr_t H5B2__redistribute3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, H5B2_internal_t *internal, unsigned *internal_flags_ptr, unsigned idx) { @@ -695,7 +685,7 @@ H5B2__redistribute3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, unsigned middle_child_flags = H5AC__NO_FLAGS_SET; /* Flags for unprotecting child nodes */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC + FUNC_ENTER_PACKAGE /* Check arguments. */ HDassert(hdr); @@ -710,17 +700,18 @@ H5B2__redistribute3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Setup information for unlocking child nodes */ child_class = H5AC_BT2_INT; - left_addr = internal->node_ptrs[idx - 1].addr; - middle_addr = internal->node_ptrs[idx].addr; - right_addr = internal->node_ptrs[idx + 1].addr; /* Lock B-tree child nodes */ - if(NULL == (left_internal = H5B2__protect_internal(hdr, dxpl_id, left_addr, internal->node_ptrs[idx - 1].node_nrec, (uint16_t)(depth - 1), H5AC__NO_FLAGS_SET))) + /* (Shadow all nodes if doing SWMR writes) */ + if(NULL == (left_internal = H5B2__protect_internal(hdr, dxpl_id, internal, &internal->node_ptrs[idx - 1], (uint16_t)(depth - 1), hdr->swmr_write, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") - if(NULL == (middle_internal = H5B2__protect_internal(hdr, dxpl_id, middle_addr, internal->node_ptrs[idx].node_nrec, (uint16_t)(depth - 1), H5AC__NO_FLAGS_SET))) + left_addr = internal->node_ptrs[idx - 1].addr; + if(NULL == (middle_internal = H5B2__protect_internal(hdr, dxpl_id, internal, &internal->node_ptrs[idx], (uint16_t)(depth - 1), hdr->swmr_write, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") - if(NULL == (right_internal = H5B2__protect_internal(hdr, dxpl_id, right_addr, internal->node_ptrs[idx + 1].node_nrec, (uint16_t)(depth - 1), H5AC__NO_FLAGS_SET))) + middle_addr = internal->node_ptrs[idx].addr; + if(NULL == (right_internal = H5B2__protect_internal(hdr, dxpl_id, internal, &internal->node_ptrs[idx + 1], (uint16_t)(depth - 1), hdr->swmr_write, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") + right_addr = internal->node_ptrs[idx + 1].addr; /* More setup for child nodes */ left_child = left_internal; @@ -743,17 +734,18 @@ H5B2__redistribute3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Setup information for unlocking child nodes */ child_class = H5AC_BT2_LEAF; - left_addr = internal->node_ptrs[idx - 1].addr; - middle_addr = internal->node_ptrs[idx].addr; - right_addr = internal->node_ptrs[idx + 1].addr; /* Lock B-tree child nodes */ - if(NULL == (left_leaf = H5B2__protect_leaf(hdr, dxpl_id, left_addr, internal->node_ptrs[idx - 1].node_nrec, H5AC__NO_FLAGS_SET))) + /* (Shadow all nodes if doing SWMR writes) */ + if(NULL == (left_leaf = H5B2__protect_leaf(hdr, dxpl_id, internal, &internal->node_ptrs[idx - 1], hdr->swmr_write, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") - if(NULL == (middle_leaf = H5B2__protect_leaf(hdr, dxpl_id, middle_addr, internal->node_ptrs[idx].node_nrec, H5AC__NO_FLAGS_SET))) + left_addr = internal->node_ptrs[idx - 1].addr; + if(NULL == (middle_leaf = H5B2__protect_leaf(hdr, dxpl_id, internal, &internal->node_ptrs[idx], hdr->swmr_write, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") - if(NULL == (right_leaf = H5B2__protect_leaf(hdr, dxpl_id, right_addr, internal->node_ptrs[idx + 1].node_nrec, H5AC__NO_FLAGS_SET))) + middle_addr = internal->node_ptrs[idx].addr; + if(NULL == (right_leaf = H5B2__protect_leaf(hdr, dxpl_id, internal, &internal->node_ptrs[idx + 1], hdr->swmr_write, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") + right_addr = internal->node_ptrs[idx + 1].addr; /* More setup for child nodes */ left_child = left_leaf; @@ -802,7 +794,7 @@ H5B2__redistribute3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Move node pointers also if this is an internal node */ if(depth > 1) { - hsize_t moved_nrec; /* Total number of records moved, for internal redistrib */ + hsize_t moved_nrec; /* Total number of records moved, for internal redistrib */ unsigned move_nptrs; /* Number of node pointers to move */ unsigned u; /* Local index variable */ @@ -820,6 +812,12 @@ H5B2__redistribute3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, HDmemmove(&(middle_node_ptrs[0]), &(middle_node_ptrs[move_nptrs]), sizeof(H5B2_node_ptr_t) * ((*middle_nrec - move_nptrs) + 1)); } /* end if */ + /* Update flush dependencies for grandchildren, if using SWMR */ + if(hdr->swmr_write && depth > 1) + if(H5B2__update_child_flush_depends(hdr, dxpl_id, depth, left_node_ptrs, + (unsigned)(*left_nrec + 1), (unsigned)(*left_nrec + moved_middle_nrec + 1), middle_child, left_child) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update child nodes to new parent") + /* Update the current number of records in middle node */ curr_middle_nrec = (uint16_t)(curr_middle_nrec - moved_middle_nrec); @@ -847,7 +845,7 @@ H5B2__redistribute3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Move node pointers also if this is an internal node */ if(depth > 1) { - hsize_t moved_nrec; /* Total number of records moved, for internal redistrib */ + hsize_t moved_nrec; /* Total number of records moved, for internal redistrib */ unsigned u; /* Local index variable */ /* Slide the node pointers in right node up */ @@ -863,6 +861,12 @@ H5B2__redistribute3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, middle_moved_nrec -= (hssize_t)(moved_nrec + right_nrec_move); } /* end if */ + /* Update flush dependencies for grandchildren, if using SWMR */ + if(hdr->swmr_write && depth > 1) + if(H5B2__update_child_flush_depends(hdr, dxpl_id, depth, right_node_ptrs, + 0, (unsigned)right_nrec_move, middle_child, right_child) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update child nodes to new parent") + /* Update the current number of records in middle node */ curr_middle_nrec = (uint16_t)(curr_middle_nrec - right_nrec_move); @@ -890,7 +894,7 @@ H5B2__redistribute3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Move node pointers also if this is an internal node */ if(depth > 1) { - hsize_t moved_nrec; /* Total number of records moved, for internal redistrib */ + hsize_t moved_nrec; /* Total number of records moved, for internal redistrib */ unsigned u; /* Local index variable */ /* Slide the node pointers in middle node up */ @@ -906,6 +910,12 @@ H5B2__redistribute3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, middle_moved_nrec += (hssize_t)(moved_nrec + left_nrec_move); } /* end if */ + /* Update flush dependencies for grandchildren, if using SWMR */ + if(hdr->swmr_write && depth > 1) + if(H5B2__update_child_flush_depends(hdr, dxpl_id, depth, middle_node_ptrs, + 0, (unsigned)left_nrec_move, left_child, middle_child) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update child nodes to new parent") + /* Update the current number of records in middle node */ curr_middle_nrec = (uint16_t)(curr_middle_nrec + left_nrec_move); @@ -932,7 +942,7 @@ H5B2__redistribute3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Move node pointers also if this is an internal node */ if(depth > 1) { - hsize_t moved_nrec; /* Total number of records moved, for internal redistrib */ + hsize_t moved_nrec; /* Total number of records moved, for internal redistrib */ unsigned u; /* Local index variable */ /* Move right node pointers into middle node */ @@ -948,6 +958,12 @@ H5B2__redistribute3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, HDmemmove(&(right_node_ptrs[0]), &(right_node_ptrs[right_nrec_move]), sizeof(H5B2_node_ptr_t) * (size_t)(new_right_nrec + 1)); } /* end if */ + /* Update flush dependencies for grandchildren, if using SWMR */ + if(hdr->swmr_write && depth > 1) + if(H5B2__update_child_flush_depends(hdr, dxpl_id, depth, middle_node_ptrs, + (unsigned)(curr_middle_nrec + 1), (unsigned)(curr_middle_nrec + right_nrec_move + 1), right_child, middle_child) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update child nodes to new parent") + /* Mark nodes as dirty */ middle_child_flags |= H5AC__DIRTIED_FLAG; right_child_flags |= H5AC__DIRTIED_FLAG; @@ -979,46 +995,6 @@ H5B2__redistribute3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Mark parent as dirty */ *internal_flags_ptr |= H5AC__DIRTIED_FLAG; -#ifdef QAK -{ - unsigned u; - - HDfprintf(stderr, "%s: Internal records:\n", FUNC); - for(u = 0; u < internal->nrec; u++) { - HDfprintf(stderr, "%s: u = %u\n", FUNC, u); - (hdr->cls->debug)(stderr, hdr->f, dxpl_id, 3, 4, H5B2_INT_NREC(internal, hdr, u), NULL); - } /* end for */ - - HDfprintf(stderr, "%s: Left Child records:\n", FUNC); - for(u = 0; u < *left_nrec; u++) { - HDfprintf(stderr, "%s: u = %u\n", FUNC, u); - (hdr->cls->debug)(stderr, hdr->f, dxpl_id, 3, 4, H5B2_NAT_NREC(left_native, hdr, u), NULL); - } /* end for */ - - HDfprintf(stderr, "%s: Middle Child records:\n", FUNC); - for(u = 0; u < *middle_nrec; u++) { - HDfprintf(stderr, "%s: u = %u\n", FUNC, u); - (hdr->cls->debug)(stderr, hdr->f, dxpl_id, 3, 4, H5B2_NAT_NREC(middle_native, hdr, u), NULL); - } /* end for */ - - HDfprintf(stderr, "%s: Right Child records:\n", FUNC); - for(u = 0; u < *right_nrec; u++) { - HDfprintf(stderr, "%s: u = %u\n", FUNC, u); - (hdr->cls->debug)(stderr, hdr->f, dxpl_id, 3, 4, H5B2_NAT_NREC(right_native, hdr, u), NULL); - } /* end for */ - - for(u = 0; u < internal->nrec + 1; u++) - HDfprintf(stderr, "%s: internal->node_ptrs[%u] = (%Hu/%u/%a)\n", FUNC, u, internal->node_ptrs[u].all_nrec, internal->node_ptrs[u].node_nrec, internal->node_ptrs[u].addr); - if(depth > 1) { - for(u = 0; u < *left_nrec + 1; u++) - HDfprintf(stderr, "%s: left_node_ptr[%u] = (%Hu/%u/%a)\n", FUNC, u, left_node_ptrs[u].all_nrec, left_node_ptrs[u].node_nrec, left_node_ptrs[u].addr); - for(u = 0; u < *middle_nrec + 1; u++) - HDfprintf(stderr, "%s: middle_node_ptr[%u] = (%Hu/%u/%a)\n", FUNC, u, middle_node_ptrs[u].all_nrec, middle_node_ptrs[u].node_nrec, middle_node_ptrs[u].addr); - for(u = 0; u < *right_nrec + 1; u++) - HDfprintf(stderr, "%s: right_node_ptr[%u] = (%Hu/%u/%a)\n", FUNC, u, right_node_ptrs[u].all_nrec, right_node_ptrs[u].node_nrec, right_node_ptrs[u].addr); - } /* end if */ -} -#endif /* QAK */ #ifdef H5B2_DEBUG H5B2__assert_internal((hsize_t)0, hdr, internal); if(depth > 1) { @@ -1062,7 +1038,7 @@ done: * *------------------------------------------------------------------------- */ -static herr_t +herr_t H5B2__merge2(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, H5B2_node_ptr_t *curr_node_ptr, unsigned *parent_cache_info_flags_ptr, H5B2_internal_t *internal, unsigned *internal_flags_ptr, unsigned idx) @@ -1076,7 +1052,7 @@ H5B2__merge2(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, unsigned left_child_flags = H5AC__NO_FLAGS_SET, right_child_flags = H5AC__NO_FLAGS_SET; /* Flags for unprotecting child nodes */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC + FUNC_ENTER_PACKAGE /* Check arguments. */ HDassert(hdr); @@ -1091,14 +1067,15 @@ H5B2__merge2(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Setup information for unlocking child nodes */ child_class = H5AC_BT2_INT; - left_addr = internal->node_ptrs[idx].addr; - right_addr = internal->node_ptrs[idx + 1].addr; /* Lock left & right B-tree child nodes */ - if(NULL == (left_internal = H5B2__protect_internal(hdr, dxpl_id, left_addr, internal->node_ptrs[idx].node_nrec, (uint16_t)(depth - 1), H5AC__NO_FLAGS_SET))) + /* (Shadow the left node if doing SWMR writes) */ + if(NULL == (left_internal = H5B2__protect_internal(hdr, dxpl_id, internal, &internal->node_ptrs[idx], (uint16_t)(depth - 1), hdr->swmr_write, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") - if(NULL == (right_internal = H5B2__protect_internal(hdr, dxpl_id, right_addr, internal->node_ptrs[idx + 1].node_nrec, (uint16_t)(depth - 1), H5AC__NO_FLAGS_SET))) + left_addr = internal->node_ptrs[idx].addr; + if(NULL == (right_internal = H5B2__protect_internal(hdr, dxpl_id, internal, &internal->node_ptrs[idx + 1], (uint16_t)(depth - 1), FALSE, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") + right_addr = internal->node_ptrs[idx + 1].addr; /* More setup for accessing child node information */ left_child = left_internal; @@ -1116,14 +1093,15 @@ H5B2__merge2(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Setup information for unlocking child nodes */ child_class = H5AC_BT2_LEAF; - left_addr = internal->node_ptrs[idx].addr; - right_addr = internal->node_ptrs[idx + 1].addr; /* Lock left & right B-tree child nodes */ - if(NULL == (left_leaf = H5B2__protect_leaf(hdr, dxpl_id, left_addr, internal->node_ptrs[idx].node_nrec, H5AC__NO_FLAGS_SET))) + /* (Shadow the left node if doing SWMR writes) */ + if(NULL == (left_leaf = H5B2__protect_leaf(hdr, dxpl_id, internal, &internal->node_ptrs[idx], hdr->swmr_write, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") - if(NULL == (right_leaf = H5B2__protect_leaf(hdr, dxpl_id, right_addr, internal->node_ptrs[idx + 1].node_nrec, H5AC__NO_FLAGS_SET))) + left_addr = internal->node_ptrs[idx].addr; + if(NULL == (right_leaf = H5B2__protect_leaf(hdr, dxpl_id, internal, &internal->node_ptrs[idx + 1], FALSE, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") + right_addr = internal->node_ptrs[idx + 1].addr; /* More setup for accessing child node information */ left_child = left_leaf; @@ -1146,12 +1124,20 @@ H5B2__merge2(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, if(depth > 1) HDmemcpy(&(left_node_ptrs[*left_nrec + 1]), &(right_node_ptrs[0]), sizeof(H5B2_node_ptr_t) * (size_t)(*right_nrec + 1)); + /* Update flush dependencies for grandchildren, if using SWMR */ + if(hdr->swmr_write && depth > 1) + if(H5B2__update_child_flush_depends(hdr, dxpl_id, depth, left_node_ptrs, + (unsigned)(*left_nrec + 1), (unsigned)(*left_nrec + *right_nrec + 2), right_child, left_child) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update child nodes to new parent") + /* Update # of records in left node */ *left_nrec = (uint16_t)(*left_nrec + *right_nrec + 1); /* Mark nodes as dirty */ left_child_flags |= H5AC__DIRTIED_FLAG; - right_child_flags |= H5AC__DIRTIED_FLAG | H5AC__DELETED_FLAG | H5AC__FREE_FILE_SPACE_FLAG; + right_child_flags |= H5AC__DELETED_FLAG; + if(!(hdr->swmr_write)) + right_child_flags |= H5AC__DIRTIED_FLAG | H5AC__FREE_FILE_SPACE_FLAG; } /* end block */ /* Update # of records in child nodes */ @@ -1215,7 +1201,7 @@ done: * *------------------------------------------------------------------------- */ -static herr_t +herr_t H5B2__merge3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, H5B2_node_ptr_t *curr_node_ptr, unsigned *parent_cache_info_flags_ptr, H5B2_internal_t *internal, unsigned *internal_flags_ptr, unsigned idx) @@ -1236,7 +1222,7 @@ H5B2__merge3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, unsigned middle_child_flags = H5AC__NO_FLAGS_SET; /* Flags for unprotecting child nodes */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC + FUNC_ENTER_PACKAGE /* Check arguments. */ HDassert(hdr); @@ -1252,17 +1238,18 @@ H5B2__merge3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Setup information for unlocking child nodes */ child_class = H5AC_BT2_INT; - left_addr = internal->node_ptrs[idx - 1].addr; - middle_addr = internal->node_ptrs[idx].addr; - right_addr = internal->node_ptrs[idx + 1].addr; /* Lock B-tree child nodes */ - if(NULL == (left_internal = H5B2__protect_internal(hdr, dxpl_id, left_addr, internal->node_ptrs[idx - 1].node_nrec, (uint16_t)(depth - 1), H5AC__NO_FLAGS_SET))) + /* (Shadow left and middle nodes if doing SWMR writes) */ + if(NULL == (left_internal = H5B2__protect_internal(hdr, dxpl_id, internal, &internal->node_ptrs[idx - 1], (uint16_t)(depth - 1), hdr->swmr_write, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") - if(NULL == (middle_internal = H5B2__protect_internal(hdr, dxpl_id, middle_addr, internal->node_ptrs[idx].node_nrec, (uint16_t)(depth - 1), H5AC__NO_FLAGS_SET))) + left_addr = internal->node_ptrs[idx - 1].addr; + if(NULL == (middle_internal = H5B2__protect_internal(hdr, dxpl_id, internal, &internal->node_ptrs[idx], (uint16_t)(depth - 1), hdr->swmr_write, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") - if(NULL == (right_internal = H5B2__protect_internal(hdr, dxpl_id, right_addr, internal->node_ptrs[idx + 1].node_nrec, (uint16_t)(depth - 1), H5AC__NO_FLAGS_SET))) + middle_addr = internal->node_ptrs[idx].addr; + if(NULL == (right_internal = H5B2__protect_internal(hdr, dxpl_id, internal, &internal->node_ptrs[idx + 1], (uint16_t)(depth - 1), FALSE, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") + right_addr = internal->node_ptrs[idx + 1].addr; /* More setup for accessing child node information */ left_child = left_internal; @@ -1285,17 +1272,18 @@ H5B2__merge3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Setup information for unlocking child nodes */ child_class = H5AC_BT2_LEAF; - left_addr = internal->node_ptrs[idx - 1].addr; - middle_addr = internal->node_ptrs[idx].addr; - right_addr = internal->node_ptrs[idx + 1].addr; /* Lock B-tree child nodes */ - if(NULL == (left_leaf = H5B2__protect_leaf(hdr, dxpl_id, left_addr, internal->node_ptrs[idx - 1].node_nrec, H5AC__NO_FLAGS_SET))) + /* (Shadow left and middle nodes if doing SWMR writes) */ + if(NULL == (left_leaf = H5B2__protect_leaf(hdr, dxpl_id, internal, &internal->node_ptrs[idx - 1], hdr->swmr_write, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") - if(NULL == (middle_leaf = H5B2__protect_leaf(hdr, dxpl_id, middle_addr, internal->node_ptrs[idx].node_nrec, H5AC__NO_FLAGS_SET))) + left_addr = internal->node_ptrs[idx - 1].addr; + if(NULL == (middle_leaf = H5B2__protect_leaf(hdr, dxpl_id, internal, &internal->node_ptrs[idx], hdr->swmr_write, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") - if(NULL == (right_leaf = H5B2__protect_leaf(hdr, dxpl_id, right_addr, internal->node_ptrs[idx + 1].node_nrec, H5AC__NO_FLAGS_SET))) + middle_addr = internal->node_ptrs[idx].addr; + if(NULL == (right_leaf = H5B2__protect_leaf(hdr, dxpl_id, internal, &internal->node_ptrs[idx + 1], FALSE, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") + right_addr = internal->node_ptrs[idx + 1].addr; /* More setup for accessing child node information */ left_child = left_leaf; @@ -1344,6 +1332,12 @@ H5B2__merge3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, HDmemmove(&(middle_node_ptrs[0]), &(middle_node_ptrs[middle_nrec_move]), sizeof(H5B2_node_ptr_t) * (size_t)((unsigned)(*middle_nrec + 1) - middle_nrec_move)); } /* end if */ + /* Update flush dependencies for grandchildren, if using SWMR */ + if(hdr->swmr_write && depth > 1) + if(H5B2__update_child_flush_depends(hdr, dxpl_id, depth, left_node_ptrs, + (unsigned)(*left_nrec + 1), (unsigned)(*left_nrec + middle_nrec_move + 1), middle_child, left_child) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update child nodes to new parent") + /* Update # of records in left & middle nodes */ *left_nrec = (uint16_t)(*left_nrec + middle_nrec_move); *middle_nrec = (uint16_t)(*middle_nrec - middle_nrec_move); @@ -1366,12 +1360,20 @@ H5B2__merge3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Copy node pointers from right node into middle node */ HDmemcpy(&(middle_node_ptrs[*middle_nrec + 1]), &(right_node_ptrs[0]), sizeof(H5B2_node_ptr_t) * (size_t)(*right_nrec + 1)); + /* Update flush dependencies for grandchildren, if using SWMR */ + if(hdr->swmr_write && depth > 1) + if(H5B2__update_child_flush_depends(hdr, dxpl_id, depth, middle_node_ptrs, + (unsigned)(*middle_nrec + 1), (unsigned)(*middle_nrec + *right_nrec + 2), right_child, middle_child) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update child nodes to new parent") + /* Update # of records in middle node */ *middle_nrec = (uint16_t)(*middle_nrec + (*right_nrec + 1)); /* Mark nodes as dirty */ middle_child_flags |= H5AC__DIRTIED_FLAG; - right_child_flags |= H5AC__DIRTIED_FLAG | H5AC__DELETED_FLAG | H5AC__FREE_FILE_SPACE_FLAG; + right_child_flags |= H5AC__DELETED_FLAG; + if(!(hdr->swmr_write)) + right_child_flags |= H5AC__DIRTIED_FLAG | H5AC__FREE_FILE_SPACE_FLAG; } /* end block */ /* Update # of records in child nodes */ @@ -1429,98 +1431,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5B2__swap_leaf - * - * Purpose: Swap a record in a node with a record in a leaf node - * - * Return: Success: Non-negative - * - * Failure: Negative - * - * Programmer: Quincey Koziol - * koziol@ncsa.uiuc.edu - * Mar 4 2005 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5B2__swap_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, - H5B2_internal_t *internal, unsigned *internal_flags_ptr, - unsigned idx, void *swap_loc) -{ - const H5AC_class_t *child_class; /* Pointer to child node's class info */ - haddr_t child_addr; /* Address of child node */ - void *child = NULL; /* Pointer to child node */ - uint8_t *child_native; /* Pointer to child's native records */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Check arguments. */ - HDassert(hdr); - HDassert(internal); - HDassert(internal_flags_ptr); - HDassert(idx <= internal->nrec); - - /* Check for the kind of B-tree node to swap */ - if(depth > 1) { - H5B2_internal_t *child_internal; /* Pointer to internal node */ - - /* Setup information for unlocking child node */ - child_class = H5AC_BT2_INT; - child_addr = internal->node_ptrs[idx].addr; - - /* Lock B-tree child nodes */ - if(NULL == (child_internal = H5B2__protect_internal(hdr, dxpl_id, child_addr, internal->node_ptrs[idx].node_nrec, (uint16_t)(depth - 1), H5AC__NO_FLAGS_SET))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") - - /* More setup for accessing child node information */ - child = child_internal; - child_native = child_internal->int_native; - } /* end if */ - else { - H5B2_leaf_t *child_leaf; /* Pointer to leaf node */ - - /* Setup information for unlocking child nodes */ - child_class = H5AC_BT2_LEAF; - child_addr = internal->node_ptrs[idx].addr; - - /* Lock B-tree child node */ - if(NULL == (child_leaf = H5B2__protect_leaf(hdr, dxpl_id, child_addr, internal->node_ptrs[idx].node_nrec, H5AC__NO_FLAGS_SET))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") - - /* More setup for accessing child node information */ - child = child_leaf; - child_native = child_leaf->leaf_native; - } /* end else */ - - /* Swap records (use disk page as temporary buffer) */ - HDmemcpy(hdr->page, H5B2_NAT_NREC(child_native, hdr, 0), hdr->cls->nrec_size); - HDmemcpy(H5B2_NAT_NREC(child_native, hdr, 0), swap_loc, hdr->cls->nrec_size); - HDmemcpy(swap_loc, hdr->page, hdr->cls->nrec_size); - - /* Mark parent as dirty */ - *internal_flags_ptr |= H5AC__DIRTIED_FLAG; - -#ifdef H5B2_DEBUG - H5B2__assert_internal((hsize_t)0, hdr, internal); - if(depth > 1) - H5B2__assert_internal(internal->node_ptrs[idx].all_nrec, hdr, (H5B2_internal_t *)child); - else - H5B2__assert_leaf(hdr, (H5B2_leaf_t *)child); -#endif /* H5B2_DEBUG */ - -done: - /* Unlock child node */ - if(child && H5AC_unprotect(hdr->f, dxpl_id, child_class, child_addr, child, H5AC__DIRTIED_FLAG) < 0) - HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree child node") - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5B2__swap_leaf() */ - - -/*------------------------------------------------------------------------- - * Function: H5B2__insert_hdr + * Function: H5B2__insert * * Purpose: Adds a new record to the B-tree. * @@ -1533,7 +1444,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5B2__insert_hdr(H5B2_hdr_t *hdr, hid_t dxpl_id, void *udata) +H5B2__insert(H5B2_hdr_t *hdr, hid_t dxpl_id, void *udata) { herr_t ret_value = SUCCEED; /* Return value */ @@ -1546,7 +1457,7 @@ H5B2__insert_hdr(H5B2_hdr_t *hdr, hid_t dxpl_id, void *udata) /* Check if the root node is allocated yet */ if(!H5F_addr_defined(hdr->root.addr)) { /* Create root node as leaf node in B-tree */ - if(H5B2__create_leaf(hdr, dxpl_id, &(hdr->root)) < 0) + if(H5B2__create_leaf(hdr, dxpl_id, hdr, &(hdr->root)) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, FAIL, "unable to create root node") } /* end if */ /* Check if we need to split the root node (equiv. to a 1->2 node split) */ @@ -1558,11 +1469,11 @@ H5B2__insert_hdr(H5B2_hdr_t *hdr, hid_t dxpl_id, void *udata) /* Attempt to insert record into B-tree */ if(hdr->depth > 0) { - if(H5B2__insert_internal(hdr, dxpl_id, hdr->depth, NULL, &hdr->root, H5B2_POS_ROOT, udata) < 0) + if(H5B2__insert_internal(hdr, dxpl_id, hdr->depth, NULL, &hdr->root, H5B2_POS_ROOT, hdr, udata) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, FAIL, "unable to insert record into B-tree internal node") } /* end if */ else { - if(H5B2__insert_leaf(hdr, dxpl_id, &hdr->root, H5B2_POS_ROOT, udata) < 0) + if(H5B2__insert_leaf(hdr, dxpl_id, &hdr->root, H5B2_POS_ROOT, hdr, udata) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, FAIL, "unable to insert record into B-tree leaf node") } /* end else */ @@ -1572,834 +1483,7 @@ H5B2__insert_hdr(H5B2_hdr_t *hdr, hid_t dxpl_id, void *udata) done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5B2__insert_hdr() */ - - -/*------------------------------------------------------------------------- - * Function: H5B2__insert_leaf - * - * Purpose: Adds a new record to a B-tree leaf node. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * koziol@ncsa.uiuc.edu - * Mar 3 2005 - * - *------------------------------------------------------------------------- - */ -herr_t -H5B2__insert_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, H5B2_node_ptr_t *curr_node_ptr, - H5B2_nodepos_t curr_pos, void *udata) -{ - H5B2_leaf_t *leaf; /* Pointer to leaf node */ - int cmp; /* Comparison value of records */ - unsigned idx; /* Location of record which matches key */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Check arguments. */ - HDassert(hdr); - HDassert(curr_node_ptr); - HDassert(H5F_addr_defined(curr_node_ptr->addr)); - - /* Lock current B-tree node */ - if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, curr_node_ptr->addr, curr_node_ptr->node_nrec, H5AC__NO_FLAGS_SET))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") - - /* Must have a leaf node with enough space to insert a record now */ - HDassert(curr_node_ptr->node_nrec < hdr->node_info[0].max_nrec); - - /* Sanity check number of records */ - HDassert(curr_node_ptr->all_nrec == curr_node_ptr->node_nrec); - HDassert(leaf->nrec == curr_node_ptr->node_nrec); - - /* Check for inserting into empty leaf */ - if(leaf->nrec == 0) - idx = 0; - else { - /* Find correct location to insert this record */ - if(H5B2__locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx, &cmp) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") - if(cmp == 0) - HGOTO_ERROR(H5E_BTREE, H5E_EXISTS, FAIL, "record is already in B-tree") - if(cmp > 0) - idx++; - - /* Make room for new record */ - if(idx < leaf->nrec) - HDmemmove(H5B2_LEAF_NREC(leaf, hdr, idx + 1), H5B2_LEAF_NREC(leaf, hdr, idx), hdr->cls->nrec_size * (leaf->nrec - idx)); - } /* end else */ - - /* Make callback to store record in native form */ - if((hdr->cls->store)(H5B2_LEAF_NREC(leaf, hdr, idx), udata) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, FAIL, "unable to insert record into leaf node") - - /* Update record count for node pointer to current node */ - curr_node_ptr->all_nrec++; - curr_node_ptr->node_nrec++; - - /* Update record count for current node */ - leaf->nrec++; - - /* Check for new record being the min or max for the tree */ - /* (Don't use 'else' for the idx check, to allow for root leaf node) */ - if(H5B2_POS_MIDDLE != curr_pos) { - if(idx == 0) { - if(H5B2_POS_LEFT == curr_pos || H5B2_POS_ROOT == curr_pos) { - if(hdr->min_native_rec == NULL) - if(NULL == (hdr->min_native_rec = H5MM_malloc(hdr->cls->nrec_size))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTALLOC, FAIL, "memory allocation failed for v2 B-tree min record info") - HDmemcpy(hdr->min_native_rec, H5B2_LEAF_NREC(leaf, hdr, idx), hdr->cls->nrec_size); - } /* end if */ - } /* end if */ - if(idx == (unsigned)(leaf->nrec - 1)) { - if(H5B2_POS_RIGHT == curr_pos || H5B2_POS_ROOT == curr_pos) { - if(hdr->max_native_rec == NULL) - if(NULL == (hdr->max_native_rec = H5MM_malloc(hdr->cls->nrec_size))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTALLOC, FAIL, "memory allocation failed for v2 B-tree max record info") - HDmemcpy(hdr->max_native_rec, H5B2_LEAF_NREC(leaf, hdr, idx), hdr->cls->nrec_size); - } /* end if */ - } /* end if */ - } /* end if */ - -done: - /* Release the B-tree leaf node (marked as dirty) */ - if(leaf && H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_LEAF, curr_node_ptr->addr, leaf, H5AC__DIRTIED_FLAG) < 0) - HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release leaf B-tree node") - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5B2__insert_leaf() */ - - -/*------------------------------------------------------------------------- - * Function: H5B2__insert_internal - * - * Purpose: Adds a new record to a B-tree node. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * koziol@ncsa.uiuc.edu - * Mar 2 2005 - * - *------------------------------------------------------------------------- - */ -herr_t -H5B2__insert_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, - unsigned *parent_cache_info_flags_ptr, H5B2_node_ptr_t *curr_node_ptr, - H5B2_nodepos_t curr_pos, void *udata) -{ - H5B2_internal_t *internal = NULL; /* Pointer to internal node */ - unsigned internal_flags = H5AC__NO_FLAGS_SET; - unsigned idx; /* Location of record which matches key */ - H5B2_nodepos_t next_pos = H5B2_POS_MIDDLE; /* Position of node */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Check arguments. */ - HDassert(hdr); - HDassert(depth > 0); - HDassert(curr_node_ptr); - HDassert(H5F_addr_defined(curr_node_ptr->addr)); - - /* Lock current B-tree node */ - if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, curr_node_ptr->addr, curr_node_ptr->node_nrec, depth, H5AC__NO_FLAGS_SET))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") - - /* Sanity check number of records */ - HDassert(internal->nrec == curr_node_ptr->node_nrec); - - /* Split or redistribute child node pointers, if necessary */ - { - int cmp; /* Comparison value of records */ - unsigned retries; /* Number of times to attempt redistribution */ - size_t split_nrec; /* Number of records to split node at */ - - /* Locate node pointer for child */ - if(H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, - udata, &idx, &cmp) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") - if(cmp == 0) - HGOTO_ERROR(H5E_BTREE, H5E_EXISTS, FAIL, "record is already in B-tree") - if(cmp > 0) - idx++; - - /* Set the number of redistribution retries */ - /* This takes care of the case where a B-tree node needs to be - * redistributed, but redistributing the node causes the index - * for insertion to move to another node, which also needs to be - * redistributed. Now, we loop trying to redistribute and then - * eventually force a split */ - retries = 2; - - /* Determine the correct number of records to split child node at */ - split_nrec = hdr->node_info[depth - 1].split_nrec; - - /* Preemptively split/redistribute a node we will enter */ - while(internal->node_ptrs[idx].node_nrec == split_nrec) { - /* Attempt to redistribute records among children */ - if(idx == 0) { /* Left-most child */ - if(retries > 0 && (internal->node_ptrs[idx + 1].node_nrec < split_nrec)) { - if(H5B2__redistribute2(hdr, dxpl_id, depth, internal, idx) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTREDISTRIBUTE, FAIL, "unable to redistribute child node records") - } /* end if */ - else { - if(H5B2__split1(hdr, dxpl_id, depth, curr_node_ptr, - parent_cache_info_flags_ptr, internal, &internal_flags, idx) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to split child node") - } /* end else */ - } /* end if */ - else if(idx == internal->nrec) { /* Right-most child */ - if(retries > 0 && (internal->node_ptrs[idx - 1].node_nrec < split_nrec)) { - if(H5B2__redistribute2(hdr, dxpl_id, depth, internal, (idx - 1)) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTREDISTRIBUTE, FAIL, "unable to redistribute child node records") - } /* end if */ - else { - if(H5B2__split1(hdr, dxpl_id, depth, curr_node_ptr, - parent_cache_info_flags_ptr, internal, &internal_flags, idx) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to split child node") - } /* end else */ - } /* end if */ - else { /* Middle child */ - if(retries > 0 && ((internal->node_ptrs[idx + 1].node_nrec < split_nrec) || - (internal->node_ptrs[idx - 1].node_nrec < split_nrec))) { - if(H5B2__redistribute3(hdr, dxpl_id, depth, internal, &internal_flags, idx) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTREDISTRIBUTE, FAIL, "unable to redistribute child node records") - } /* end if */ - else { - if(H5B2__split1(hdr, dxpl_id, depth, curr_node_ptr, - parent_cache_info_flags_ptr, internal, &internal_flags, idx) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to split child node") - } /* end else */ - } /* end else */ - - /* Locate node pointer for child (after split/redistribute) */ - /* Actually, this can be easily updated (for 2-node redistrib.) and shouldn't require re-searching */ - if(H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, - udata, &idx, &cmp) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") - if(cmp == 0) - HGOTO_ERROR(H5E_BTREE, H5E_EXISTS, FAIL, "record is already in B-tree") - if(cmp > 0) - idx++; - - /* Decrement the number of redistribution retries left */ - retries--; - } /* end while */ - } /* end block */ - - /* Check if this node is left/right-most */ - if(H5B2_POS_MIDDLE != curr_pos) { - if(idx == 0) { - if(H5B2_POS_LEFT == curr_pos || H5B2_POS_ROOT == curr_pos) - next_pos = H5B2_POS_LEFT; - } /* end if */ - else if(idx == internal->nrec) { - if(H5B2_POS_RIGHT == curr_pos || H5B2_POS_ROOT == curr_pos) - next_pos = H5B2_POS_RIGHT; - } /* end else */ - } /* end if */ - - /* Attempt to insert node */ - if(depth > 1) { - if(H5B2__insert_internal(hdr, dxpl_id, (uint16_t)(depth - 1), &internal_flags, &internal->node_ptrs[idx], next_pos, udata) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, FAIL, "unable to insert record into B-tree internal node") - } /* end if */ - else { - if(H5B2__insert_leaf(hdr, dxpl_id, &internal->node_ptrs[idx], next_pos, udata) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, FAIL, "unable to insert record into B-tree leaf node") - } /* end else */ - - /* Update record count for node pointer to current node */ - curr_node_ptr->all_nrec++; - - /* Mark node as dirty */ - internal_flags |= H5AC__DIRTIED_FLAG; - -done: - /* Release the B-tree internal node */ - if(internal && H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr->addr, internal, internal_flags) < 0) - HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release internal B-tree node") - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5B2__insert_internal() */ - - -/*------------------------------------------------------------------------- - * Function: H5B2__update_leaf - * - * Purpose: Insert or modify a record in a B-tree leaf node. - * If the record exists already, it is modified as if H5B2_modify - * was called). If it doesn't exist, it is inserted as if - * H5B2_insert was called. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * koziol@hdfgroup.org - * Dec 23 2015 - * - *------------------------------------------------------------------------- - */ -herr_t -H5B2__update_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, H5B2_node_ptr_t *curr_node_ptr, - H5B2_update_status_t *status, H5B2_nodepos_t curr_pos, void *udata, - H5B2_modify_t op, void *op_data) -{ - H5B2_leaf_t *leaf; /* Pointer to leaf node */ - unsigned leaf_flags = H5AC__NO_FLAGS_SET; /* Flags for unprotecting the leaf node */ - int cmp = -1; /* Comparison value of records */ - unsigned idx; /* Location of record which matches key */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Check arguments. */ - HDassert(hdr); - HDassert(curr_node_ptr); - HDassert(H5F_addr_defined(curr_node_ptr->addr)); - - /* Lock current B-tree node */ - if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, curr_node_ptr->addr, curr_node_ptr->node_nrec, H5AC__NO_FLAGS_SET))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") - - /* Sanity check number of records */ - HDassert(curr_node_ptr->all_nrec == curr_node_ptr->node_nrec); - HDassert(leaf->nrec == curr_node_ptr->node_nrec); - - /* Check for inserting into empty leaf */ - if(leaf->nrec == 0) - idx = 0; - else { - /* Find correct location to insert this record */ - if(H5B2__locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx, &cmp) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") - - /* Check for inserting a record */ - if(0 != cmp) { - /* Check if the leaf node is full */ - if(curr_node_ptr->node_nrec == hdr->node_info[0].split_nrec) { - /* Indicate that the leaf is full, but we need to insert */ - *status = H5B2_UPDATE_INSERT_CHILD_FULL; - - /* Let calling routine handle insertion */ - HGOTO_DONE(SUCCEED) - } /* end if */ - - /* Adjust index to leave room for record to insert */ - if(cmp > 0) - idx++; - - /* Make room for new record */ - if(idx < leaf->nrec) - HDmemmove(H5B2_LEAF_NREC(leaf, hdr, idx + 1), H5B2_LEAF_NREC(leaf, hdr, idx), hdr->cls->nrec_size * (leaf->nrec - idx)); - } /* end if */ - } /* end else */ - - /* Check for modifying existing record */ - if(0 == cmp) { - hbool_t changed = FALSE; /* Whether the 'modify' callback changed the record */ - - /* Make callback for current record */ - if((op)(H5B2_LEAF_NREC(leaf, hdr, idx), op_data, &changed) < 0) { - /* Make certain that the callback didn't modify the value if it failed */ - HDassert(changed == FALSE); - - HGOTO_ERROR(H5E_BTREE, H5E_CANTMODIFY, FAIL, "'modify' callback failed for B-tree update operation") - } /* end if */ - - /* Mark the node as dirty if it changed */ - leaf_flags |= (changed ? H5AC__DIRTIED_FLAG : 0); - - /* Indicate that the record was modified */ - *status = H5B2_UPDATE_MODIFY_DONE; - } /* end if */ - else { - /* Must have a leaf node with enough space to insert a record now */ - HDassert(curr_node_ptr->node_nrec < hdr->node_info[0].max_nrec); - - /* Make callback to store record in native form */ - if((hdr->cls->store)(H5B2_LEAF_NREC(leaf, hdr, idx), udata) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, FAIL, "unable to insert record into leaf node") - - /* Mark the node as dirty if it changed */ - leaf_flags |= H5AC__DIRTIED_FLAG; - - /* Indicate that the record was inserted */ - *status = H5B2_UPDATE_INSERT_DONE; - - /* Update record count for node pointer to current node */ - curr_node_ptr->all_nrec++; - curr_node_ptr->node_nrec++; - - /* Update record count for current node */ - leaf->nrec++; - } /* end else */ - - /* Check for new record being the min or max for the tree */ - /* (Don't use 'else' for the idx check, to allow for root leaf node) */ - if(H5B2_POS_MIDDLE != curr_pos) { - if(idx == 0) { - if(H5B2_POS_LEFT == curr_pos || H5B2_POS_ROOT == curr_pos) { - if(hdr->min_native_rec == NULL) - if(NULL == (hdr->min_native_rec = H5MM_malloc(hdr->cls->nrec_size))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTALLOC, FAIL, "memory allocation failed for v2 B-tree min record info") - HDmemcpy(hdr->min_native_rec, H5B2_LEAF_NREC(leaf, hdr, idx), hdr->cls->nrec_size); - } /* end if */ - } /* end if */ - if(idx == (unsigned)(leaf->nrec - 1)) { - if(H5B2_POS_RIGHT == curr_pos || H5B2_POS_ROOT == curr_pos) { - if(hdr->max_native_rec == NULL) - if(NULL == (hdr->max_native_rec = H5MM_malloc(hdr->cls->nrec_size))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTALLOC, FAIL, "memory allocation failed for v2 B-tree max record info") - HDmemcpy(hdr->max_native_rec, H5B2_LEAF_NREC(leaf, hdr, idx), hdr->cls->nrec_size); - } /* end if */ - } /* end if */ - } /* end if */ - -done: - /* Release the B-tree leaf node */ - if(leaf && H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_LEAF, curr_node_ptr->addr, leaf, leaf_flags) < 0) - HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release leaf B-tree node") - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5B2__update_leaf() */ - - -/*------------------------------------------------------------------------- - * Function: H5B2__update_internal - * - * Purpose: Insert or modify a record in a B-tree leaf node. - * If the record exists already, it is modified as if H5B2_modify - * was called). If it doesn't exist, it is inserted as if - * H5B2_insert was called. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * koziol@hdfgroup.org - * Dec 24 2015 - * - *------------------------------------------------------------------------- - */ -herr_t -H5B2__update_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, - unsigned *parent_cache_info_flags_ptr, H5B2_node_ptr_t *curr_node_ptr, - H5B2_update_status_t *status, H5B2_nodepos_t curr_pos, void *udata, - H5B2_modify_t op, void *op_data) -{ - H5B2_internal_t *internal = NULL; /* Pointer to internal node */ - unsigned internal_flags = H5AC__NO_FLAGS_SET; - int cmp; /* Comparison value of records */ - unsigned idx; /* Location of record which matches key */ - H5B2_nodepos_t next_pos = H5B2_POS_MIDDLE; /* Position of node */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Check arguments. */ - HDassert(hdr); - HDassert(depth > 0); - HDassert(curr_node_ptr); - HDassert(H5F_addr_defined(curr_node_ptr->addr)); - - /* Lock current B-tree node */ - if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, curr_node_ptr->addr, curr_node_ptr->node_nrec, depth, H5AC__NO_FLAGS_SET))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") - - /* Sanity check number of records */ - HDassert(internal->nrec == curr_node_ptr->node_nrec); - - /* Locate node pointer for child */ - if(H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx, &cmp) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") - - /* Check for modifying existing record */ - if(0 == cmp) { - hbool_t changed = FALSE; /* Whether the 'modify' callback changed the record */ - - /* Make callback for current record */ - if((op)(H5B2_INT_NREC(internal, hdr, idx), op_data, &changed) < 0) { - /* Make certain that the callback didn't modify the value if it failed */ - HDassert(changed == FALSE); - - HGOTO_ERROR(H5E_BTREE, H5E_CANTMODIFY, FAIL, "'modify' callback failed for B-tree update operation") - } /* end if */ - - /* Mark the node as dirty if it changed */ - internal_flags |= (changed ? H5AC__DIRTIED_FLAG : 0); - - /* Indicate that the record was modified */ - *status = H5B2_UPDATE_MODIFY_DONE; - } /* end if */ - else { - /* Adjust index to leave room for node to insert */ - if(cmp > 0) - idx++; - - /* Check if this node is left/right-most */ - if(H5B2_POS_MIDDLE != curr_pos) { - if(idx == 0) { - if(H5B2_POS_LEFT == curr_pos || H5B2_POS_ROOT == curr_pos) - next_pos = H5B2_POS_LEFT; - } /* end if */ - else if(idx == internal->nrec) { - if(H5B2_POS_RIGHT == curr_pos || H5B2_POS_ROOT == curr_pos) - next_pos = H5B2_POS_RIGHT; - } /* end else */ - } /* end if */ - - /* Attempt to update record in child */ - if(depth > 1) { - if(H5B2__update_internal(hdr, dxpl_id, (uint16_t)(depth - 1), &internal_flags, &internal->node_ptrs[idx], status, next_pos, udata, op, op_data) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update record in internal B-tree node") - } /* end if */ - else { - if(H5B2__update_leaf(hdr, dxpl_id, &internal->node_ptrs[idx], status, next_pos, udata, op, op_data) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update record in leaf B-tree node") - } /* end else */ - - /* Take actions based on child's status report */ - switch(*status) { - case H5B2_UPDATE_MODIFY_DONE: - /* No action */ - break; - - case H5B2_UPDATE_INSERT_DONE: - /* Mark node as dirty */ - internal_flags |= H5AC__DIRTIED_FLAG; - - /* Update total record count for node pointer to current node */ - curr_node_ptr->all_nrec++; - break; - - case H5B2_UPDATE_INSERT_CHILD_FULL: - /* Split/redistribute this node */ - if(internal->nrec == hdr->node_info[depth].split_nrec) { - hbool_t could_split = FALSE; /* Whether the child node could split */ - -#ifdef QAK -HDfprintf(stderr, "%s: idx = %u, internal->nrec = %u\n", FUNC, idx, internal->nrec); -HDfprintf(stderr, "%s: hdr->node_info[%u].split_nrec = %u\n", FUNC, (unsigned)depth, (unsigned)hdr->node_info[depth].split_nrec); -HDfprintf(stderr, "%s: hdr->node_info[%u].split_nrec = %u\n", FUNC, (unsigned)(depth - 1), (unsigned)hdr->node_info[depth - 1].split_nrec); -#endif /* QAK */ - if(idx == 0) { /* Left-most child */ -#ifdef QAK -HDfprintf(stderr, "%s: Left-most child\n", FUNC); -HDfprintf(stderr, "%s: internal->node_ptrs[%u].node_nrec = %u\n", FUNC, (unsigned)idx, (unsigned)internal->node_ptrs[idx].node_nrec); -HDfprintf(stderr, "%s: internal->node_ptrs[%u].node_nrec = %u\n", FUNC, (unsigned)(idx + 1), (unsigned)internal->node_ptrs[idx + 1].node_nrec); -#endif /* QAK */ - /* Check for left-most child and its neighbor being close to full */ - if((internal->node_ptrs[idx].node_nrec + internal->node_ptrs[idx + 1].node_nrec) - >= ((hdr->node_info[depth - 1].split_nrec * 2) - 1)) - could_split = TRUE; - } /* end if */ - else if(idx == internal->nrec) { /* Right-most child */ -#ifdef QAK -HDfprintf(stderr, "%s: Right-most child\n", FUNC); -HDfprintf(stderr, "%s: internal->node_ptrs[%u].node_nrec = %u\n", FUNC, (unsigned)(idx - 1), (unsigned)internal->node_ptrs[idx - 1].node_nrec); -HDfprintf(stderr, "%s: internal->node_ptrs[%u].node_nrec = %u\n", FUNC, (unsigned)idx, (unsigned)internal->node_ptrs[idx].node_nrec); -#endif /* QAK */ - /* Check for right-most child and its neighbor being close to full */ - if((internal->node_ptrs[idx - 1].node_nrec + internal->node_ptrs[idx].node_nrec) - >= ((hdr->node_info[depth - 1].split_nrec * 2) - 1)) - could_split = TRUE; - } /* end else-if */ - else { /* Middle child */ -#ifdef QAK -HDfprintf(stderr, "%s: Middle child\n", FUNC); -HDfprintf(stderr, "%s: internal->node_ptrs[%u].node_nrec = %u\n", FUNC, (unsigned)(idx - 1), (unsigned)internal->node_ptrs[idx - 1].node_nrec); -HDfprintf(stderr, "%s: internal->node_ptrs[%u].node_nrec = %u\n", FUNC, (unsigned)idx, (unsigned)internal->node_ptrs[idx].node_nrec); -HDfprintf(stderr, "%s: internal->node_ptrs[%u].node_nrec = %u\n", FUNC, (unsigned)(idx + 1), (unsigned)internal->node_ptrs[idx + 1].node_nrec); -#endif /* QAK */ - /* Check for middle child and its left neighbor being close to full */ - if((internal->node_ptrs[idx - 1].node_nrec + internal->node_ptrs[idx].node_nrec) - >= ((hdr->node_info[depth - 1].split_nrec * 2) - 1)) - could_split = TRUE; - /* Check for middle child and its right neighbor being close to full */ - else if((internal->node_ptrs[idx].node_nrec + internal->node_ptrs[idx + 1].node_nrec) - >= ((hdr->node_info[depth - 1].split_nrec * 2) - 1)) - could_split = TRUE; - } /* end if */ - - /* If this node is full and the child node insertion could - * cause a split, punt back up to caller, leaving the - * "insert child full" status. - */ - if(could_split) { - /* Release the internal B-tree node */ - if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr->addr, internal, internal_flags) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release internal B-tree node") - internal = NULL; - -#ifdef QAK -HDfprintf(stderr, "%s: Punting back to caller\n", FUNC); -#endif /* QAK */ - /* Punt back to caller */ - HGOTO_DONE(SUCCEED); - } /* end if */ - } /* end if */ - - /* Release the internal B-tree node */ - if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr->addr, internal, internal_flags) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release internal B-tree node") - internal = NULL; - - /* Indicate that the record was inserted */ - *status = H5B2_UPDATE_INSERT_DONE; - - /* Dodge sideways into inserting a record into this node */ - if(H5B2__insert_internal(hdr, dxpl_id, depth, parent_cache_info_flags_ptr, curr_node_ptr, curr_pos, udata) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, FAIL, "unable to insert record into internal B-tree node") - break; - - case H5B2_UPDATE_UNKNOWN: - default: - HDassert(0 && "Invalid update status"); - HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "invalid update status") - } /* end switch */ - } /* end else */ - -done: - /* Release the internal B-tree node */ - if(internal && H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr->addr, internal, internal_flags) < 0) - HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release internal B-tree node") - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5B2__update_internal() */ - - -/*------------------------------------------------------------------------- - * Function: H5B2__create_leaf - * - * Purpose: Creates empty leaf node of a B-tree and update node pointer - * to point to it. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * koziol@ncsa.uiuc.edu - * Feb 2 2005 - * - *------------------------------------------------------------------------- - */ -herr_t -H5B2__create_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, H5B2_node_ptr_t *node_ptr) -{ - H5B2_leaf_t *leaf = NULL; /* Pointer to new leaf node created */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Check arguments. */ - HDassert(hdr); - HDassert(node_ptr); - - /* Allocate memory for leaf information */ - if(NULL == (leaf = H5FL_MALLOC(H5B2_leaf_t))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree leaf info") - - /* Set metadata cache info */ - HDmemset(&leaf->cache_info, 0, sizeof(H5AC_info_t)); - - /* Increment ref. count on B-tree header */ - if(H5B2__hdr_incr(hdr) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTINC, FAIL, "can't increment ref. count on B-tree header") - - /* Share B-tree header information */ - leaf->hdr = hdr; - - /* Allocate space for the native keys in memory */ - if(NULL == (leaf->leaf_native = (uint8_t *)H5FL_FAC_MALLOC(hdr->node_info[0].nat_rec_fac))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree leaf native keys") - HDmemset(leaf->leaf_native, 0, hdr->cls->nrec_size * hdr->node_info[0].max_nrec); - - /* Set number of records */ - leaf->nrec = 0; - - /* Allocate space on disk for the leaf */ - if(HADDR_UNDEF == (node_ptr->addr = H5MF_alloc(hdr->f, H5FD_MEM_BTREE, dxpl_id, (hsize_t)hdr->node_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "file allocation failed for B-tree leaf node") - - /* Cache the new B-tree node */ - if(H5AC_insert_entry(hdr->f, dxpl_id, H5AC_BT2_LEAF, node_ptr->addr, leaf, H5AC__NO_FLAGS_SET) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, FAIL, "can't add B-tree leaf to cache") - -done: - if(ret_value < 0) { - if(leaf) - if(H5B2__leaf_free(leaf) < 0) - HDONE_ERROR(H5E_BTREE, H5E_CANTFREE, FAIL, "unable to release v2 B-tree leaf node") - } /* end if */ - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5B2__create_leaf() */ - - -/*------------------------------------------------------------------------- - * Function: H5B2__protect_leaf - * - * Purpose: "Protect" an leaf node in the metadata cache - * - * Return: Pointer to leaf node on success/NULL on failure - * - * Programmer: Quincey Koziol - * koziol@hdfgroup.org - * May 5 2010 - * - *------------------------------------------------------------------------- - */ -H5B2_leaf_t * -H5B2__protect_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, haddr_t addr, uint16_t nrec, - unsigned flags) -{ - H5B2_leaf_cache_ud_t udata; /* User-data for callback */ - H5B2_leaf_t *ret_value = NULL; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Check arguments. */ - HDassert(hdr); - HDassert(H5F_addr_defined(addr)); - - /* only H5AC__READ_ONLY_FLAG may appear in flags */ - HDassert((flags & (unsigned)(~H5AC__READ_ONLY_FLAG)) == 0); - - /* Set up user data for callback */ - udata.f = hdr->f; - udata.hdr = hdr; - H5_CHECKED_ASSIGN(udata.nrec, uint16_t, nrec, unsigned) - - /* Protect the leaf node */ - if(NULL == (ret_value = (H5B2_leaf_t *)H5AC_protect(hdr->f, dxpl_id, H5AC_BT2_LEAF, addr, &udata, flags))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, NULL, "unable to protect B-tree leaf node") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5B2__protect_leaf() */ - - -/*------------------------------------------------------------------------- - * Function: H5B2__create_internal - * - * Purpose: Creates empty internal node of a B-tree and update node pointer - * to point to it. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * koziol@ncsa.uiuc.edu - * Feb 3 2005 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5B2__create_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, H5B2_node_ptr_t *node_ptr, - uint16_t depth) -{ - H5B2_internal_t *internal = NULL; /* Pointer to new internal node created */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Check arguments. */ - HDassert(hdr); - HDassert(node_ptr); - HDassert(depth > 0); - - /* Allocate memory for internal node information */ - if(NULL == (internal = H5FL_MALLOC(H5B2_internal_t))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree internal info") - - /* Set metadata cache info */ - HDmemset(&internal->cache_info, 0, sizeof(H5AC_info_t)); - - /* Increment ref. count on B-tree header */ - if(H5B2__hdr_incr(hdr) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTINC, FAIL, "can't increment ref. count on B-tree header") - - /* Share B-tree header information */ - internal->hdr = hdr; - - /* Allocate space for the native keys in memory */ - if(NULL == (internal->int_native = (uint8_t *)H5FL_FAC_MALLOC(hdr->node_info[depth].nat_rec_fac))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree internal native keys") - HDmemset(internal->int_native, 0, hdr->cls->nrec_size * hdr->node_info[depth].max_nrec); - - /* Allocate space for the node pointers in memory */ - if(NULL == (internal->node_ptrs = (H5B2_node_ptr_t *)H5FL_FAC_MALLOC(hdr->node_info[depth].node_ptr_fac))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree internal node pointers") - HDmemset(internal->node_ptrs, 0, sizeof(H5B2_node_ptr_t) * (hdr->node_info[depth].max_nrec + 1)); - - /* Set number of records & depth of the node */ - internal->nrec = 0; - internal->depth = depth; - - /* Allocate space on disk for the internal node */ - if(HADDR_UNDEF == (node_ptr->addr = H5MF_alloc(hdr->f, H5FD_MEM_BTREE, dxpl_id, (hsize_t)hdr->node_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "file allocation failed for B-tree internal node") - - /* Cache the new B-tree node */ - if(H5AC_insert_entry(hdr->f, dxpl_id, H5AC_BT2_INT, node_ptr->addr, internal, H5AC__NO_FLAGS_SET) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, FAIL, "can't add B-tree internal node to cache") - -done: - if(ret_value < 0) { - if(internal) - if(H5B2__internal_free(internal) < 0) - HDONE_ERROR(H5E_BTREE, H5E_CANTFREE, FAIL, "unable to release v2 B-tree internal node") - } /* end if */ - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5B2__create_internal() */ - - -/*------------------------------------------------------------------------- - * Function: H5B2__protect_internal - * - * Purpose: "Protect" an internal node in the metadata cache - * - * Return: Pointer to internal node on success/NULL on failure - * - * Programmer: Quincey Koziol - * koziol@hdfgroup.org - * Aug 25 2006 - * - *------------------------------------------------------------------------- - */ -H5B2_internal_t * -H5B2__protect_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, haddr_t addr, - uint16_t nrec, uint16_t depth, unsigned flags) -{ - H5B2_internal_cache_ud_t udata; /* User data to pass through to cache 'deserialize' callback */ - H5B2_internal_t *ret_value = NULL; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Check arguments. */ - HDassert(hdr); - HDassert(H5F_addr_defined(addr)); - HDassert(depth > 0); - - /* only H5AC__READ_ONLY_FLAG may appear in flags */ - HDassert((flags & (unsigned)(~H5AC__READ_ONLY_FLAG)) == 0); - - /* Set up user data for callback */ - udata.f = hdr->f; - udata.hdr = hdr; - udata.nrec = nrec; - udata.depth = depth; - - /* Protect the internal node */ - if(NULL == (ret_value = (H5B2_internal_t *)H5AC_protect(hdr->f, dxpl_id, H5AC_BT2_INT, addr, &udata, flags))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, NULL, "unable to protect B-tree internal node") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5B2__protect_internal() */ +} /* H5B2__insert() */ /*------------------------------------------------------------------------- @@ -2421,13 +1505,15 @@ done: */ herr_t H5B2__iterate_node(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, - const H5B2_node_ptr_t *curr_node, H5B2_operator_t op, void *op_data) + const H5B2_node_ptr_t *curr_node, void *parent, H5B2_operator_t op, + void *op_data) { const H5AC_class_t *curr_node_class = NULL; /* Pointer to current node's class info */ void *node = NULL; /* Pointers to current node */ uint8_t *node_native; /* Pointers to node's native records */ uint8_t *native = NULL; /* Pointers to copy of node's native records */ H5B2_node_ptr_t *node_ptrs = NULL; /* Pointers to node's node pointers */ + hbool_t node_pinned = FALSE; /* Whether node is pinned */ unsigned u; /* Local index */ herr_t ret_value = H5_ITER_CONT; /* Iterator return value */ @@ -2443,7 +1529,7 @@ H5B2__iterate_node(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, H5B2_internal_t *internal; /* Pointer to internal node */ /* Lock the current B-tree node */ - if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, curr_node->addr, curr_node->node_nrec, depth, H5AC__READ_ONLY_FLAG))) + if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, parent, (H5B2_node_ptr_t *)curr_node, depth, FALSE, H5AC__READ_ONLY_FLAG))) /* Casting away const OK -QAK */ HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") /* Set up information about current node */ @@ -2462,7 +1548,7 @@ H5B2__iterate_node(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, H5B2_leaf_t *leaf; /* Pointer to leaf node */ /* Lock the current B-tree node */ - if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, curr_node->addr, curr_node->node_nrec, H5AC__READ_ONLY_FLAG))) + if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, parent, (H5B2_node_ptr_t *)curr_node, FALSE, H5AC__READ_ONLY_FLAG))) /* Casting away const OK -QAK */ HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") /* Set up information about current node */ @@ -2479,15 +1565,18 @@ H5B2__iterate_node(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, HDmemcpy(native, node_native, (hdr->cls->nrec_size * curr_node->node_nrec)); /* Unlock the node */ - if(H5AC_unprotect(hdr->f, dxpl_id, curr_node_class, curr_node->addr, node, H5AC__NO_FLAGS_SET) < 0) + if(H5AC_unprotect(hdr->f, dxpl_id, curr_node_class, curr_node->addr, node, (unsigned)(hdr->swmr_write ? H5AC__PIN_ENTRY_FLAG : H5AC__NO_FLAGS_SET)) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree node") - node = NULL; + if(hdr->swmr_write) + node_pinned = TRUE; + else + node = NULL; /* Iterate through records, in order */ for(u = 0; u < curr_node->node_nrec && !ret_value; u++) { /* Descend into child node, if current node is an internal node */ if(depth > 0) { - if((ret_value = H5B2__iterate_node(hdr, dxpl_id, (uint16_t)(depth - 1), &(node_ptrs[u]), op, op_data)) < 0) + if((ret_value = H5B2__iterate_node(hdr, dxpl_id, (uint16_t)(depth - 1), &(node_ptrs[u]), node, op, op_data)) < 0) HERROR(H5E_BTREE, H5E_CANTLIST, "node iteration failed"); } /* end if */ @@ -2500,11 +1589,15 @@ H5B2__iterate_node(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Descend into last child node, if current node is an internal node */ if(!ret_value && depth > 0) { - if((ret_value = H5B2__iterate_node(hdr, dxpl_id, (uint16_t)(depth - 1), &(node_ptrs[u]), op, op_data)) < 0) + if((ret_value = H5B2__iterate_node(hdr, dxpl_id, (uint16_t)(depth - 1), &(node_ptrs[u]), node, op, op_data)) < 0) HERROR(H5E_BTREE, H5E_CANTLIST, "node iteration failed"); } /* end if */ done: + /* Unpin the node if it was pinned */ + if(node_pinned && H5AC_unpin_entry(node) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPIN, FAIL, "can't unpin node") + /* Release the node pointers & native records, if they were copied */ if(node_ptrs) node_ptrs = (H5B2_node_ptr_t *)H5FL_FAC_FREE(hdr->node_info[depth].node_ptr_fac, node_ptrs); @@ -2516,867 +1609,6 @@ done: /*------------------------------------------------------------------------- - * Function: H5B2__remove_leaf - * - * Purpose: Removes a record from a B-tree leaf node. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * koziol@ncsa.uiuc.edu - * Mar 3 2005 - * - *------------------------------------------------------------------------- - */ -herr_t -H5B2__remove_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, H5B2_node_ptr_t *curr_node_ptr, - H5B2_nodepos_t curr_pos, void *udata, H5B2_remove_t op, void *op_data) -{ - H5B2_leaf_t *leaf; /* Pointer to leaf node */ - haddr_t leaf_addr = HADDR_UNDEF; /* Leaf address on disk */ - unsigned leaf_flags = H5AC__NO_FLAGS_SET; /* Flags for unprotecting leaf node */ - unsigned idx; /* Location of record which matches key */ - int cmp; /* Comparison value of records */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Check arguments. */ - HDassert(hdr); - HDassert(curr_node_ptr); - HDassert(H5F_addr_defined(curr_node_ptr->addr)); - - /* Lock current B-tree node */ - leaf_addr = curr_node_ptr->addr; - if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, leaf_addr, curr_node_ptr->node_nrec, H5AC__NO_FLAGS_SET))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") - - /* Sanity check number of records */ - HDassert(curr_node_ptr->all_nrec == curr_node_ptr->node_nrec); - HDassert(leaf->nrec == curr_node_ptr->node_nrec); - - /* Find correct location to remove this record */ - if(H5B2__locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx, &cmp) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") - if(cmp != 0) - HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, FAIL, "record is not in B-tree") - - /* Check for invalidating the min/max record for the tree */ - if(H5B2_POS_MIDDLE != curr_pos) { - /* (Don't use 'else' for the idx check, to allow for root leaf node) */ - if(idx == 0) { - if(H5B2_POS_LEFT == curr_pos || H5B2_POS_ROOT == curr_pos) { - if(hdr->min_native_rec) - hdr->min_native_rec = H5MM_xfree(hdr->min_native_rec); - } /* end if */ - } /* end if */ - if(idx == (unsigned)(leaf->nrec - 1)) { - if(H5B2_POS_RIGHT == curr_pos || H5B2_POS_ROOT == curr_pos) { - if(hdr->max_native_rec) - hdr->max_native_rec = H5MM_xfree(hdr->max_native_rec); - } /* end if */ - } /* end if */ - } /* end if */ - - /* Make 'remove' callback if there is one */ - if(op) - if((op)(H5B2_LEAF_NREC(leaf, hdr, idx), op_data) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTDELETE, FAIL, "unable to remove record into leaf node") - - /* Update number of records in node */ - leaf->nrec--; - - /* Mark leaf node as dirty also */ - leaf_flags |= H5AC__DIRTIED_FLAG; - - if(leaf->nrec > 0) { - /* Pack record out of leaf */ - if(idx < leaf->nrec) - HDmemmove(H5B2_LEAF_NREC(leaf, hdr, idx), H5B2_LEAF_NREC(leaf, hdr, (idx + 1)), hdr->cls->nrec_size * (leaf->nrec - idx)); - } /* end if */ - else { - /* Let the cache know that the object is deleted */ - leaf_flags |= H5AC__DELETED_FLAG | H5AC__FREE_FILE_SPACE_FLAG; - - /* Reset address of parent node pointer */ - curr_node_ptr->addr = HADDR_UNDEF; - } /* end else */ - - /* Update record count for parent of leaf node */ - curr_node_ptr->node_nrec--; - -done: - /* Release the B-tree leaf node */ - if(leaf && H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_LEAF, leaf_addr, leaf, leaf_flags) < 0) - HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release leaf B-tree node") - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5B2__remove_leaf() */ - - -/*------------------------------------------------------------------------- - * Function: H5B2__remove_internal - * - * Purpose: Removes a record from a B-tree node. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * koziol@ncsa.uiuc.edu - * Mar 3 2005 - * - *------------------------------------------------------------------------- - */ -herr_t -H5B2__remove_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, hbool_t *depth_decreased, - void *swap_loc, uint16_t depth, H5AC_info_t *parent_cache_info, - unsigned *parent_cache_info_flags_ptr, H5B2_nodepos_t curr_pos, - H5B2_node_ptr_t *curr_node_ptr, void *udata, H5B2_remove_t op, void *op_data) -{ - H5AC_info_t *new_cache_info; /* Pointer to new cache info */ - unsigned *new_cache_info_flags_ptr = NULL; - H5B2_node_ptr_t *new_node_ptr; /* Pointer to new node pointer */ - H5B2_internal_t *internal; /* Pointer to internal node */ - H5B2_nodepos_t next_pos = H5B2_POS_MIDDLE; /* Position of next node */ - unsigned internal_flags = H5AC__NO_FLAGS_SET; - haddr_t internal_addr; /* Address of internal node */ - size_t merge_nrec; /* Number of records to merge node at */ - hbool_t collapsed_root = FALSE; /* Whether the root was collapsed */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Check arguments. */ - HDassert(hdr); - HDassert(depth > 0); - HDassert(parent_cache_info); - HDassert(curr_node_ptr); - HDassert(H5F_addr_defined(curr_node_ptr->addr)); - - /* Lock current B-tree node */ - internal_addr = curr_node_ptr->addr; - if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, internal_addr, curr_node_ptr->node_nrec, depth, H5AC__NO_FLAGS_SET))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") - - /* Determine the correct number of records to merge at */ - merge_nrec = hdr->node_info[depth - 1].merge_nrec; - - /* Check for needing to collapse the root node */ - /* (The root node is the only internal node allowed to have 1 record) */ - if(internal->nrec == 1 && - ((internal->node_ptrs[0].node_nrec + internal->node_ptrs[1].node_nrec) <= ((merge_nrec * 2) + 1))) { - - /* Merge children of root node */ - if(H5B2__merge2(hdr, dxpl_id, depth, curr_node_ptr, - parent_cache_info_flags_ptr, internal, &internal_flags, 0) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to merge child node") - - /* Let the cache know that the object is deleted */ - internal_flags |= H5AC__DELETED_FLAG | H5AC__FREE_FILE_SPACE_FLAG; - - /* Reset information in header's root node pointer */ - curr_node_ptr->addr = internal->node_ptrs[0].addr; - curr_node_ptr->node_nrec = internal->node_ptrs[0].node_nrec; - - /* Indicate that the level of the B-tree decreased */ - *depth_decreased = TRUE; - - /* Set pointers for advancing to child node */ - new_cache_info = parent_cache_info; - new_cache_info_flags_ptr = parent_cache_info_flags_ptr; - new_node_ptr = curr_node_ptr; - - /* Set flag to indicate root was collapsed */ - collapsed_root = TRUE; - - /* Indicate position of next node */ - next_pos = H5B2_POS_ROOT; - } /* end if */ - /* Merge or redistribute child node pointers, if necessary */ - else { - unsigned idx; /* Location of record which matches key */ - int cmp = 0; /* Comparison value of records */ - unsigned retries; /* Number of times to attempt redistribution */ - - /* Locate node pointer for child */ - if(swap_loc) - idx = 0; - else { - if(H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, - udata, &idx, &cmp) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") - if(cmp >= 0) - idx++; - } /* end else */ - - /* Set the number of redistribution retries */ - /* This takes care of the case where a B-tree node needs to be - * redistributed, but redistributing the node causes the index - * for removal to move to another node, which also needs to be - * redistributed. Now, we loop trying to redistribute and then - * eventually force a merge */ - retries = 2; - - /* Preemptively merge/redistribute a node we will enter */ - while(internal->node_ptrs[idx].node_nrec == merge_nrec) { - /* Attempt to redistribute records among children */ - /* (NOTE: These 2-node redistributions should actually get the - * record to promote from the node with more records. - QAK) - */ - /* (NOTE: This code is the same in both H5B2__remove_internal() and - * H5B2__remove_internal_by_idx(), fix bugs in both places! - QAK) - */ - if(idx == 0) { /* Left-most child */ - if(retries > 0 && (internal->node_ptrs[idx + 1].node_nrec > merge_nrec)) { - if(H5B2__redistribute2(hdr, dxpl_id, depth, internal, idx) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTREDISTRIBUTE, FAIL, "unable to redistribute child node records") - } /* end if */ - else { - if(H5B2__merge2(hdr, dxpl_id, depth, curr_node_ptr, - parent_cache_info_flags_ptr, internal, &internal_flags, idx) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to merge child node") - } /* end else */ - } /* end if */ - else if(idx == internal->nrec) { /* Right-most child */ - if(retries > 0 && (internal->node_ptrs[idx - 1].node_nrec > merge_nrec)) { - if(H5B2__redistribute2(hdr, dxpl_id, depth, internal, (idx - 1)) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTREDISTRIBUTE, FAIL, "unable to redistribute child node records") - } /* end if */ - else { - if(H5B2__merge2(hdr, dxpl_id, depth, curr_node_ptr, - parent_cache_info_flags_ptr, internal, &internal_flags, (idx - 1)) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to merge child node") - } /* end else */ - } /* end if */ - else { /* Middle child */ - if(retries > 0 && ((internal->node_ptrs[idx + 1].node_nrec > merge_nrec) || - (internal->node_ptrs[idx - 1].node_nrec > merge_nrec))) { - if(H5B2__redistribute3(hdr, dxpl_id, depth, internal, &internal_flags, idx) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTREDISTRIBUTE, FAIL, "unable to redistribute child node records") - } /* end if */ - else { - if(H5B2__merge3(hdr, dxpl_id, depth, curr_node_ptr, - parent_cache_info_flags_ptr, internal, &internal_flags, idx) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to merge child node") - } /* end else */ - } /* end else */ - - /* Locate node pointer for child (after merge/redistribute) */ - if(swap_loc) - idx = 0; - else { -/* Actually, this can be easily updated (for 2-node redistrib.) and shouldn't require re-searching */ - if(H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx, &cmp) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") - if(cmp >= 0) - idx++; - } /* end else */ - - /* Decrement the number of redistribution retries left */ - retries--; - } /* end while */ - - /* Handle deleting a record from an internal node */ - if(!swap_loc && cmp == 0) - swap_loc = H5B2_INT_NREC(internal, hdr, idx - 1); - - /* Swap record to delete with record from leaf, if we are the last internal node */ - if(swap_loc && depth == 1) - if(H5B2__swap_leaf(hdr, dxpl_id, depth, internal, &internal_flags, idx, swap_loc) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTSWAP, FAIL, "Can't swap records in B-tree") - - /* Set pointers for advancing to child node */ - new_cache_info_flags_ptr = &internal_flags; - new_cache_info = &internal->cache_info; - new_node_ptr = &internal->node_ptrs[idx]; - - /* Indicate position of next node */ - if(H5B2_POS_MIDDLE != curr_pos) { - if(idx == 0) { - if(H5B2_POS_LEFT == curr_pos || H5B2_POS_ROOT == curr_pos) - next_pos = H5B2_POS_LEFT; - } /* end if */ - else if(idx == internal->nrec) { - if(H5B2_POS_RIGHT == curr_pos || H5B2_POS_ROOT == curr_pos) - next_pos = H5B2_POS_RIGHT; - } /* end if */ - } /* end if */ - } /* end else */ - - /* Attempt to remove record from child node */ - if(depth > 1) { - if(H5B2__remove_internal(hdr, dxpl_id, depth_decreased, swap_loc, (uint16_t)(depth - 1), - new_cache_info, new_cache_info_flags_ptr, next_pos, new_node_ptr, udata, op, op_data) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTDELETE, FAIL, "unable to remove record from B-tree internal node") - } /* end if */ - else { - if(H5B2__remove_leaf(hdr, dxpl_id, new_node_ptr, next_pos, udata, op, op_data) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTDELETE, FAIL, "unable to remove record from B-tree leaf node") - } /* end else */ - - /* Update record count for node pointer to current node */ - if(!collapsed_root) - new_node_ptr->all_nrec--; - - /* Mark node as dirty */ - internal_flags |= H5AC__DIRTIED_FLAG; - -#ifdef H5B2_DEBUG - H5B2__assert_internal((!collapsed_root ? (curr_node_ptr->all_nrec - 1) : new_node_ptr->all_nrec), hdr, internal); -#endif /* H5B2_DEBUG */ - -done: - /* Release the B-tree internal node */ - if(internal && H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, internal_addr, internal, internal_flags) < 0) - HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release internal B-tree node") - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5B2__remove_internal() */ - - -/*------------------------------------------------------------------------- - * Function: H5B2__remove_leaf_by_idx - * - * Purpose: Removes a record from a B-tree leaf node, according to the - * offset in the B-tree records. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * koziol@hdfgroup.org - * Nov 14 2006 - * - *------------------------------------------------------------------------- - */ -herr_t -H5B2__remove_leaf_by_idx(H5B2_hdr_t *hdr, hid_t dxpl_id, - H5B2_node_ptr_t *curr_node_ptr, H5B2_nodepos_t curr_pos, - unsigned idx, H5B2_remove_t op, void *op_data) -{ - H5B2_leaf_t *leaf; /* Pointer to leaf node */ - haddr_t leaf_addr = HADDR_UNDEF; /* Leaf address on disk */ - unsigned leaf_flags = H5AC__NO_FLAGS_SET; /* Flags for unprotecting leaf node */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Check arguments. */ - HDassert(hdr); - HDassert(curr_node_ptr); - HDassert(H5F_addr_defined(curr_node_ptr->addr)); - - /* Lock B-tree leaf node */ - leaf_addr = curr_node_ptr->addr; - if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, leaf_addr, curr_node_ptr->node_nrec, H5AC__NO_FLAGS_SET))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") - - /* Sanity check number of records */ - HDassert(curr_node_ptr->all_nrec == curr_node_ptr->node_nrec); - HDassert(leaf->nrec == curr_node_ptr->node_nrec); - HDassert(idx < leaf->nrec); - - /* Check for invalidating the min/max record for the tree */ - if(H5B2_POS_MIDDLE != curr_pos) { - /* (Don't use 'else' for the idx check, to allow for root leaf node) */ - if(idx == 0) { - if(H5B2_POS_LEFT == curr_pos || H5B2_POS_ROOT == curr_pos) { - if(hdr->min_native_rec) - hdr->min_native_rec = H5MM_xfree(hdr->min_native_rec); - } /* end if */ - } /* end if */ - if(idx == (unsigned)(leaf->nrec - 1)) { - if(H5B2_POS_RIGHT == curr_pos || H5B2_POS_ROOT == curr_pos) { - if(hdr->max_native_rec) - hdr->max_native_rec = H5MM_xfree(hdr->max_native_rec); - } /* end if */ - } /* end if */ - } /* end if */ - - /* Make 'remove' callback if there is one */ - if(op) - if((op)(H5B2_LEAF_NREC(leaf, hdr, idx), op_data) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTDELETE, FAIL, "unable to remove record into leaf node") - - /* Update number of records in node */ - leaf->nrec--; - - /* Mark leaf node as dirty also */ - leaf_flags |= H5AC__DIRTIED_FLAG; - - if(leaf->nrec > 0) { - /* Pack record out of leaf */ - if(idx < leaf->nrec) - HDmemmove(H5B2_LEAF_NREC(leaf, hdr, idx), H5B2_LEAF_NREC(leaf, hdr, (idx + 1)), hdr->cls->nrec_size * (leaf->nrec - idx)); - } /* end if */ - else { - /* Let the cache know that the object is deleted */ - leaf_flags |= H5AC__DELETED_FLAG | H5AC__FREE_FILE_SPACE_FLAG; - - /* Reset address of parent node pointer */ - curr_node_ptr->addr = HADDR_UNDEF; - } /* end else */ - - /* Update record count for parent of leaf node */ - curr_node_ptr->node_nrec--; - -done: - /* Release the B-tree leaf node */ - if(leaf && H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_LEAF, leaf_addr, leaf, leaf_flags) < 0) - HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release leaf B-tree node") - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5B2__remove_leaf_by_idx() */ - - -/*------------------------------------------------------------------------- - * Function: H5B2__remove_internal_by_idx - * - * Purpose: Removes a record from a B-tree node, according to the offset - * in the B-tree records - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * koziol@hdfgroup.org - * Nov 14 2006 - * - *------------------------------------------------------------------------- - */ -herr_t -H5B2__remove_internal_by_idx(H5B2_hdr_t *hdr, hid_t dxpl_id, - hbool_t *depth_decreased, void *swap_loc, uint16_t depth, - H5AC_info_t *parent_cache_info, unsigned *parent_cache_info_flags_ptr, - H5B2_node_ptr_t *curr_node_ptr, H5B2_nodepos_t curr_pos, hsize_t n, - H5B2_remove_t op, void *op_data) -{ - H5AC_info_t *new_cache_info; /* Pointer to new cache info */ - unsigned *new_cache_info_flags_ptr = NULL; - H5B2_node_ptr_t *new_node_ptr; /* Pointer to new node pointer */ - H5B2_internal_t *internal; /* Pointer to internal node */ - H5B2_nodepos_t next_pos = H5B2_POS_MIDDLE; /* Position of next node */ - unsigned internal_flags = H5AC__NO_FLAGS_SET; - haddr_t internal_addr; /* Address of internal node */ - size_t merge_nrec; /* Number of records to merge node at */ - hbool_t collapsed_root = FALSE; /* Whether the root was collapsed */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Check arguments. */ - HDassert(hdr); - HDassert(depth > 0); - HDassert(parent_cache_info); - HDassert(curr_node_ptr); - HDassert(H5F_addr_defined(curr_node_ptr->addr)); - - /* Lock current B-tree node */ - internal_addr = curr_node_ptr->addr; - if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, internal_addr, curr_node_ptr->node_nrec, depth, H5AC__NO_FLAGS_SET))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") - HDassert(internal->nrec == curr_node_ptr->node_nrec); - HDassert(depth == hdr->depth || internal->nrec > 1); - - /* Determine the correct number of records to merge at */ - merge_nrec = hdr->node_info[depth - 1].merge_nrec; - - /* Check for needing to collapse the root node */ - /* (The root node is the only internal node allowed to have 1 record) */ - if(internal->nrec == 1 && - ((internal->node_ptrs[0].node_nrec + internal->node_ptrs[1].node_nrec) <= ((merge_nrec * 2) + 1))) { - HDassert(depth == hdr->depth); - - /* Merge children of root node */ - if(H5B2__merge2(hdr, dxpl_id, depth, curr_node_ptr, - parent_cache_info_flags_ptr, internal, &internal_flags, 0) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to merge child node") - - /* Let the cache know that the object is deleted */ - internal_flags |= H5AC__DELETED_FLAG | H5AC__FREE_FILE_SPACE_FLAG; - - /* Reset information in header's root node pointer */ - curr_node_ptr->addr = internal->node_ptrs[0].addr; - curr_node_ptr->node_nrec = internal->node_ptrs[0].node_nrec; - - /* Indicate that the level of the B-tree decreased */ - *depth_decreased = TRUE; - - /* Set pointers for advancing to child node */ - new_cache_info = parent_cache_info; - new_cache_info_flags_ptr = parent_cache_info_flags_ptr; - new_node_ptr = curr_node_ptr; - - /* Set flag to indicate root was collapsed */ - collapsed_root = TRUE; - - /* Indicate position of next node */ - next_pos = H5B2_POS_ROOT; - } /* end if */ - /* Merge or redistribute child node pointers, if necessary */ - else { - hsize_t orig_n = n; /* Original index looked for */ - unsigned idx; /* Location of record which matches key */ - hbool_t found = FALSE; /* Comparison value of records */ - unsigned retries; /* Number of times to attempt redistribution */ - - /* Locate node pointer for child */ - if(swap_loc) - idx = 0; - else { - /* Search for record with correct index */ - for(idx = 0; idx < internal->nrec; idx++) { - /* Check which child node contains indexed record */ - if(internal->node_ptrs[idx].all_nrec >= n) { - /* Check if record is in this node */ - if(internal->node_ptrs[idx].all_nrec == n) { - /* Indicate the record was found and that the index - * in child nodes is zero from now on - */ - found = TRUE; - n = 0; - - /* Increment to next record */ - idx++; - } /* end if */ - - /* Break out of loop early */ - break; - } /* end if */ - - /* Decrement index we are looking for to account for the node we - * just advanced past. - */ - n -= (internal->node_ptrs[idx].all_nrec + 1); - } /* end for */ - } /* end else */ - - /* Set the number of redistribution retries */ - /* This takes care of the case where a B-tree node needs to be - * redistributed, but redistributing the node causes the index - * for removal to move to another node, which also needs to be - * redistributed. Now, we loop trying to redistribute and then - * eventually force a merge */ - retries = 2; - - /* Preemptively merge/redistribute a node we will enter */ - while(internal->node_ptrs[idx].node_nrec == merge_nrec) { - /* Attempt to redistribute records among children */ - /* (NOTE: These 2-node redistributions should actually get the - * record to promote from the node with more records. - QAK) - */ - /* (NOTE: This code is the same in both H5B2__remove_internal() and - * H5B2__remove_internal_by_idx(), fix bugs in both places! - QAK) - */ - if(idx == 0) { /* Left-most child */ - if(retries > 0 && (internal->node_ptrs[idx + 1].node_nrec > merge_nrec)) { - if(H5B2__redistribute2(hdr, dxpl_id, depth, internal, idx) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTREDISTRIBUTE, FAIL, "unable to redistribute child node records") - } /* end if */ - else { - if(H5B2__merge2(hdr, dxpl_id, depth, curr_node_ptr, - parent_cache_info_flags_ptr, internal, &internal_flags, idx) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to merge child node") - } /* end else */ - } /* end if */ - else if(idx == internal->nrec) { /* Right-most child */ - if(retries > 0 && (internal->node_ptrs[idx - 1].node_nrec > merge_nrec)) { - if(H5B2__redistribute2(hdr, dxpl_id, depth, internal, (idx - 1)) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTREDISTRIBUTE, FAIL, "unable to redistribute child node records") - } /* end if */ - else { - if(H5B2__merge2(hdr, dxpl_id, depth, curr_node_ptr, - parent_cache_info_flags_ptr, internal, &internal_flags, (idx - 1)) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to merge child node") - } /* end else */ - } /* end if */ - else { /* Middle child */ - if(retries > 0 && ((internal->node_ptrs[idx + 1].node_nrec > merge_nrec) || - (internal->node_ptrs[idx - 1].node_nrec > merge_nrec))) { - if(H5B2__redistribute3(hdr, dxpl_id, depth, internal, &internal_flags, idx) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTREDISTRIBUTE, FAIL, "unable to redistribute child node records") - } /* end if */ - else { - if(H5B2__merge3(hdr, dxpl_id, depth, curr_node_ptr, - parent_cache_info_flags_ptr, internal, &internal_flags, idx) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to merge child node") - } /* end else */ - } /* end else */ - - /* Locate node pointer for child (after merge/redistribute) */ - if(swap_loc) - idx = 0; - else { - /* Count from the orginal index value again */ - n = orig_n; - - /* Reset "found" flag - the record may have shifted during the - * redistribute/merge - */ - found = FALSE; - - /* Search for record with correct index */ - for(idx = 0; idx < internal->nrec; idx++) { - /* Check which child node contains indexed record */ - if(internal->node_ptrs[idx].all_nrec >= n) { - /* Check if record is in this node */ - if(internal->node_ptrs[idx].all_nrec == n) { - /* Indicate the record was found and that the index - * in child nodes is zero from now on - */ - found = TRUE; - n = 0; - - /* Increment to next record */ - idx++; - } /* end if */ - - /* Break out of loop early */ - break; - } /* end if */ - - /* Decrement index we are looking for to account for the node we - * just advanced past. - */ - n -= (internal->node_ptrs[idx].all_nrec + 1); - } /* end for */ - } /* end else */ - - /* Decrement the number of redistribution retries left */ - retries--; - } /* end while */ - - /* Handle deleting a record from an internal node */ - if(!swap_loc && found) - swap_loc = H5B2_INT_NREC(internal, hdr, idx - 1); - - /* Swap record to delete with record from leaf, if we are the last internal node */ - if(swap_loc && depth == 1) - if(H5B2__swap_leaf(hdr, dxpl_id, depth, internal, &internal_flags, idx, swap_loc) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTSWAP, FAIL, "can't swap records in B-tree") - - /* Set pointers for advancing to child node */ - new_cache_info_flags_ptr = &internal_flags; - new_cache_info = &internal->cache_info; - new_node_ptr = &internal->node_ptrs[idx]; - - /* Indicate position of next node */ - if(H5B2_POS_MIDDLE != curr_pos) { - if(idx == 0) { - if(H5B2_POS_LEFT == curr_pos || H5B2_POS_ROOT == curr_pos) - next_pos = H5B2_POS_LEFT; - } /* end if */ - else if(idx == internal->nrec) { - if(H5B2_POS_RIGHT == curr_pos || H5B2_POS_ROOT == curr_pos) - next_pos = H5B2_POS_RIGHT; - } /* end if */ - } /* end if */ - } /* end else */ - - /* Attempt to remove record from child node */ - if(depth > 1) { - if(H5B2__remove_internal_by_idx(hdr, dxpl_id, depth_decreased, swap_loc, (uint16_t)(depth - 1), - new_cache_info, new_cache_info_flags_ptr, new_node_ptr, next_pos, n, op, op_data) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTDELETE, FAIL, "unable to remove record from B-tree internal node") - } /* end if */ - else { - if(H5B2__remove_leaf_by_idx(hdr, dxpl_id, new_node_ptr, next_pos, (unsigned)n, op, op_data) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTDELETE, FAIL, "unable to remove record from B-tree leaf node") - } /* end else */ - - /* Update record count for node pointer to child node */ - if(!collapsed_root) - new_node_ptr->all_nrec--; - - /* Mark node as dirty */ - internal_flags |= H5AC__DIRTIED_FLAG; - -#ifdef H5B2_DEBUG - H5B2__assert_internal((!collapsed_root ? (curr_node_ptr->all_nrec - 1) : new_node_ptr->all_nrec), hdr, internal); -#endif /* H5B2_DEBUG */ - -done: - /* Release the B-tree internal node */ - if(internal && H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, internal_addr, internal, internal_flags) < 0) - HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release internal B-tree node") - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5B2__remove_internal_by_idx() */ - - -/*------------------------------------------------------------------------- - * Function: H5B2__neighbor_leaf - * - * Purpose: Locate a record relative to the specified information in a - * B-tree leaf node and return that information by filling in - * fields of the - * caller-supplied UDATA pointer depending on the type of leaf node - * requested. The UDATA can point to additional data passed - * to the key comparison function. - * - * The 'OP' routine is called with the record found and the - * OP_DATA pointer, to allow caller to return information about - * the record. - * - * The RANGE indicates whether to search for records less than or - * equal to, or greater than or equal to the information passed - * in with UDATA. - * - * Return: Non-negative on success, negative on failure. - * - * Programmer: Quincey Koziol - * koziol@ncsa.uiuc.edu - * Mar 9 2005 - * - *------------------------------------------------------------------------- - */ -herr_t -H5B2__neighbor_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, H5B2_node_ptr_t *curr_node_ptr, - void *neighbor_loc, H5B2_compare_t comp, void *udata, H5B2_found_t op, - void *op_data) -{ - H5B2_leaf_t *leaf; /* Pointer to leaf node */ - unsigned idx; /* Location of record which matches key */ - int cmp = 0; /* Comparison value of records */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Check arguments. */ - HDassert(hdr); - HDassert(curr_node_ptr); - HDassert(H5F_addr_defined(curr_node_ptr->addr)); - HDassert(op); - - /* Lock current B-tree node */ - if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, curr_node_ptr->addr, curr_node_ptr->node_nrec, H5AC__READ_ONLY_FLAG))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") - - /* Locate node pointer for child */ - if(H5B2__locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx, &cmp) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") - if(cmp > 0) - idx++; - else - if(cmp == 0 && comp == H5B2_COMPARE_GREATER) - idx++; - - /* Set the neighbor location, if appropriate */ - if(comp == H5B2_COMPARE_LESS) { - if(idx > 0) - neighbor_loc = H5B2_LEAF_NREC(leaf, hdr, idx - 1); - } /* end if */ - else { - HDassert(comp == H5B2_COMPARE_GREATER); - - if(idx < leaf->nrec) - neighbor_loc = H5B2_LEAF_NREC(leaf, hdr, idx); - } /* end else */ - - /* Make callback if neighbor record has been found */ - if(neighbor_loc) { - /* Make callback for current record */ - if((op)(neighbor_loc, op_data) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, FAIL, "'found' callback failed for B-tree neighbor operation") - } /* end if */ - else - HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, FAIL, "unable to find neighbor record in B-tree") - -done: - /* Release the B-tree internal node */ - if(leaf && H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_LEAF, curr_node_ptr->addr, leaf, H5AC__NO_FLAGS_SET) < 0) - HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree leaf node") - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5B2__neighbor_leaf() */ - - -/*------------------------------------------------------------------------- - * Function: H5B2__neighbor_internal - * - * Purpose: Locate a record relative to the specified information in a - * B-tree internal node and return that information by filling in - * fields of the - * caller-supplied UDATA pointer depending on the type of leaf node - * requested. The UDATA can point to additional data passed - * to the key comparison function. - * - * The 'OP' routine is called with the record found and the - * OP_DATA pointer, to allow caller to return information about - * the record. - * - * The RANGE indicates whether to search for records less than or - * equal to, or greater than or equal to the information passed - * in with UDATA. - * - * Return: Non-negative on success, negative on failure. - * - * Programmer: Quincey Koziol - * koziol@ncsa.uiuc.edu - * Mar 9 2005 - * - *------------------------------------------------------------------------- - */ -herr_t -H5B2__neighbor_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, - H5B2_node_ptr_t *curr_node_ptr, void *neighbor_loc, H5B2_compare_t comp, - void *udata, H5B2_found_t op, void *op_data) -{ - H5B2_internal_t *internal; /* Pointer to internal node */ - unsigned idx; /* Location of record which matches key */ - int cmp = 0; /* Comparison value of records */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Check arguments. */ - HDassert(hdr); - HDassert(depth > 0); - HDassert(curr_node_ptr); - HDassert(H5F_addr_defined(curr_node_ptr->addr)); - HDassert(op); - - /* Lock current B-tree node */ - if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, curr_node_ptr->addr, curr_node_ptr->node_nrec, depth, H5AC__READ_ONLY_FLAG))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") - - /* Locate node pointer for child */ - if(H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, - udata, &idx, &cmp) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") - if(cmp > 0) - idx++; - - /* Set the neighbor location, if appropriate */ - if(comp == H5B2_COMPARE_LESS) { - if(idx > 0) - neighbor_loc = H5B2_INT_NREC(internal, hdr, idx - 1); - } /* end if */ - else { - HDassert(comp == H5B2_COMPARE_GREATER); - - if(idx < internal->nrec) - neighbor_loc = H5B2_INT_NREC(internal, hdr, idx); - } /* end else */ - - /* Attempt to find neighboring record */ - if(depth > 1) { - if(H5B2__neighbor_internal(hdr, dxpl_id, (uint16_t)(depth - 1), &internal->node_ptrs[idx], neighbor_loc, comp, udata, op, op_data) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, FAIL, "unable to find neighbor record in B-tree internal node") - } /* end if */ - else { - if(H5B2__neighbor_leaf(hdr, dxpl_id, &internal->node_ptrs[idx], neighbor_loc, comp, udata, op, op_data) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, FAIL, "unable to find neighbor record in B-tree leaf node") - } /* end else */ - -done: - /* Release the B-tree internal node */ - if(internal && H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr->addr, internal, H5AC__NO_FLAGS_SET) < 0) - HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release internal B-tree node") - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5B2__neighbor_internal() */ - - -/*------------------------------------------------------------------------- * Function: H5B2__delete_node * * Purpose: Iterate over all the nodes in a B-tree node deleting them @@ -3392,7 +1624,8 @@ done: */ herr_t H5B2__delete_node(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, - const H5B2_node_ptr_t *curr_node, H5B2_remove_t op, void *op_data) + const H5B2_node_ptr_t *curr_node, void *parent, H5B2_remove_t op, + void *op_data) { const H5AC_class_t *curr_node_class = NULL; /* Pointer to current node's class info */ void *node = NULL; /* Pointers to current node */ @@ -3410,7 +1643,7 @@ H5B2__delete_node(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, unsigned u; /* Local index */ /* Lock the current B-tree node */ - if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, curr_node->addr, curr_node->node_nrec, depth, H5AC__NO_FLAGS_SET))) + if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, parent, (H5B2_node_ptr_t *)curr_node, depth, FALSE, H5AC__NO_FLAGS_SET))) /* Casting away const OK -QAK */ HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") /* Set up information about current node */ @@ -3420,14 +1653,14 @@ H5B2__delete_node(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Descend into children */ for(u = 0; u < internal->nrec + (unsigned)1; u++) - if(H5B2__delete_node(hdr, dxpl_id, (uint16_t)(depth - 1), &(internal->node_ptrs[u]), op, op_data) < 0) + if(H5B2__delete_node(hdr, dxpl_id, (uint16_t)(depth - 1), &(internal->node_ptrs[u]), internal, op, op_data) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTLIST, FAIL, "node descent failed") } /* end if */ else { H5B2_leaf_t *leaf; /* Pointer to leaf node */ /* Lock the current B-tree node */ - if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, curr_node->addr, curr_node->node_nrec, H5AC__NO_FLAGS_SET))) + if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, parent, (H5B2_node_ptr_t *)curr_node, FALSE, H5AC__NO_FLAGS_SET))) /* Casting away const OK -QAK */ HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") /* Set up information about current node */ @@ -3450,7 +1683,7 @@ H5B2__delete_node(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, done: /* Unlock & delete current node */ - if(node && H5AC_unprotect(hdr->f, dxpl_id, curr_node_class, curr_node->addr, node, H5AC__DELETED_FLAG | H5AC__FREE_FILE_SPACE_FLAG) < 0) + if(node && H5AC_unprotect(hdr->f, dxpl_id, curr_node_class, curr_node->addr, node, (unsigned)(H5AC__DELETED_FLAG | (hdr->swmr_write ? 0 : H5AC__FREE_FILE_SPACE_FLAG))) < 0) HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree node") FUNC_LEAVE_NOAPI(ret_value) @@ -3472,7 +1705,7 @@ done: */ herr_t H5B2__node_size(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, - const H5B2_node_ptr_t *curr_node, hsize_t *btree_size) + const H5B2_node_ptr_t *curr_node, void *parent, hsize_t *btree_size) { H5B2_internal_t *internal = NULL; /* Pointer to internal node */ herr_t ret_value = SUCCEED; /* Iterator return value */ @@ -3486,7 +1719,7 @@ H5B2__node_size(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, HDassert(depth > 0); /* Lock the current B-tree node */ - if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, curr_node->addr, curr_node->node_nrec, depth, H5AC__READ_ONLY_FLAG))) + if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, parent, (H5B2_node_ptr_t *)curr_node, depth, FALSE, H5AC__READ_ONLY_FLAG))) /* Casting away const OK -QAK */ HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") /* Recursively descend into child nodes, if we are above the "twig" level in the B-tree */ @@ -3495,7 +1728,7 @@ H5B2__node_size(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, /* Descend into children */ for(u = 0; u < internal->nrec + (unsigned)1; u++) - if(H5B2__node_size(hdr, dxpl_id, (uint16_t)(depth - 1), &(internal->node_ptrs[u]), btree_size) < 0) + if(H5B2__node_size(hdr, dxpl_id, (uint16_t)(depth - 1), &(internal->node_ptrs[u]), internal, btree_size) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTLIST, FAIL, "node iteration failed") } /* end if */ else /* depth is 1: count all the leaf nodes from this node */ @@ -3513,219 +1746,205 @@ done: /*------------------------------------------------------------------------- - * Function: H5B2__internal_free + * Function: H5B2__create_flush_depend * - * Purpose: Destroys a B-tree internal node in memory. + * Purpose: Create a flush dependency between two data structure components * - * Return: Non-negative on success/Negative on failure + * Return: SUCCEED/FAIL * - * Programmer: Quincey Koziol - * koziol@ncsa.uiuc.edu - * Feb 2 2005 + * Programmer: Dana Robinson + * Fall 2012 * *------------------------------------------------------------------------- */ herr_t -H5B2__internal_free(H5B2_internal_t *internal) +H5B2__create_flush_depend(H5AC_info_t *parent_entry, H5AC_info_t *child_entry) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_PACKAGE - - /* - * Check arguments. - */ - HDassert(internal); - - /* Release internal node's native key buffer */ - if(internal->int_native) - internal->int_native = (uint8_t *)H5FL_FAC_FREE(internal->hdr->node_info[internal->depth].nat_rec_fac, internal->int_native); - - /* Release internal node's node pointer buffer */ - if(internal->node_ptrs) - internal->node_ptrs = (H5B2_node_ptr_t *)H5FL_FAC_FREE(internal->hdr->node_info[internal->depth].node_ptr_fac, internal->node_ptrs); - - /* Decrement ref. count on B-tree header */ - if(H5B2__hdr_decr(internal->hdr) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTDEC, FAIL, "can't decrement ref. count on B-tree header") + FUNC_ENTER_NOAPI_NOINIT + + /* Sanity check */ + HDassert(parent_entry); + HDassert(child_entry); - /* Free B-tree internal node info */ - internal = H5FL_FREE(H5B2_internal_t, internal); + /* Create a flush dependency between parent and child entry */ + if(H5AC_create_flush_dependency(parent_entry, child_entry) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTDEPEND, FAIL, "unable to create flush dependency") done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5B2__internal_free() */ +} /* end H5B2__create_flush_depend() */ /*------------------------------------------------------------------------- - * Function: H5B2__leaf_free + * Function: H5B2__update_flush_depend * - * Purpose: Destroys a B-tree leaf node in memory. + * Purpose: Update flush dependencies for children of a node * - * Return: Non-negative on success/Negative on failure + * Return: SUCCEED/FAIL * - * Programmer: Quincey Koziol - * koziol@ncsa.uiuc.edu - * Feb 2 2005 + * Programmer: Quincey Koziol + * koziol@lbl.gov + * Dec 1 2016 * *------------------------------------------------------------------------- */ herr_t -H5B2__leaf_free(H5B2_leaf_t *leaf) +H5B2__update_flush_depend(H5B2_hdr_t *hdr, hid_t dxpl_id, unsigned depth, + const H5B2_node_ptr_t *node_ptr, void *old_parent, void *new_parent) { + const H5AC_class_t *child_class; /* Pointer to child node's class info */ + void *child = NULL; /* Pointer to child node */ + unsigned node_status = 0; /* Node's status in the metadata cache */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE + + /* Sanity checks */ + HDassert(hdr); + HDassert(depth > 0); + HDassert(node_ptr); + HDassert(old_parent); + HDassert(new_parent); - /* - * Check arguments. - */ - HDassert(leaf); + /* Check the node's entry status in the metadata cache */ + if(H5AC_get_entry_status(hdr->f, node_ptr->addr, &node_status) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "unable to check status of B-tree node") - /* Release leaf's native key buffer */ - if(leaf->leaf_native) - leaf->leaf_native = (uint8_t *)H5FL_FAC_FREE(leaf->hdr->node_info[0].nat_rec_fac, leaf->leaf_native); + /* If the node is in the cache, check for retargeting its parent */ + if(node_status & H5AC_ES__IN_CACHE) { + void **parent_ptr; /* Pointer to child node's parent */ + hbool_t update_deps = FALSE; /* Whether to update flush dependencies */ - /* Decrement ref. count on B-tree header */ - if(H5B2__hdr_decr(leaf->hdr) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTDEC, FAIL, "can't decrement ref. count on B-tree header") + /* Get child node pointer */ + if(depth > 1) { + H5B2_internal_t *child_int; - /* Free B-tree leaf node info */ - leaf = H5FL_FREE(H5B2_leaf_t, leaf); + /* Protect child */ + if(NULL == (child_int = H5B2__protect_internal(hdr, dxpl_id, new_parent, (H5B2_node_ptr_t *)node_ptr, (uint16_t)(depth - 1), FALSE, H5AC__NO_FLAGS_SET))) /* Casting away const OK -QAK */ + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") + child_class = H5AC_BT2_INT; + child = child_int; -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5B2__leaf_free() */ + if(child_int->parent == old_parent) { + parent_ptr = &child_int->parent; + update_deps = TRUE; + } /* end if */ + else + HDassert(child_int->parent == new_parent); + } /* end if */ + else { + H5B2_leaf_t *child_leaf; -#ifdef H5B2_DEBUG - -/*------------------------------------------------------------------------- - * Function: H5B2__assert_leaf - * - * Purpose: Verify than a leaf node is mostly sane - * - * Return: Non-negative on success, negative on failure - * - * Programmer: Quincey Koziol - * koziol@ncsa.uiuc.edu - * Feb 19 2005 - * - *------------------------------------------------------------------------- - */ -H5_ATTR_PURE static herr_t -H5B2__assert_leaf(const H5B2_hdr_t *hdr, const H5B2_leaf_t *leaf) -{ - /* General sanity checking on node */ - HDassert(leaf->nrec <= hdr->node_info->split_nrec); + /* Protect child */ + if(NULL == (child_leaf = H5B2__protect_leaf(hdr, dxpl_id, new_parent, (H5B2_node_ptr_t *)node_ptr, FALSE, H5AC__NO_FLAGS_SET))) /* Casting away const OK -QAK */ + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") + child_class = H5AC_BT2_LEAF; + child = child_leaf; - return(0); -} /* end H5B2__assert_leaf() */ + if(child_leaf->parent == old_parent) { + parent_ptr = &child_leaf->parent; + update_deps = TRUE; + } /* end if */ + else + HDassert(child_leaf->parent == new_parent); + } /* end else */ - -/*------------------------------------------------------------------------- - * Function: H5B2__assert_leaf2 - * - * Purpose: Verify than a leaf node is mostly sane - * - * Return: Non-negative on success, negative on failure - * - * Programmer: Quincey Koziol - * koziol@ncsa.uiuc.edu - * Feb 19 2005 - * - *------------------------------------------------------------------------- - */ -H5_ATTR_PURE static herr_t -H5B2__assert_leaf2(const H5B2_hdr_t *hdr, const H5B2_leaf_t *leaf, const H5B2_leaf_t H5_ATTR_UNUSED *leaf2) -{ - /* General sanity checking on node */ - HDassert(leaf->nrec <= hdr->node_info->split_nrec); + /* Update flush dependencies if necessary */ + if(update_deps) { + /* Sanity check */ + HDassert(parent_ptr); + + /* Switch the flush dependency for the node */ + if(H5B2__destroy_flush_depend((H5AC_info_t *)old_parent, (H5AC_info_t *)child) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNDEPEND, FAIL, "unable to destroy flush dependency") + *parent_ptr = new_parent; + if(H5B2__create_flush_depend((H5AC_info_t *)new_parent, (H5AC_info_t *)child) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTDEPEND, FAIL, "unable to create flush dependency") + } /* end if */ + } /* end if */ - return(0); -} /* end H5B2__assert_leaf2() */ +done: + /* Unprotect the child */ + if(child) + if(H5AC_unprotect(hdr->f, dxpl_id, child_class, node_ptr->addr, child, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree node") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B2__update_flush_depend() */ /*------------------------------------------------------------------------- - * Function: H5B2__assert_internal + * Function: H5B2__update_child_flush_depends * - * Purpose: Verify than an internal node is mostly sane + * Purpose: Update flush dependencies for children of a node * - * Return: Non-negative on success, negative on failure + * Return: SUCCEED/FAIL * - * Programmer: Quincey Koziol - * koziol@ncsa.uiuc.edu - * Feb 19 2005 + * Programmer: Quincey Koziol + * koziol@lbl.gov + * Dec 1 2016 * *------------------------------------------------------------------------- */ -H5_ATTR_PURE static herr_t -H5B2__assert_internal(hsize_t parent_all_nrec, const H5B2_hdr_t *hdr, const H5B2_internal_t *internal) +static herr_t +H5B2__update_child_flush_depends(H5B2_hdr_t *hdr, hid_t dxpl_id, unsigned depth, + const H5B2_node_ptr_t *node_ptrs, unsigned start_idx, unsigned end_idx, + void *old_parent, void *new_parent) { - hsize_t tot_all_nrec; /* Total number of records at or below this node */ - uint16_t u, v; /* Local index variables */ - - /* General sanity checking on node */ - HDassert(internal->nrec <= hdr->node_info->split_nrec); - - /* Sanity checking on node pointers */ - tot_all_nrec = internal->nrec; - for(u = 0; u < internal->nrec + 1; u++) { - tot_all_nrec += internal->node_ptrs[u].all_nrec; - - HDassert(H5F_addr_defined(internal->node_ptrs[u].addr)); - HDassert(internal->node_ptrs[u].addr > 0); - for(v = 0; v < u; v++) - HDassert(internal->node_ptrs[u].addr != internal->node_ptrs[v].addr); - } /* end for */ + unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ - /* Sanity check all_nrec total in parent */ - if(parent_all_nrec > 0) - HDassert(tot_all_nrec == parent_all_nrec); + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(hdr); + HDassert(depth > 1); + HDassert(node_ptrs); + HDassert(start_idx <= end_idx); + HDassert(old_parent); + HDassert(new_parent); + + /* Loop over children */ + for(u = start_idx; u < end_idx; u++) + /* Update parent for children */ + if(H5B2__update_flush_depend(hdr, dxpl_id, depth - 1, &node_ptrs[u], old_parent, new_parent) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update child node to new parent") - return(0); -} /* end H5B2__assert_internal() */ +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B2__update_child_flush_depends() */ /*------------------------------------------------------------------------- - * Function: H5B2__assert_internal2 + * Function: H5B2__destroy_flush_depend * - * Purpose: Verify than internal nodes are mostly sane + * Purpose: Destroy a flush dependency between two data structure components * - * Return: Non-negative on success, negative on failure + * Return: SUCCEED/FAIL * - * Programmer: Quincey Koziol - * koziol@ncsa.uiuc.edu - * Feb 19 2005 + * Programmer: Dana Robinson + * Fall 2012 * *------------------------------------------------------------------------- */ -H5_ATTR_PURE static herr_t -H5B2__assert_internal2(hsize_t parent_all_nrec, const H5B2_hdr_t *hdr, const H5B2_internal_t *internal, const H5B2_internal_t *internal2) +herr_t +H5B2__destroy_flush_depend(H5AC_info_t *parent_entry, H5AC_info_t *child_entry) { - hsize_t tot_all_nrec; /* Total number of records at or below this node */ - uint16_t u, v; /* Local index variables */ - - /* General sanity checking on node */ - HDassert(internal->nrec <= hdr->node_info->split_nrec); - - /* Sanity checking on node pointers */ - tot_all_nrec =internal->nrec; - for(u =0; u < internal->nrec + 1; u++) { - tot_all_nrec += internal->node_ptrs[u].all_nrec; - - HDassert(H5F_addr_defined(internal->node_ptrs[u].addr)); - HDassert(internal->node_ptrs[u].addr > 0); - for(v = 0; v < u; v++) - HDassert(internal->node_ptrs[u].addr != internal->node_ptrs[v].addr); - for(v = 0; v < internal2->nrec + 1; v++) - HDassert(internal->node_ptrs[u].addr != internal2->node_ptrs[v].addr); - } /* end for */ + herr_t ret_value = SUCCEED; /* Return value */ - /* Sanity check all_nrec total in parent */ - if(parent_all_nrec > 0) - HDassert(tot_all_nrec == parent_all_nrec); + FUNC_ENTER_NOAPI_NOINIT + + /* Sanity check */ + HDassert(parent_entry); + HDassert(child_entry); - return(0); -} /* end H5B2__assert_internal2() */ -#endif /* H5B2_DEBUG */ + /* Destroy a flush dependency between parent and child entry */ + if(H5AC_destroy_flush_dependency(parent_entry, child_entry) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNDEPEND, FAIL, "unable to destroy flush dependency") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B2__destroy_flush_depend() */ diff --git a/src/H5B2internal.c b/src/H5B2internal.c new file mode 100644 index 0000000..e74ae59 --- /dev/null +++ b/src/H5B2internal.c @@ -0,0 +1,1443 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5B2internal.c + * Dec 01 2016 + * Quincey Koziol <koziol@lbl.gov> + * + * Purpose: Routines for managing v2 B-tree internal ndoes. + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5B2module.h" /* This source code file is part of the H5B2 module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5B2pkg.h" /* v2 B-trees */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5MFprivate.h" /* File memory management */ + + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ +static herr_t H5B2__shadow_internal(H5B2_internal_t *internal, hid_t dxpl_id, + H5B2_node_ptr_t *curr_node_ptr); + + +/*********************/ +/* Package Variables */ +/*********************/ + +/* Declare a free list to manage the H5B2_internal_t struct */ +H5FL_DEFINE(H5B2_internal_t); + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + + + +/*------------------------------------------------------------------------- + * Function: H5B2__create_internal + * + * Purpose: Creates empty internal node of a B-tree and update node pointer + * to point to it. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@ncsa.uiuc.edu + * Feb 3 2005 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2__create_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, void *parent, + H5B2_node_ptr_t *node_ptr, uint16_t depth) +{ + H5B2_internal_t *internal = NULL; /* Pointer to new internal node created */ + hbool_t inserted = FALSE; /* Whether the internal node was inserted into cache */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check arguments. */ + HDassert(hdr); + HDassert(node_ptr); + HDassert(depth > 0); + + /* Allocate memory for internal node information */ + if(NULL == (internal = H5FL_CALLOC(H5B2_internal_t))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree internal info") + + /* Increment ref. count on B-tree header */ + if(H5B2__hdr_incr(hdr) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTINC, FAIL, "can't increment ref. count on B-tree header") + + /* Share B-tree header information */ + internal->hdr = hdr; + + /* Allocate space for the native keys in memory */ + if(NULL == (internal->int_native = (uint8_t *)H5FL_FAC_MALLOC(hdr->node_info[depth].nat_rec_fac))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree internal native keys") + HDmemset(internal->int_native, 0, hdr->cls->nrec_size * hdr->node_info[depth].max_nrec); + + /* Allocate space for the node pointers in memory */ + if(NULL == (internal->node_ptrs = (H5B2_node_ptr_t *)H5FL_FAC_MALLOC(hdr->node_info[depth].node_ptr_fac))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree internal node pointers") + HDmemset(internal->node_ptrs, 0, sizeof(H5B2_node_ptr_t) * (hdr->node_info[depth].max_nrec + 1)); + + /* Set depth of the node */ + internal->depth = depth; + + /* Set parent */ + internal->parent = parent; + + /* Set shadowed epoch to header's epoch */ + internal->shadow_epoch = hdr->shadow_epoch; + + /* Allocate space on disk for the internal node */ + if(HADDR_UNDEF == (node_ptr->addr = H5MF_alloc(hdr->f, H5FD_MEM_BTREE, dxpl_id, (hsize_t)hdr->node_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "file allocation failed for B-tree internal node") + + /* Cache the new B-tree node */ + if(H5AC_insert_entry(hdr->f, dxpl_id, H5AC_BT2_INT, node_ptr->addr, internal, H5AC__NO_FLAGS_SET) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, FAIL, "can't add B-tree internal node to cache") + inserted = TRUE; + + /* Add internal node as child of 'top' proxy */ + if(hdr->top_proxy) { + if(H5AC_proxy_entry_add_child(hdr->top_proxy, hdr->f, dxpl_id, internal) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSET, FAIL, "unable to add v2 B-tree node as child of proxy") + internal->top_proxy = hdr->top_proxy; + } /* end if */ + +done: + if(ret_value < 0) { + if(internal) { + /* Remove from cache, if inserted */ + if(inserted) + if(H5AC_remove_entry(internal) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTREMOVE, FAIL, "unable to remove v2 B-tree internal node from cache") + + /* Release internal node's disk space */ + if(H5F_addr_defined(node_ptr->addr) && H5MF_xfree(hdr->f, H5FD_MEM_BTREE, dxpl_id, node_ptr->addr, (hsize_t)hdr->node_size) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTFREE, FAIL, "unable to release file space for v2 B-tree internal node") + + /* Destroy internal node */ + if(H5B2__internal_free(internal) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTFREE, FAIL, "unable to release v2 B-tree internal node") + } /* end if */ + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5B2__create_internal() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__protect_internal + * + * Purpose: "Protect" an internal node in the metadata cache + * + * Return: Pointer to internal node on success/NULL on failure + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Aug 25 2006 + * + *------------------------------------------------------------------------- + */ +H5B2_internal_t * +H5B2__protect_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, void *parent, + H5B2_node_ptr_t *node_ptr, uint16_t depth, hbool_t shadow, unsigned flags) +{ + H5B2_internal_cache_ud_t udata; /* User data to pass through to cache 'deserialize' callback */ + H5B2_internal_t *internal = NULL; /* v2 B-tree internal node */ + H5B2_internal_t *ret_value = NULL; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check arguments. */ + HDassert(hdr); + HDassert(node_ptr); + HDassert(H5F_addr_defined(node_ptr->addr)); + HDassert(depth > 0); + + /* only H5AC__READ_ONLY_FLAG may appear in flags */ + HDassert((flags & (unsigned)(~H5AC__READ_ONLY_FLAG)) == 0); + + /* Set up user data for callback */ + udata.f = hdr->f; + udata.hdr = hdr; + udata.parent = parent; + udata.nrec = node_ptr->node_nrec; + udata.depth = depth; + + /* Protect the internal node */ + if(NULL == (internal = (H5B2_internal_t *)H5AC_protect(hdr->f, dxpl_id, H5AC_BT2_INT, node_ptr->addr, &udata, flags))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, NULL, "unable to protect B-tree internal node") + + /* Create top proxy, if it doesn't exist */ + if(hdr->top_proxy && NULL == internal->top_proxy) { + /* Add internal node as child of 'top' proxy */ + if(H5AC_proxy_entry_add_child(hdr->top_proxy, hdr->f, dxpl_id, internal) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSET, NULL, "unable to add v2 B-tree internal node as child of proxy") + internal->top_proxy = hdr->top_proxy; + } /* end if */ + + /* Shadow the node, if requested */ + if(shadow) + if(H5B2__shadow_internal(internal, dxpl_id, node_ptr) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCOPY, NULL, "unable to shadow internal node") + + /* Set return value */ + ret_value = internal; + +done: + /* Clean up on error */ + if(!ret_value) { + /* Release the internal node, if it was protected */ + if(internal) { + /* Remove from v2 B-tree's proxy, if added */ + if(internal->top_proxy) { + if(H5AC_proxy_entry_remove_child(internal->top_proxy, internal) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNDEPEND, NULL, "unable to destroy flush dependency between internal node and v2 B-tree 'top' proxy") + internal->top_proxy = NULL; + } /* end if */ + + /* Unprotect internal node */ + if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, node_ptr->addr, internal, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, NULL, "unable to unprotect v2 B-tree internal node, address = %llu", (unsigned long long)node_ptr->addr) + } /* end if */ + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5B2__protect_internal() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__neighbor_internal + * + * Purpose: Locate a record relative to the specified information in a + * B-tree internal node and return that information by filling in + * fields of the + * caller-supplied UDATA pointer depending on the type of leaf node + * requested. The UDATA can point to additional data passed + * to the key comparison function. + * + * The 'OP' routine is called with the record found and the + * OP_DATA pointer, to allow caller to return information about + * the record. + * + * The RANGE indicates whether to search for records less than or + * equal to, or greater than or equal to the information passed + * in with UDATA. + * + * Return: Non-negative on success, negative on failure. + * + * Programmer: Quincey Koziol + * koziol@ncsa.uiuc.edu + * Mar 9 2005 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2__neighbor_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, + H5B2_node_ptr_t *curr_node_ptr, void *neighbor_loc, H5B2_compare_t comp, + void *parent, void *udata, H5B2_found_t op, void *op_data) +{ + H5B2_internal_t *internal; /* Pointer to internal node */ + unsigned idx = 0; /* Location of record which matches key */ + int cmp = 0; /* Comparison value of records */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check arguments. */ + HDassert(hdr); + HDassert(depth > 0); + HDassert(curr_node_ptr); + HDassert(H5F_addr_defined(curr_node_ptr->addr)); + HDassert(op); + + /* Lock current B-tree node */ + if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, parent, curr_node_ptr, depth, FALSE, H5AC__READ_ONLY_FLAG))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") + + /* Locate node pointer for child */ + if(H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx, &cmp) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") + if(cmp > 0) + idx++; + + /* Set the neighbor location, if appropriate */ + if(comp == H5B2_COMPARE_LESS) { + if(idx > 0) + neighbor_loc = H5B2_INT_NREC(internal, hdr, idx - 1); + } /* end if */ + else { + HDassert(comp == H5B2_COMPARE_GREATER); + + if(idx < internal->nrec) + neighbor_loc = H5B2_INT_NREC(internal, hdr, idx); + } /* end else */ + + /* Attempt to find neighboring record */ + if(depth > 1) { + if(H5B2__neighbor_internal(hdr, dxpl_id, (uint16_t)(depth - 1), &internal->node_ptrs[idx], neighbor_loc, comp, internal, udata, op, op_data) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, FAIL, "unable to find neighbor record in B-tree internal node") + } /* end if */ + else { + if(H5B2__neighbor_leaf(hdr, dxpl_id, &internal->node_ptrs[idx], neighbor_loc, comp, internal, udata, op, op_data) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, FAIL, "unable to find neighbor record in B-tree leaf node") + } /* end else */ + +done: + /* Release the B-tree internal node */ + if(internal && H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr->addr, internal, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release internal B-tree node") + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5B2__neighbor_internal() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__insert_internal + * + * Purpose: Adds a new record to a B-tree node. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@ncsa.uiuc.edu + * Mar 2 2005 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2__insert_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, + unsigned *parent_cache_info_flags_ptr, H5B2_node_ptr_t *curr_node_ptr, + H5B2_nodepos_t curr_pos, void *parent, void *udata) +{ + H5B2_internal_t *internal = NULL; /* Pointer to internal node */ + unsigned internal_flags = H5AC__NO_FLAGS_SET; + unsigned idx = 0; /* Location of record which matches key */ + H5B2_nodepos_t next_pos = H5B2_POS_MIDDLE; /* Position of node */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check arguments. */ + HDassert(hdr); + HDassert(depth > 0); + HDassert(curr_node_ptr); + HDassert(H5F_addr_defined(curr_node_ptr->addr)); + + /* Lock current B-tree node */ + if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, parent, curr_node_ptr, depth, FALSE, H5AC__NO_FLAGS_SET))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") + + /* Sanity check number of records */ + HDassert(internal->nrec == curr_node_ptr->node_nrec); + + /* Split or redistribute child node pointers, if necessary */ + { + int cmp; /* Comparison value of records */ + unsigned retries; /* Number of times to attempt redistribution */ + size_t split_nrec; /* Number of records to split node at */ + + /* Locate node pointer for child */ + if(H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, + udata, &idx, &cmp) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") + if(cmp == 0) + HGOTO_ERROR(H5E_BTREE, H5E_EXISTS, FAIL, "record is already in B-tree") + if(cmp > 0) + idx++; + + /* Set the number of redistribution retries */ + /* This takes care of the case where a B-tree node needs to be + * redistributed, but redistributing the node causes the index + * for insertion to move to another node, which also needs to be + * redistributed. Now, we loop trying to redistribute and then + * eventually force a split */ + retries = 2; + + /* Determine the correct number of records to split child node at */ + split_nrec = hdr->node_info[depth - 1].split_nrec; + + /* Preemptively split/redistribute a node we will enter */ + while(internal->node_ptrs[idx].node_nrec == split_nrec) { + /* Attempt to redistribute records among children */ + if(idx == 0) { /* Left-most child */ + if(retries > 0 && (internal->node_ptrs[idx + 1].node_nrec < split_nrec)) { + if(H5B2__redistribute2(hdr, dxpl_id, depth, internal, idx) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTREDISTRIBUTE, FAIL, "unable to redistribute child node records") + } /* end if */ + else { + if(H5B2__split1(hdr, dxpl_id, depth, curr_node_ptr, + parent_cache_info_flags_ptr, internal, &internal_flags, idx) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to split child node") + } /* end else */ + } /* end if */ + else if(idx == internal->nrec) { /* Right-most child */ + if(retries > 0 && (internal->node_ptrs[idx - 1].node_nrec < split_nrec)) { + if(H5B2__redistribute2(hdr, dxpl_id, depth, internal, (idx - 1)) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTREDISTRIBUTE, FAIL, "unable to redistribute child node records") + } /* end if */ + else { + if(H5B2__split1(hdr, dxpl_id, depth, curr_node_ptr, + parent_cache_info_flags_ptr, internal, &internal_flags, idx) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to split child node") + } /* end else */ + } /* end if */ + else { /* Middle child */ + if(retries > 0 && ((internal->node_ptrs[idx + 1].node_nrec < split_nrec) || + (internal->node_ptrs[idx - 1].node_nrec < split_nrec))) { + if(H5B2__redistribute3(hdr, dxpl_id, depth, internal, &internal_flags, idx) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTREDISTRIBUTE, FAIL, "unable to redistribute child node records") + } /* end if */ + else { + if(H5B2__split1(hdr, dxpl_id, depth, curr_node_ptr, + parent_cache_info_flags_ptr, internal, &internal_flags, idx) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to split child node") + } /* end else */ + } /* end else */ + + /* Locate node pointer for child (after split/redistribute) */ + /* Actually, this can be easily updated (for 2-node redistrib.) and shouldn't require re-searching */ + if(H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, + udata, &idx, &cmp) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") + if(cmp == 0) + HGOTO_ERROR(H5E_BTREE, H5E_EXISTS, FAIL, "record is already in B-tree") + if(cmp > 0) + idx++; + + /* Decrement the number of redistribution retries left */ + retries--; + } /* end while */ + } /* end block */ + + /* Check if this node is left/right-most */ + if(H5B2_POS_MIDDLE != curr_pos) { + if(idx == 0) { + if(H5B2_POS_LEFT == curr_pos || H5B2_POS_ROOT == curr_pos) + next_pos = H5B2_POS_LEFT; + } /* end if */ + else if(idx == internal->nrec) { + if(H5B2_POS_RIGHT == curr_pos || H5B2_POS_ROOT == curr_pos) + next_pos = H5B2_POS_RIGHT; + } /* end else */ + } /* end if */ + + /* Attempt to insert node */ + if(depth > 1) { + if(H5B2__insert_internal(hdr, dxpl_id, (uint16_t)(depth - 1), &internal_flags, &internal->node_ptrs[idx], next_pos, internal, udata) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, FAIL, "unable to insert record into B-tree internal node") + } /* end if */ + else { + if(H5B2__insert_leaf(hdr, dxpl_id, &internal->node_ptrs[idx], next_pos, internal, udata) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, FAIL, "unable to insert record into B-tree leaf node") + } /* end else */ + + /* Update record count for node pointer to current node */ + curr_node_ptr->all_nrec++; + + /* Mark node as dirty */ + internal_flags |= H5AC__DIRTIED_FLAG; + +done: + /* Release the B-tree internal node */ + if(internal) { + /* Shadow the node if doing SWMR writes */ + if(hdr->swmr_write && (internal_flags & H5AC__DIRTIED_FLAG)) + if(H5B2__shadow_internal(internal, dxpl_id, curr_node_ptr) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTCOPY, FAIL, "unable to shadow internal B-tree node") + + /* Unprotect node */ + if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr->addr, internal, internal_flags) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release internal B-tree node") + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5B2__insert_internal() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__update_internal + * + * Purpose: Insert or modify a record in a B-tree internal node. + * If the record exists already, it is modified as if H5B2_modify + * was called). If it doesn't exist, it is inserted as if + * H5B2_insert was called. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Dec 24 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2__update_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, + unsigned *parent_cache_info_flags_ptr, H5B2_node_ptr_t *curr_node_ptr, + H5B2_update_status_t *status, H5B2_nodepos_t curr_pos, void *parent, + void *udata, H5B2_modify_t op, void *op_data) +{ + H5B2_internal_t *internal = NULL; /* Pointer to internal node */ + unsigned internal_flags = H5AC__NO_FLAGS_SET; + int cmp; /* Comparison value of records */ + unsigned idx = 0; /* Location of record which matches key */ + H5B2_nodepos_t next_pos = H5B2_POS_MIDDLE; /* Position of node */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check arguments. */ + HDassert(hdr); + HDassert(depth > 0); + HDassert(curr_node_ptr); + HDassert(H5F_addr_defined(curr_node_ptr->addr)); + + /* Lock current B-tree node */ + if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, parent, curr_node_ptr, depth, FALSE, H5AC__NO_FLAGS_SET))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") + + /* Sanity check number of records */ + HDassert(internal->nrec == curr_node_ptr->node_nrec); + + /* Locate node pointer for child */ + if(H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx, &cmp) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") + + /* Check for modifying existing record */ + if(0 == cmp) { + hbool_t changed = FALSE; /* Whether the 'modify' callback changed the record */ + + /* Make callback for current record */ + if((op)(H5B2_INT_NREC(internal, hdr, idx), op_data, &changed) < 0) { + /* Make certain that the callback didn't modify the value if it failed */ + HDassert(changed == FALSE); + + HGOTO_ERROR(H5E_BTREE, H5E_CANTMODIFY, FAIL, "'modify' callback failed for B-tree update operation") + } /* end if */ + + /* Mark the node as dirty if it changed */ + internal_flags |= (changed ? H5AC__DIRTIED_FLAG : 0); + + /* Indicate that the record was modified */ + *status = H5B2_UPDATE_MODIFY_DONE; + } /* end if */ + else { + /* Adjust index to leave room for node to insert */ + if(cmp > 0) + idx++; + + /* Check if this node is left/right-most */ + if(H5B2_POS_MIDDLE != curr_pos) { + if(idx == 0) { + if(H5B2_POS_LEFT == curr_pos || H5B2_POS_ROOT == curr_pos) + next_pos = H5B2_POS_LEFT; + } /* end if */ + else if(idx == internal->nrec) { + if(H5B2_POS_RIGHT == curr_pos || H5B2_POS_ROOT == curr_pos) + next_pos = H5B2_POS_RIGHT; + } /* end else */ + } /* end if */ + + /* Attempt to update record in child */ + if(depth > 1) { + if(H5B2__update_internal(hdr, dxpl_id, (uint16_t)(depth - 1), &internal_flags, &internal->node_ptrs[idx], status, next_pos, internal, udata, op, op_data) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update record in internal B-tree node") + } /* end if */ + else { + if(H5B2__update_leaf(hdr, dxpl_id, &internal->node_ptrs[idx], status, next_pos, internal, udata, op, op_data) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update record in leaf B-tree node") + } /* end else */ + + /* Take actions based on child's status report */ + switch(*status) { + case H5B2_UPDATE_MODIFY_DONE: + /* No action */ + break; + + case H5B2_UPDATE_SHADOW_DONE: + /* If child node was shadowed (if SWMR is enabled), mark this node dirty */ + if(hdr->swmr_write) + internal_flags |= H5AC__DIRTIED_FLAG; + + /* No further modifications up the tree are necessary though, so downgrade to merely "modified" */ + *status = H5B2_UPDATE_MODIFY_DONE; + break; + + case H5B2_UPDATE_INSERT_DONE: + /* Mark node as dirty */ + internal_flags |= H5AC__DIRTIED_FLAG; + + /* Update total record count for node pointer to current node */ + curr_node_ptr->all_nrec++; + break; + + case H5B2_UPDATE_INSERT_CHILD_FULL: + /* Split/redistribute this node */ + if(internal->nrec == hdr->node_info[depth].split_nrec) { + hbool_t could_split = FALSE; /* Whether the child node could split */ + + if(idx == 0) { /* Left-most child */ + /* Check for left-most child and its neighbor being close to full */ + if((internal->node_ptrs[idx].node_nrec + internal->node_ptrs[idx + 1].node_nrec) + >= ((hdr->node_info[depth - 1].split_nrec * 2) - 1)) + could_split = TRUE; + } /* end if */ + else if(idx == internal->nrec) { /* Right-most child */ + /* Check for right-most child and its neighbor being close to full */ + if((internal->node_ptrs[idx - 1].node_nrec + internal->node_ptrs[idx].node_nrec) + >= ((hdr->node_info[depth - 1].split_nrec * 2) - 1)) + could_split = TRUE; + } /* end else-if */ + else { /* Middle child */ + /* Check for middle child and its left neighbor being close to full */ + if((internal->node_ptrs[idx - 1].node_nrec + internal->node_ptrs[idx].node_nrec) + >= ((hdr->node_info[depth - 1].split_nrec * 2) - 1)) + could_split = TRUE; + /* Check for middle child and its right neighbor being close to full */ + else if((internal->node_ptrs[idx].node_nrec + internal->node_ptrs[idx + 1].node_nrec) + >= ((hdr->node_info[depth - 1].split_nrec * 2) - 1)) + could_split = TRUE; + } /* end if */ + + /* If this node is full and the child node insertion could + * cause a split, punt back up to caller, leaving the + * "insert child full" status. + */ + if(could_split) { + /* Release the internal B-tree node */ + if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr->addr, internal, internal_flags) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release internal B-tree node") + internal = NULL; + + /* Punt back to caller */ + HGOTO_DONE(SUCCEED); + } /* end if */ + } /* end if */ + + /* Release the internal B-tree node */ + if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr->addr, internal, internal_flags) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release internal B-tree node") + internal = NULL; + + /* Indicate that the record was inserted */ + *status = H5B2_UPDATE_INSERT_DONE; + + /* Dodge sideways into inserting a record into this node */ + if(H5B2__insert_internal(hdr, dxpl_id, depth, parent_cache_info_flags_ptr, curr_node_ptr, curr_pos, parent, udata) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, FAIL, "unable to insert record into internal B-tree node") + break; + + case H5B2_UPDATE_UNKNOWN: + default: + HDassert(0 && "Invalid update status"); + HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "invalid update status") + } /* end switch */ + } /* end else */ + +done: + /* Release the internal B-tree node */ + if(internal) { + /* Check if we should shadow this node */ + if(hdr->swmr_write && (internal_flags & H5AC__DIRTIED_FLAG)) { + /* Attempt to shadow the node if doing SWMR writes */ + if(H5B2__shadow_internal(internal, dxpl_id, curr_node_ptr) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTCOPY, FAIL, "unable to shadow internal B-tree node") + + /* Change the state to "shadowed" if only modified currently */ + /* (Triggers parent to be marked dirty) */ + if(*status == H5B2_UPDATE_MODIFY_DONE) + *status = H5B2_UPDATE_SHADOW_DONE; + } /* end if */ + + /* Unprotect node */ + if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr->addr, internal, internal_flags) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release internal B-tree node") + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5B2__update_internal() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__shadow_internal + * + * Purpose: "Shadow" an internal node - copy it to a new location, + * leaving the data in the old location intact (for now). + * This is done when writing in SWMR mode to ensure that + * readers do not see nodes that are out of date with + * respect to each other and thereby inconsistent. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * Apr 27 2012 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5B2__shadow_internal(H5B2_internal_t *internal, hid_t dxpl_id, + H5B2_node_ptr_t *curr_node_ptr) +{ + H5B2_hdr_t *hdr; /* B-tree header */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* + * Check arguments. + */ + HDassert(internal); + HDassert(curr_node_ptr); + HDassert(H5F_addr_defined(curr_node_ptr->addr)); + hdr = internal->hdr; + HDassert(hdr); + HDassert(hdr->swmr_write); + + /* We only need to shadow the node if it has not been shadowed since the + * last time the header was flushed, as otherwise it will be unreachable by + * the readers so there will be no need to shadow. To check if it has been + * shadowed, compare the epoch of this node and the header. If this node's + * epoch is <= to the header's, it hasn't been shadowed yet. */ + if(internal->shadow_epoch <= hdr->shadow_epoch) { + haddr_t new_node_addr; /* Address to move node to */ + + /* + * We must clone the old node so readers with an out-of-date version of + * the parent can still see the correct number of children, via the + * shadowed node. Remove it from cache but do not mark it free on disk. + */ + /* Allocate space for the cloned node */ + if(HADDR_UNDEF == (new_node_addr = H5MF_alloc(hdr->f, H5FD_MEM_BTREE, dxpl_id, (hsize_t)hdr->node_size))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTALLOC, FAIL, "unable to allocate file space to move B-tree node") + + /* Move the location of the node on the disk */ + if(H5AC_move_entry(hdr->f, H5AC_BT2_INT, curr_node_ptr->addr, new_node_addr, dxpl_id) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTMOVE, FAIL, "unable to move B-tree node") + curr_node_ptr->addr = new_node_addr; + + /* Should free the space in the file, but this is not supported by + * SWMR_WRITE code yet - QAK, 2016/12/01 + */ + + /* Set shadow epoch for node ahead of header */ + internal->shadow_epoch = hdr->shadow_epoch + 1; + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B2__shadow_internal() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__remove_internal + * + * Purpose: Removes a record from a B-tree node. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@ncsa.uiuc.edu + * Mar 3 2005 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2__remove_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, hbool_t *depth_decreased, + void *swap_loc, void *swap_parent, uint16_t depth, H5AC_info_t *parent_cache_info, + unsigned *parent_cache_info_flags_ptr, H5B2_nodepos_t curr_pos, + H5B2_node_ptr_t *curr_node_ptr, void *udata, H5B2_remove_t op, void *op_data) +{ + H5AC_info_t *new_cache_info; /* Pointer to new cache info */ + unsigned *new_cache_info_flags_ptr = NULL; + H5B2_node_ptr_t *new_node_ptr; /* Pointer to new node pointer */ + H5B2_internal_t *internal; /* Pointer to internal node */ + H5B2_nodepos_t next_pos = H5B2_POS_MIDDLE; /* Position of next node */ + unsigned internal_flags = H5AC__NO_FLAGS_SET; + haddr_t internal_addr = HADDR_UNDEF; /* Address of internal node */ + size_t merge_nrec; /* Number of records to merge node at */ + hbool_t collapsed_root = FALSE; /* Whether the root was collapsed */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check arguments. */ + HDassert(hdr); + HDassert(depth > 0); + HDassert(parent_cache_info); + HDassert(curr_node_ptr); + HDassert(H5F_addr_defined(curr_node_ptr->addr)); + + /* Lock current B-tree node */ + if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, parent_cache_info, curr_node_ptr, depth, FALSE, H5AC__NO_FLAGS_SET))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") + internal_addr = curr_node_ptr->addr; + + /* Determine the correct number of records to merge at */ + merge_nrec = hdr->node_info[depth - 1].merge_nrec; + + /* Check for needing to collapse the root node */ + /* (The root node is the only internal node allowed to have 1 record) */ + if(internal->nrec == 1 && + ((internal->node_ptrs[0].node_nrec + internal->node_ptrs[1].node_nrec) <= ((merge_nrec * 2) + 1))) { + + /* Merge children of root node */ + if(H5B2__merge2(hdr, dxpl_id, depth, curr_node_ptr, + parent_cache_info_flags_ptr, internal, &internal_flags, 0) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to merge child node") + + /* Let the cache know that the object is deleted */ + internal_flags |= H5AC__DELETED_FLAG; + if(!hdr->swmr_write) + internal_flags |= H5AC__FREE_FILE_SPACE_FLAG; + + /* Reset information in header's root node pointer */ + curr_node_ptr->addr = internal->node_ptrs[0].addr; + curr_node_ptr->node_nrec = internal->node_ptrs[0].node_nrec; + + /* Update flush dependency for child, if using SWMR */ + if(hdr->swmr_write) + if(H5B2__update_flush_depend(hdr, dxpl_id, depth, curr_node_ptr, internal, hdr) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update child node to new parent") + + /* Indicate that the level of the B-tree decreased */ + *depth_decreased = TRUE; + + /* Set pointers for advancing to child node */ + new_cache_info = parent_cache_info; + new_cache_info_flags_ptr = parent_cache_info_flags_ptr; + new_node_ptr = curr_node_ptr; + + /* Set flag to indicate root was collapsed */ + collapsed_root = TRUE; + + /* Indicate position of next node */ + next_pos = H5B2_POS_ROOT; + } /* end if */ + /* Merge or redistribute child node pointers, if necessary */ + else { + unsigned idx = 0; /* Location of record which matches key */ + int cmp = 0; /* Comparison value of records */ + unsigned retries; /* Number of times to attempt redistribution */ + + /* Shadow the node if doing SWMR writes */ + if(hdr->swmr_write) { + if(H5B2__shadow_internal(internal, dxpl_id, curr_node_ptr) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCOPY, FAIL, "unable to shadow internal node") + internal_addr = curr_node_ptr->addr; + } /* end if */ + + /* Locate node pointer for child */ + if(swap_loc) + idx = 0; + else { + if(H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx, &cmp) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") + if(cmp >= 0) + idx++; + } /* end else */ + + /* Set the number of redistribution retries */ + /* This takes care of the case where a B-tree node needs to be + * redistributed, but redistributing the node causes the index + * for removal to move to another node, which also needs to be + * redistributed. Now, we loop trying to redistribute and then + * eventually force a merge */ + retries = 2; + + /* Preemptively merge/redistribute a node we will enter */ + while(internal->node_ptrs[idx].node_nrec == merge_nrec) { + /* Attempt to redistribute records among children */ + /* (NOTE: These 2-node redistributions should actually get the + * record to promote from the node with more records. - QAK) + */ + /* (NOTE: This code is the same in both H5B2__remove_internal() and + * H5B2__remove_internal_by_idx(), fix bugs in both places! - QAK) + */ + if(idx == 0) { /* Left-most child */ + if(retries > 0 && (internal->node_ptrs[idx + 1].node_nrec > merge_nrec)) { + if(H5B2__redistribute2(hdr, dxpl_id, depth, internal, idx) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTREDISTRIBUTE, FAIL, "unable to redistribute child node records") + } /* end if */ + else { + if(H5B2__merge2(hdr, dxpl_id, depth, curr_node_ptr, + parent_cache_info_flags_ptr, internal, &internal_flags, idx) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to merge child node") + } /* end else */ + } /* end if */ + else if(idx == internal->nrec) { /* Right-most child */ + if(retries > 0 && (internal->node_ptrs[idx - 1].node_nrec > merge_nrec)) { + if(H5B2__redistribute2(hdr, dxpl_id, depth, internal, (idx - 1)) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTREDISTRIBUTE, FAIL, "unable to redistribute child node records") + } /* end if */ + else { + if(H5B2__merge2(hdr, dxpl_id, depth, curr_node_ptr, + parent_cache_info_flags_ptr, internal, &internal_flags, (idx - 1)) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to merge child node") + } /* end else */ + } /* end if */ + else { /* Middle child */ + if(retries > 0 && ((internal->node_ptrs[idx + 1].node_nrec > merge_nrec) || + (internal->node_ptrs[idx - 1].node_nrec > merge_nrec))) { + if(H5B2__redistribute3(hdr, dxpl_id, depth, internal, &internal_flags, idx) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTREDISTRIBUTE, FAIL, "unable to redistribute child node records") + } /* end if */ + else { + if(H5B2__merge3(hdr, dxpl_id, depth, curr_node_ptr, + parent_cache_info_flags_ptr, internal, &internal_flags, idx) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to merge child node") + } /* end else */ + } /* end else */ + + /* Locate node pointer for child (after merge/redistribute) */ + if(swap_loc) + idx = 0; + else { +/* Actually, this can be easily updated (for 2-node redistrib.) and shouldn't require re-searching */ + if(H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx, &cmp) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") + if(cmp >= 0) + idx++; + } /* end else */ + + /* Decrement the number of redistribution retries left */ + retries--; + } /* end while */ + + /* Handle deleting a record from an internal node */ + if(!swap_loc && cmp == 0) { + swap_loc = H5B2_INT_NREC(internal, hdr, idx - 1); + swap_parent = internal; + } /* end if */ + + /* Swap record to delete with record from leaf, if we are the last internal node */ + if(swap_loc && depth == 1) + if(H5B2__swap_leaf(hdr, dxpl_id, depth, internal, &internal_flags, idx, swap_loc) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSWAP, FAIL, "Can't swap records in B-tree") + + /* Set pointers for advancing to child node */ + new_cache_info_flags_ptr = &internal_flags; + new_cache_info = &internal->cache_info; + new_node_ptr = &internal->node_ptrs[idx]; + + /* Indicate position of next node */ + if(H5B2_POS_MIDDLE != curr_pos) { + if(idx == 0) { + if(H5B2_POS_LEFT == curr_pos || H5B2_POS_ROOT == curr_pos) + next_pos = H5B2_POS_LEFT; + } /* end if */ + else if(idx == internal->nrec) { + if(H5B2_POS_RIGHT == curr_pos || H5B2_POS_ROOT == curr_pos) + next_pos = H5B2_POS_RIGHT; + } /* end if */ + } /* end if */ + } /* end else */ + + /* Attempt to remove record from child node */ + if(depth > 1) { + if(H5B2__remove_internal(hdr, dxpl_id, depth_decreased, swap_loc, swap_parent, (uint16_t)(depth - 1), + new_cache_info, new_cache_info_flags_ptr, next_pos, new_node_ptr, udata, op, op_data) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTDELETE, FAIL, "unable to remove record from B-tree internal node") + } /* end if */ + else { + if(H5B2__remove_leaf(hdr, dxpl_id, new_node_ptr, next_pos, new_cache_info, udata, op, op_data) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTDELETE, FAIL, "unable to remove record from B-tree leaf node") + } /* end else */ + + /* Update record count for node pointer to current node */ + if(!collapsed_root) + new_node_ptr->all_nrec--; + + /* Mark node as dirty */ + if(!(hdr->swmr_write && collapsed_root)) + internal_flags |= H5AC__DIRTIED_FLAG; + +#ifdef H5B2_DEBUG + H5B2__assert_internal((!collapsed_root ? (curr_node_ptr->all_nrec - 1) : new_node_ptr->all_nrec), hdr, internal); +#endif /* H5B2_DEBUG */ + +done: + /* Release the B-tree internal node */ + if(internal && H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, internal_addr, internal, internal_flags) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release internal B-tree node") + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5B2__remove_internal() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__remove_internal_by_idx + * + * Purpose: Removes a record from a B-tree node, according to the offset + * in the B-tree records + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Nov 14 2006 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2__remove_internal_by_idx(H5B2_hdr_t *hdr, hid_t dxpl_id, + hbool_t *depth_decreased, void *swap_loc, void *swap_parent, uint16_t depth, + H5AC_info_t *parent_cache_info, unsigned *parent_cache_info_flags_ptr, + H5B2_node_ptr_t *curr_node_ptr, H5B2_nodepos_t curr_pos, hsize_t n, + H5B2_remove_t op, void *op_data) +{ + H5AC_info_t *new_cache_info; /* Pointer to new cache info */ + unsigned *new_cache_info_flags_ptr = NULL; + H5B2_node_ptr_t *new_node_ptr; /* Pointer to new node pointer */ + H5B2_internal_t *internal; /* Pointer to internal node */ + H5B2_nodepos_t next_pos = H5B2_POS_MIDDLE; /* Position of next node */ + unsigned internal_flags = H5AC__NO_FLAGS_SET; + haddr_t internal_addr = HADDR_UNDEF; /* Address of internal node */ + size_t merge_nrec; /* Number of records to merge node at */ + hbool_t collapsed_root = FALSE; /* Whether the root was collapsed */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check arguments. */ + HDassert(hdr); + HDassert(depth > 0); + HDassert(parent_cache_info); + HDassert(curr_node_ptr); + HDassert(H5F_addr_defined(curr_node_ptr->addr)); + + /* Lock current B-tree node */ + if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, parent_cache_info, curr_node_ptr, depth, FALSE, H5AC__NO_FLAGS_SET))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") + internal_addr = curr_node_ptr->addr; + HDassert(internal->nrec == curr_node_ptr->node_nrec); + HDassert(depth == hdr->depth || internal->nrec > 1); + + /* Determine the correct number of records to merge at */ + merge_nrec = hdr->node_info[depth - 1].merge_nrec; + + /* Check for needing to collapse the root node */ + /* (The root node is the only internal node allowed to have 1 record) */ + if(internal->nrec == 1 && + ((internal->node_ptrs[0].node_nrec + internal->node_ptrs[1].node_nrec) <= ((merge_nrec * 2) + 1))) { + HDassert(depth == hdr->depth); + + /* Merge children of root node */ + if(H5B2__merge2(hdr, dxpl_id, depth, curr_node_ptr, + parent_cache_info_flags_ptr, internal, &internal_flags, 0) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to merge child node") + + /* Let the cache know that the object is deleted */ + internal_flags |= H5AC__DELETED_FLAG; + if(!hdr->swmr_write) + internal_flags |= H5AC__FREE_FILE_SPACE_FLAG; + + /* Reset information in header's root node pointer */ + curr_node_ptr->addr = internal->node_ptrs[0].addr; + curr_node_ptr->node_nrec = internal->node_ptrs[0].node_nrec; + + /* Update flush dependency for child, if using SWMR */ + if(hdr->swmr_write) + if(H5B2__update_flush_depend(hdr, dxpl_id, depth, curr_node_ptr, internal, hdr) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUPDATE, FAIL, "unable to update child node to new parent") + + /* Indicate that the level of the B-tree decreased */ + *depth_decreased = TRUE; + + /* Set pointers for advancing to child node */ + new_cache_info = parent_cache_info; + new_cache_info_flags_ptr = parent_cache_info_flags_ptr; + new_node_ptr = curr_node_ptr; + + /* Set flag to indicate root was collapsed */ + collapsed_root = TRUE; + + /* Indicate position of next node */ + next_pos = H5B2_POS_ROOT; + } /* end if */ + /* Merge or redistribute child node pointers, if necessary */ + else { + hsize_t orig_n = n; /* Original index looked for */ + unsigned idx; /* Location of record which matches key */ + hbool_t found = FALSE; /* Comparison value of records */ + unsigned retries; /* Number of times to attempt redistribution */ + + /* Shadow the node if doing SWMR writes */ + if(hdr->swmr_write) { + if(H5B2__shadow_internal(internal, dxpl_id, curr_node_ptr) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCOPY, FAIL, "unable to shadow internal node") + internal_addr = curr_node_ptr->addr; + } /* end if */ + + /* Locate node pointer for child */ + if(swap_loc) + idx = 0; + else { + /* Search for record with correct index */ + for(idx = 0; idx < internal->nrec; idx++) { + /* Check which child node contains indexed record */ + if(internal->node_ptrs[idx].all_nrec >= n) { + /* Check if record is in this node */ + if(internal->node_ptrs[idx].all_nrec == n) { + /* Indicate the record was found and that the index + * in child nodes is zero from now on + */ + found = TRUE; + n = 0; + + /* Increment to next record */ + idx++; + } /* end if */ + + /* Break out of loop early */ + break; + } /* end if */ + + /* Decrement index we are looking for to account for the node we + * just advanced past. + */ + n -= (internal->node_ptrs[idx].all_nrec + 1); + } /* end for */ + } /* end else */ + + /* Set the number of redistribution retries */ + /* This takes care of the case where a B-tree node needs to be + * redistributed, but redistributing the node causes the index + * for removal to move to another node, which also needs to be + * redistributed. Now, we loop trying to redistribute and then + * eventually force a merge */ + retries = 2; + + /* Preemptively merge/redistribute a node we will enter */ + while(internal->node_ptrs[idx].node_nrec == merge_nrec) { + /* Attempt to redistribute records among children */ + /* (NOTE: These 2-node redistributions should actually get the + * record to promote from the node with more records. - QAK) + */ + /* (NOTE: This code is the same in both H5B2__remove_internal() and + * H5B2__remove_internal_by_idx(), fix bugs in both places! - QAK) + */ + if(idx == 0) { /* Left-most child */ + if(retries > 0 && (internal->node_ptrs[idx + 1].node_nrec > merge_nrec)) { + if(H5B2__redistribute2(hdr, dxpl_id, depth, internal, idx) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTREDISTRIBUTE, FAIL, "unable to redistribute child node records") + } /* end if */ + else { + if(H5B2__merge2(hdr, dxpl_id, depth, curr_node_ptr, + parent_cache_info_flags_ptr, internal, &internal_flags, idx) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to merge child node") + } /* end else */ + } /* end if */ + else if(idx == internal->nrec) { /* Right-most child */ + if(retries > 0 && (internal->node_ptrs[idx - 1].node_nrec > merge_nrec)) { + if(H5B2__redistribute2(hdr, dxpl_id, depth, internal, (idx - 1)) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTREDISTRIBUTE, FAIL, "unable to redistribute child node records") + } /* end if */ + else { + if(H5B2__merge2(hdr, dxpl_id, depth, curr_node_ptr, + parent_cache_info_flags_ptr, internal, &internal_flags, (idx - 1)) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to merge child node") + } /* end else */ + } /* end if */ + else { /* Middle child */ + if(retries > 0 && ((internal->node_ptrs[idx + 1].node_nrec > merge_nrec) || + (internal->node_ptrs[idx - 1].node_nrec > merge_nrec))) { + if(H5B2__redistribute3(hdr, dxpl_id, depth, internal, &internal_flags, idx) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTREDISTRIBUTE, FAIL, "unable to redistribute child node records") + } /* end if */ + else { + if(H5B2__merge3(hdr, dxpl_id, depth, curr_node_ptr, + parent_cache_info_flags_ptr, internal, &internal_flags, idx) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to merge child node") + } /* end else */ + } /* end else */ + + /* Locate node pointer for child (after merge/redistribute) */ + if(swap_loc) + idx = 0; + else { + /* Count from the orginal index value again */ + n = orig_n; + + /* Reset "found" flag - the record may have shifted during the + * redistribute/merge + */ + found = FALSE; + + /* Search for record with correct index */ + for(idx = 0; idx < internal->nrec; idx++) { + /* Check which child node contains indexed record */ + if(internal->node_ptrs[idx].all_nrec >= n) { + /* Check if record is in this node */ + if(internal->node_ptrs[idx].all_nrec == n) { + /* Indicate the record was found and that the index + * in child nodes is zero from now on + */ + found = TRUE; + n = 0; + + /* Increment to next record */ + idx++; + } /* end if */ + + /* Break out of loop early */ + break; + } /* end if */ + + /* Decrement index we are looking for to account for the node we + * just advanced past. + */ + n -= (internal->node_ptrs[idx].all_nrec + 1); + } /* end for */ + } /* end else */ + + /* Decrement the number of redistribution retries left */ + retries--; + } /* end while */ + + /* Handle deleting a record from an internal node */ + if(!swap_loc && found) { + swap_loc = H5B2_INT_NREC(internal, hdr, idx - 1); + swap_parent = internal; + } /* end if */ + + /* Swap record to delete with record from leaf, if we are the last internal node */ + if(swap_loc && depth == 1) + if(H5B2__swap_leaf(hdr, dxpl_id, depth, internal, &internal_flags, idx, swap_loc) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSWAP, FAIL, "can't swap records in B-tree") + + /* Set pointers for advancing to child node */ + new_cache_info_flags_ptr = &internal_flags; + new_cache_info = &internal->cache_info; + new_node_ptr = &internal->node_ptrs[idx]; + + /* Indicate position of next node */ + if(H5B2_POS_MIDDLE != curr_pos) { + if(idx == 0) { + if(H5B2_POS_LEFT == curr_pos || H5B2_POS_ROOT == curr_pos) + next_pos = H5B2_POS_LEFT; + } /* end if */ + else if(idx == internal->nrec) { + if(H5B2_POS_RIGHT == curr_pos || H5B2_POS_ROOT == curr_pos) + next_pos = H5B2_POS_RIGHT; + } /* end if */ + } /* end if */ + } /* end else */ + + /* Attempt to remove record from child node */ + if(depth > 1) { + if(H5B2__remove_internal_by_idx(hdr, dxpl_id, depth_decreased, swap_loc, swap_parent, (uint16_t)(depth - 1), + new_cache_info, new_cache_info_flags_ptr, new_node_ptr, next_pos, n, op, op_data) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTDELETE, FAIL, "unable to remove record from B-tree internal node") + } /* end if */ + else { + if(H5B2__remove_leaf_by_idx(hdr, dxpl_id, new_node_ptr, next_pos, new_cache_info, (unsigned)n, op, op_data) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTDELETE, FAIL, "unable to remove record from B-tree leaf node") + } /* end else */ + + /* Update record count for node pointer to child node */ + if(!collapsed_root) + new_node_ptr->all_nrec--; + + /* Mark node as dirty */ + if(!(hdr->swmr_write && collapsed_root)) + internal_flags |= H5AC__DIRTIED_FLAG; + +#ifdef H5B2_DEBUG + H5B2__assert_internal((!collapsed_root ? (curr_node_ptr->all_nrec - 1) : new_node_ptr->all_nrec), hdr, internal); +#endif /* H5B2_DEBUG */ + +done: + /* Release the B-tree internal node */ + if(internal && H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, internal_addr, internal, internal_flags) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release internal B-tree node") + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5B2__remove_internal_by_idx() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__internal_free + * + * Purpose: Destroys a B-tree internal node in memory. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@ncsa.uiuc.edu + * Feb 2 2005 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2__internal_free(H5B2_internal_t *internal) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* + * Check arguments. + */ + HDassert(internal); + + /* Release internal node's native key buffer */ + if(internal->int_native) + internal->int_native = (uint8_t *)H5FL_FAC_FREE(internal->hdr->node_info[internal->depth].nat_rec_fac, internal->int_native); + + /* Release internal node's node pointer buffer */ + if(internal->node_ptrs) + internal->node_ptrs = (H5B2_node_ptr_t *)H5FL_FAC_FREE(internal->hdr->node_info[internal->depth].node_ptr_fac, internal->node_ptrs); + + /* Decrement ref. count on B-tree header */ + if(H5B2__hdr_decr(internal->hdr) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTDEC, FAIL, "can't decrement ref. count on B-tree header") + + /* Sanity check */ + HDassert(NULL == internal->top_proxy); + + /* Free B-tree internal node info */ + internal = H5FL_FREE(H5B2_internal_t, internal); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B2__internal_free() */ + +#ifdef H5B2_DEBUG + +/*------------------------------------------------------------------------- + * Function: H5B2__assert_internal + * + * Purpose: Verify than an internal node is mostly sane + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Quincey Koziol + * koziol@ncsa.uiuc.edu + * Feb 19 2005 + * + *------------------------------------------------------------------------- + */ +H5_ATTR_PURE herr_t +H5B2__assert_internal(hsize_t parent_all_nrec, const H5B2_hdr_t *hdr, const H5B2_internal_t *internal) +{ + hsize_t tot_all_nrec; /* Total number of records at or below this node */ + uint16_t u, v; /* Local index variables */ + + /* General sanity checking on node */ + HDassert(internal->nrec <= hdr->node_info->split_nrec); + + /* Sanity checking on node pointers */ + tot_all_nrec = internal->nrec; + for(u = 0; u < internal->nrec + 1; u++) { + tot_all_nrec += internal->node_ptrs[u].all_nrec; + + HDassert(H5F_addr_defined(internal->node_ptrs[u].addr)); + HDassert(internal->node_ptrs[u].addr > 0); + for(v = 0; v < u; v++) + HDassert(internal->node_ptrs[u].addr != internal->node_ptrs[v].addr); + } /* end for */ + + /* Sanity check all_nrec total in parent */ + if(parent_all_nrec > 0) + HDassert(tot_all_nrec == parent_all_nrec); + + return(0); +} /* end H5B2__assert_internal() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__assert_internal2 + * + * Purpose: Verify than internal nodes are mostly sane + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Quincey Koziol + * koziol@ncsa.uiuc.edu + * Feb 19 2005 + * + *------------------------------------------------------------------------- + */ +H5_ATTR_PURE herr_t +H5B2__assert_internal2(hsize_t parent_all_nrec, const H5B2_hdr_t *hdr, const H5B2_internal_t *internal, const H5B2_internal_t *internal2) +{ + hsize_t tot_all_nrec; /* Total number of records at or below this node */ + uint16_t u, v; /* Local index variables */ + + /* General sanity checking on node */ + HDassert(internal->nrec <= hdr->node_info->split_nrec); + + /* Sanity checking on node pointers */ + tot_all_nrec =internal->nrec; + for(u =0; u < internal->nrec + 1; u++) { + tot_all_nrec += internal->node_ptrs[u].all_nrec; + + HDassert(H5F_addr_defined(internal->node_ptrs[u].addr)); + HDassert(internal->node_ptrs[u].addr > 0); + for(v = 0; v < u; v++) + HDassert(internal->node_ptrs[u].addr != internal->node_ptrs[v].addr); + for(v = 0; v < internal2->nrec + 1; v++) + HDassert(internal->node_ptrs[u].addr != internal2->node_ptrs[v].addr); + } /* end for */ + + /* Sanity check all_nrec total in parent */ + if(parent_all_nrec > 0) + HDassert(tot_all_nrec == parent_all_nrec); + + return(0); +} /* end H5B2__assert_internal2() */ +#endif /* H5B2_DEBUG */ + diff --git a/src/H5B2leaf.c b/src/H5B2leaf.c new file mode 100644 index 0000000..4f8b8e6 --- /dev/null +++ b/src/H5B2leaf.c @@ -0,0 +1,1065 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5B2leaf.c + * Dec 01 2016 + * Quincey Koziol <koziol@lbl.gov> + * + * Purpose: Routines for managing v2 B-tree leaf ndoes. + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5B2module.h" /* This source code file is part of the H5B2 module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5B2pkg.h" /* v2 B-trees */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5MFprivate.h" /* File memory management */ +#include "H5MMprivate.h" /* Memory management */ + + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ +static herr_t H5B2__shadow_leaf(H5B2_leaf_t *leaf, hid_t dxpl_id, + H5B2_node_ptr_t *curr_node_ptr); + + +/*********************/ +/* Package Variables */ +/*********************/ + +/* Declare a free list to manage the H5B2_leaf_t struct */ +H5FL_DEFINE(H5B2_leaf_t); + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + + + +/*------------------------------------------------------------------------- + * Function: H5B2__create_leaf + * + * Purpose: Creates empty leaf node of a B-tree and update node pointer + * to point to it. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@ncsa.uiuc.edu + * Feb 2 2005 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2__create_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, void *parent, H5B2_node_ptr_t *node_ptr) +{ + H5B2_leaf_t *leaf = NULL; /* Pointer to new leaf node created */ + hbool_t inserted = FALSE; /* Whether the leaf node was inserted into cache */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check arguments. */ + HDassert(hdr); + HDassert(node_ptr); + + /* Allocate memory for leaf information */ + if(NULL == (leaf = H5FL_CALLOC(H5B2_leaf_t))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree leaf info") + + /* Increment ref. count on B-tree header */ + if(H5B2__hdr_incr(hdr) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTINC, FAIL, "can't increment ref. count on B-tree header") + + /* Share B-tree header information */ + leaf->hdr = hdr; + + /* Allocate space for the native keys in memory */ + if(NULL == (leaf->leaf_native = (uint8_t *)H5FL_FAC_MALLOC(hdr->node_info[0].nat_rec_fac))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree leaf native keys") + HDmemset(leaf->leaf_native, 0, hdr->cls->nrec_size * hdr->node_info[0].max_nrec); + + /* Set parent */ + leaf->parent = parent; + + /* Set shadowed epoch to header's epoch */ + leaf->shadow_epoch = hdr->shadow_epoch; + + /* Allocate space on disk for the leaf */ + if(HADDR_UNDEF == (node_ptr->addr = H5MF_alloc(hdr->f, H5FD_MEM_BTREE, dxpl_id, (hsize_t)hdr->node_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "file allocation failed for B-tree leaf node") + + /* Cache the new B-tree node */ + if(H5AC_insert_entry(hdr->f, dxpl_id, H5AC_BT2_LEAF, node_ptr->addr, leaf, H5AC__NO_FLAGS_SET) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, FAIL, "can't add B-tree leaf to cache") + inserted = TRUE; + + /* Add leaf node as child of 'top' proxy */ + if(hdr->top_proxy) { + if(H5AC_proxy_entry_add_child(hdr->top_proxy, hdr->f, dxpl_id, leaf) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSET, FAIL, "unable to add v2 B-tree node as child of proxy") + leaf->top_proxy = hdr->top_proxy; + } /* end if */ + +done: + if(ret_value < 0) { + if(leaf) { + /* Remove from cache, if inserted */ + if(inserted) + if(H5AC_remove_entry(leaf) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTREMOVE, FAIL, "unable to remove v2 B-tree leaf node from cache") + + /* Release leaf node's disk space */ + if(H5F_addr_defined(node_ptr->addr) && H5MF_xfree(hdr->f, H5FD_MEM_BTREE, dxpl_id, node_ptr->addr, (hsize_t)hdr->node_size) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTFREE, FAIL, "unable to release file space for v2 B-tree leaf node") + + /* Destroy leaf node */ + if(H5B2__leaf_free(leaf) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTFREE, FAIL, "unable to release v2 B-tree leaf node") + } /* end if */ + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5B2__create_leaf() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__protect_leaf + * + * Purpose: "Protect" an leaf node in the metadata cache + * + * Return: Pointer to leaf node on success/NULL on failure + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * May 5 2010 + * + *------------------------------------------------------------------------- + */ +H5B2_leaf_t * +H5B2__protect_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, void *parent, + H5B2_node_ptr_t *node_ptr, hbool_t shadow, unsigned flags) +{ + H5B2_leaf_cache_ud_t udata; /* User-data for callback */ + H5B2_leaf_t *leaf; /* v2 B-tree leaf node */ + H5B2_leaf_t *ret_value = NULL; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check arguments. */ + HDassert(hdr); + HDassert(node_ptr); + HDassert(H5F_addr_defined(node_ptr->addr)); + + /* only H5AC__READ_ONLY_FLAG may appear in flags */ + HDassert((flags & (unsigned)(~H5AC__READ_ONLY_FLAG)) == 0); + + /* Set up user data for callback */ + udata.f = hdr->f; + udata.hdr = hdr; + udata.parent = parent; + udata.nrec = node_ptr->node_nrec; + + /* Protect the leaf node */ + if(NULL == (leaf = (H5B2_leaf_t *)H5AC_protect(hdr->f, dxpl_id, H5AC_BT2_LEAF, node_ptr->addr, &udata, flags))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, NULL, "unable to protect B-tree leaf node") + + /* Create top proxy, if it doesn't exist */ + if(hdr->top_proxy && NULL == leaf->top_proxy) { + /* Add leaf node as child of 'top' proxy */ + if(H5AC_proxy_entry_add_child(hdr->top_proxy, hdr->f, dxpl_id, leaf) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTSET, NULL, "unable to add v2 B-tree leaf node as child of proxy") + leaf->top_proxy = hdr->top_proxy; + } /* end if */ + + /* Shadow the node, if requested */ + if(shadow) + if(H5B2__shadow_leaf(leaf, dxpl_id, node_ptr) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCOPY, NULL, "unable to shadow leaf node") + + /* Set return value */ + ret_value = leaf; + +done: + /* Clean up on error */ + if(!ret_value) { + /* Release the leaf node, if it was protected */ + if(leaf) { + /* Remove from v2 B-tree's proxy, if added */ + if(leaf->top_proxy) { + if(H5AC_proxy_entry_remove_child(leaf->top_proxy, leaf) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNDEPEND, NULL, "unable to destroy flush dependency between leaf node and v2 B-tree 'top' proxy") + leaf->top_proxy = NULL; + } /* end if */ + + /* Unprotect leaf node */ + if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_LEAF, node_ptr->addr, leaf, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, NULL, "unable to unprotect v2 B-tree leaf node, address = %llu", (unsigned long long)node_ptr->addr) + } /* end if */ + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5B2__protect_leaf() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__neighbor_leaf + * + * Purpose: Locate a record relative to the specified information in a + * B-tree leaf node and return that information by filling in + * fields of the + * caller-supplied UDATA pointer depending on the type of leaf node + * requested. The UDATA can point to additional data passed + * to the key comparison function. + * + * The 'OP' routine is called with the record found and the + * OP_DATA pointer, to allow caller to return information about + * the record. + * + * The RANGE indicates whether to search for records less than or + * equal to, or greater than or equal to the information passed + * in with UDATA. + * + * Return: Non-negative on success, negative on failure. + * + * Programmer: Quincey Koziol + * koziol@ncsa.uiuc.edu + * Mar 9 2005 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2__neighbor_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, H5B2_node_ptr_t *curr_node_ptr, + void *neighbor_loc, H5B2_compare_t comp, void *parent, void *udata, H5B2_found_t op, + void *op_data) +{ + H5B2_leaf_t *leaf; /* Pointer to leaf node */ + unsigned idx = 0; /* Location of record which matches key */ + int cmp = 0; /* Comparison value of records */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check arguments. */ + HDassert(hdr); + HDassert(curr_node_ptr); + HDassert(H5F_addr_defined(curr_node_ptr->addr)); + HDassert(op); + + /* Lock current B-tree node */ + if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, parent, curr_node_ptr, FALSE, H5AC__READ_ONLY_FLAG))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") + + /* Locate node pointer for child */ + if(H5B2__locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx, &cmp) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") + if(cmp > 0) + idx++; + else + if(cmp == 0 && comp == H5B2_COMPARE_GREATER) + idx++; + + /* Set the neighbor location, if appropriate */ + if(comp == H5B2_COMPARE_LESS) { + if(idx > 0) + neighbor_loc = H5B2_LEAF_NREC(leaf, hdr, idx - 1); + } /* end if */ + else { + HDassert(comp == H5B2_COMPARE_GREATER); + + if(idx < leaf->nrec) + neighbor_loc = H5B2_LEAF_NREC(leaf, hdr, idx); + } /* end else */ + + /* Make callback if neighbor record has been found */ + if(neighbor_loc) { + /* Make callback for current record */ + if((op)(neighbor_loc, op_data) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, FAIL, "'found' callback failed for B-tree neighbor operation") + } /* end if */ + else + HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, FAIL, "unable to find neighbor record in B-tree") + +done: + /* Release the B-tree leaf node */ + if(leaf && H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_LEAF, curr_node_ptr->addr, leaf, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree leaf node") + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5B2__neighbor_leaf() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__insert_leaf + * + * Purpose: Adds a new record to a B-tree leaf node. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@ncsa.uiuc.edu + * Mar 3 2005 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2__insert_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, H5B2_node_ptr_t *curr_node_ptr, + H5B2_nodepos_t curr_pos, void *parent, void *udata) +{ + H5B2_leaf_t *leaf; /* Pointer to leaf node */ + unsigned leaf_flags = H5AC__NO_FLAGS_SET; /* Flags for unprotecting the leaf node */ + int cmp; /* Comparison value of records */ + unsigned idx = 0; /* Location of record which matches key */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check arguments. */ + HDassert(hdr); + HDassert(curr_node_ptr); + HDassert(H5F_addr_defined(curr_node_ptr->addr)); + + /* Lock current B-tree node */ + if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, parent, curr_node_ptr, FALSE, H5AC__NO_FLAGS_SET))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") + + /* Must have a leaf node with enough space to insert a record now */ + HDassert(curr_node_ptr->node_nrec < hdr->node_info[0].max_nrec); + + /* Sanity check number of records */ + HDassert(curr_node_ptr->all_nrec == curr_node_ptr->node_nrec); + HDassert(leaf->nrec == curr_node_ptr->node_nrec); + + /* Check for inserting into empty leaf */ + if(leaf->nrec == 0) + idx = 0; + else { + /* Find correct location to insert this record */ + if(H5B2__locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx, &cmp) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") + if(cmp == 0) + HGOTO_ERROR(H5E_BTREE, H5E_EXISTS, FAIL, "record is already in B-tree") + if(cmp > 0) + idx++; + + /* Make room for new record */ + if(idx < leaf->nrec) + HDmemmove(H5B2_LEAF_NREC(leaf, hdr, idx + 1), H5B2_LEAF_NREC(leaf, hdr, idx), hdr->cls->nrec_size * (leaf->nrec - idx)); + } /* end else */ + + /* Make callback to store record in native form */ + if((hdr->cls->store)(H5B2_LEAF_NREC(leaf, hdr, idx), udata) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, FAIL, "unable to insert record into leaf node") + + /* Mark the node as dirty */ + leaf_flags |= H5AC__DIRTIED_FLAG; + + /* Update record count for node pointer to current node */ + curr_node_ptr->all_nrec++; + curr_node_ptr->node_nrec++; + + /* Update record count for current node */ + leaf->nrec++; + + /* Check for new record being the min or max for the tree */ + /* (Don't use 'else' for the idx check, to allow for root leaf node) */ + if(H5B2_POS_MIDDLE != curr_pos) { + if(idx == 0) { + if(H5B2_POS_LEFT == curr_pos || H5B2_POS_ROOT == curr_pos) { + if(hdr->min_native_rec == NULL) + if(NULL == (hdr->min_native_rec = H5MM_malloc(hdr->cls->nrec_size))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTALLOC, FAIL, "memory allocation failed for v2 B-tree min record info") + HDmemcpy(hdr->min_native_rec, H5B2_LEAF_NREC(leaf, hdr, idx), hdr->cls->nrec_size); + } /* end if */ + } /* end if */ + if(idx == (unsigned)(leaf->nrec - 1)) { + if(H5B2_POS_RIGHT == curr_pos || H5B2_POS_ROOT == curr_pos) { + if(hdr->max_native_rec == NULL) + if(NULL == (hdr->max_native_rec = H5MM_malloc(hdr->cls->nrec_size))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTALLOC, FAIL, "memory allocation failed for v2 B-tree max record info") + HDmemcpy(hdr->max_native_rec, H5B2_LEAF_NREC(leaf, hdr, idx), hdr->cls->nrec_size); + } /* end if */ + } /* end if */ + } /* end if */ + +done: + /* Release the B-tree leaf node (marked as dirty) */ + if(leaf) { + /* Shadow the node if doing SWMR writes */ + if(hdr->swmr_write && (leaf_flags & H5AC__DIRTIED_FLAG)) + if(H5B2__shadow_leaf(leaf, dxpl_id, curr_node_ptr) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTCOPY, FAIL, "unable to shadow leaf B-tree node") + + /* Unprotect leaf node */ + if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_LEAF, curr_node_ptr->addr, leaf, leaf_flags) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release leaf B-tree node") + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5B2__insert_leaf() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__update_leaf + * + * Purpose: Insert or modify a record in a B-tree leaf node. + * If the record exists already, it is modified as if H5B2_modify + * was called). If it doesn't exist, it is inserted as if + * H5B2_insert was called. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Dec 23 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2__update_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, H5B2_node_ptr_t *curr_node_ptr, + H5B2_update_status_t *status, H5B2_nodepos_t curr_pos, void *parent, + void *udata, H5B2_modify_t op, void *op_data) +{ + H5B2_leaf_t *leaf; /* Pointer to leaf node */ + unsigned leaf_flags = H5AC__NO_FLAGS_SET; /* Flags for unprotecting the leaf node */ + int cmp = -1; /* Comparison value of records */ + unsigned idx = 0; /* Location of record which matches key */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check arguments. */ + HDassert(hdr); + HDassert(curr_node_ptr); + HDassert(H5F_addr_defined(curr_node_ptr->addr)); + + /* Lock current B-tree node */ + if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, parent, curr_node_ptr, FALSE, H5AC__NO_FLAGS_SET))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") + + /* Sanity check number of records */ + HDassert(curr_node_ptr->all_nrec == curr_node_ptr->node_nrec); + HDassert(leaf->nrec == curr_node_ptr->node_nrec); + + /* Check for inserting into empty leaf */ + if(leaf->nrec == 0) + idx = 0; + else { + /* Find correct location to insert this record */ + if(H5B2__locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx, &cmp) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") + + /* Check for inserting a record */ + if(0 != cmp) { + /* Check if the leaf node is full */ + if(curr_node_ptr->node_nrec == hdr->node_info[0].split_nrec) { + /* Indicate that the leaf is full, but we need to insert */ + *status = H5B2_UPDATE_INSERT_CHILD_FULL; + + /* Let calling routine handle insertion */ + HGOTO_DONE(SUCCEED) + } /* end if */ + + /* Adjust index to leave room for record to insert */ + if(cmp > 0) + idx++; + + /* Make room for new record */ + if(idx < leaf->nrec) + HDmemmove(H5B2_LEAF_NREC(leaf, hdr, idx + 1), H5B2_LEAF_NREC(leaf, hdr, idx), hdr->cls->nrec_size * (leaf->nrec - idx)); + } /* end if */ + } /* end else */ + + /* Check for modifying existing record */ + if(0 == cmp) { + hbool_t changed = FALSE; /* Whether the 'modify' callback changed the record */ + + /* Make callback for current record */ + if((op)(H5B2_LEAF_NREC(leaf, hdr, idx), op_data, &changed) < 0) { + /* Make certain that the callback didn't modify the value if it failed */ + HDassert(changed == FALSE); + + HGOTO_ERROR(H5E_BTREE, H5E_CANTMODIFY, FAIL, "'modify' callback failed for B-tree update operation") + } /* end if */ + + /* Mark the node as dirty if it changed */ + leaf_flags |= (changed ? H5AC__DIRTIED_FLAG : 0); + + /* Indicate that the record was modified */ + *status = H5B2_UPDATE_MODIFY_DONE; + } /* end if */ + else { + /* Must have a leaf node with enough space to insert a record now */ + HDassert(curr_node_ptr->node_nrec < hdr->node_info[0].max_nrec); + + /* Make callback to store record in native form */ + if((hdr->cls->store)(H5B2_LEAF_NREC(leaf, hdr, idx), udata) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, FAIL, "unable to insert record into leaf node") + + /* Mark the node as dirty */ + leaf_flags |= H5AC__DIRTIED_FLAG; + + /* Indicate that the record was inserted */ + *status = H5B2_UPDATE_INSERT_DONE; + + /* Update record count for node pointer to current node */ + curr_node_ptr->all_nrec++; + curr_node_ptr->node_nrec++; + + /* Update record count for current node */ + leaf->nrec++; + } /* end else */ + + /* Check for new record being the min or max for the tree */ + /* (Don't use 'else' for the idx check, to allow for root leaf node) */ + if(H5B2_POS_MIDDLE != curr_pos) { + if(idx == 0) { + if(H5B2_POS_LEFT == curr_pos || H5B2_POS_ROOT == curr_pos) { + if(hdr->min_native_rec == NULL) + if(NULL == (hdr->min_native_rec = H5MM_malloc(hdr->cls->nrec_size))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTALLOC, FAIL, "memory allocation failed for v2 B-tree min record info") + HDmemcpy(hdr->min_native_rec, H5B2_LEAF_NREC(leaf, hdr, idx), hdr->cls->nrec_size); + } /* end if */ + } /* end if */ + if(idx == (unsigned)(leaf->nrec - 1)) { + if(H5B2_POS_RIGHT == curr_pos || H5B2_POS_ROOT == curr_pos) { + if(hdr->max_native_rec == NULL) + if(NULL == (hdr->max_native_rec = H5MM_malloc(hdr->cls->nrec_size))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTALLOC, FAIL, "memory allocation failed for v2 B-tree max record info") + HDmemcpy(hdr->max_native_rec, H5B2_LEAF_NREC(leaf, hdr, idx), hdr->cls->nrec_size); + } /* end if */ + } /* end if */ + } /* end if */ + +done: + /* Release the B-tree leaf node */ + if(leaf) { + /* Check if we should shadow this node */ + if(hdr->swmr_write && (leaf_flags & H5AC__DIRTIED_FLAG)) { + /* Attempt to shadow the node if doing SWMR writes */ + if(H5B2__shadow_leaf(leaf, dxpl_id, curr_node_ptr) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTCOPY, FAIL, "unable to shadow leaf B-tree node") + + /* Change the state to "shadowed" if only modified currently */ + /* (Triggers parent to be marked dirty) */ + if(*status == H5B2_UPDATE_MODIFY_DONE) + *status = H5B2_UPDATE_SHADOW_DONE; + } /* end if */ + + /* Unprotect leaf node */ + if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_LEAF, curr_node_ptr->addr, leaf, leaf_flags) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release leaf B-tree node") + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5B2__update_leaf() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__swap_leaf + * + * Purpose: Swap a record in a node with a record in a leaf node + * + * Return: Success: Non-negative + * + * Failure: Negative + * + * Programmer: Quincey Koziol + * koziol@ncsa.uiuc.edu + * Mar 4 2005 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2__swap_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, + H5B2_internal_t *internal, unsigned *internal_flags_ptr, + unsigned idx, void *swap_loc) +{ + const H5AC_class_t *child_class; /* Pointer to child node's class info */ + haddr_t child_addr; /* Address of child node */ + void *child = NULL; /* Pointer to child node */ + uint8_t *child_native; /* Pointer to child's native records */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check arguments. */ + HDassert(hdr); + HDassert(internal); + HDassert(internal_flags_ptr); + HDassert(idx <= internal->nrec); + + /* Check for the kind of B-tree node to swap */ + if(depth > 1) { + H5B2_internal_t *child_internal; /* Pointer to internal node */ + + /* Setup information for unlocking child node */ + child_class = H5AC_BT2_INT; + + /* Lock B-tree child nodes */ + if(NULL == (child_internal = H5B2__protect_internal(hdr, dxpl_id, internal, &internal->node_ptrs[idx], (uint16_t)(depth - 1), FALSE, H5AC__NO_FLAGS_SET))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") + child_addr = internal->node_ptrs[idx].addr; + + /* More setup for accessing child node information */ + child = child_internal; + child_native = child_internal->int_native; + } /* end if */ + else { + H5B2_leaf_t *child_leaf; /* Pointer to leaf node */ + + /* Setup information for unlocking child nodes */ + child_class = H5AC_BT2_LEAF; + + /* Lock B-tree child node */ + if(NULL == (child_leaf = H5B2__protect_leaf(hdr, dxpl_id, internal, &internal->node_ptrs[idx], FALSE, H5AC__NO_FLAGS_SET))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") + child_addr = internal->node_ptrs[idx].addr; + + /* More setup for accessing child node information */ + child = child_leaf; + child_native = child_leaf->leaf_native; + } /* end else */ + + /* Swap records (use disk page as temporary buffer) */ + HDmemcpy(hdr->page, H5B2_NAT_NREC(child_native, hdr, 0), hdr->cls->nrec_size); + HDmemcpy(H5B2_NAT_NREC(child_native, hdr, 0), swap_loc, hdr->cls->nrec_size); + HDmemcpy(swap_loc, hdr->page, hdr->cls->nrec_size); + + /* Mark parent as dirty */ + *internal_flags_ptr |= H5AC__DIRTIED_FLAG; + +#ifdef H5B2_DEBUG + H5B2__assert_internal((hsize_t)0, hdr, internal); + if(depth > 1) + H5B2__assert_internal(internal->node_ptrs[idx].all_nrec, hdr, (H5B2_internal_t *)child); + else + H5B2__assert_leaf(hdr, (H5B2_leaf_t *)child); +#endif /* H5B2_DEBUG */ + +done: + /* Unlock child node */ + if(child && H5AC_unprotect(hdr->f, dxpl_id, child_class, child_addr, child, H5AC__DIRTIED_FLAG) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree child node") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B2__swap_leaf() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__shadow_leaf + * + * Purpose: "Shadow" a leaf node - copy it to a new location, leaving + * the data in the old location intact (for now). This is + * done when writing in SWMR mode to ensure that readers do + * not see nodes that are out of date with respect to each + * other and thereby inconsistent. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * Apr 27 2012 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5B2__shadow_leaf(H5B2_leaf_t *leaf, hid_t dxpl_id, H5B2_node_ptr_t *curr_node_ptr) +{ + H5B2_hdr_t *hdr; /* B-tree header */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* + * Check arguments. + */ + HDassert(leaf); + HDassert(curr_node_ptr); + HDassert(H5F_addr_defined(curr_node_ptr->addr)); + hdr = leaf->hdr; + HDassert(hdr); + HDassert(hdr->swmr_write); + + /* We only need to shadow the node if it has not been shadowed since the + * last time the header was flushed, as otherwise it will be unreachable by + * the readers so there will be no need to shadow. To check if it has been + * shadowed, compare the epoch of this node and the header. If this node's + * epoch is <= to the header's, it hasn't been shadowed yet. */ + if(leaf->shadow_epoch <= hdr->shadow_epoch) { + haddr_t new_node_addr; /* Address to move node to */ + + /* + * We must clone the old node so readers with an out-of-date version of + * the parent can still see the correct number of children, via the + * shadowed node. Remove it from cache but do not mark it free on disk. + */ + /* Allocate space for the cloned node */ + if(HADDR_UNDEF == (new_node_addr = H5MF_alloc(hdr->f, H5FD_MEM_BTREE, dxpl_id, (hsize_t)hdr->node_size))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTALLOC, FAIL, "unable to allocate file space to move B-tree node") + + /* Move the location of the old child on the disk */ + if(H5AC_move_entry(hdr->f, H5AC_BT2_LEAF, curr_node_ptr->addr, new_node_addr, dxpl_id) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTMOVE, FAIL, "unable to move B-tree node") + curr_node_ptr->addr = new_node_addr; + + /* Should free the space in the file, but this is not supported by + * SWMR_WRITE code yet - QAK, 2016/12/01 + */ + + /* Set shadow epoch for node ahead of header */ + leaf->shadow_epoch = hdr->shadow_epoch + 1; + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B2__shadow_leaf() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__remove_leaf + * + * Purpose: Removes a record from a B-tree leaf node. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@ncsa.uiuc.edu + * Mar 3 2005 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2__remove_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, H5B2_node_ptr_t *curr_node_ptr, + H5B2_nodepos_t curr_pos, void *parent, void *udata, H5B2_remove_t op, void *op_data) +{ + H5B2_leaf_t *leaf; /* Pointer to leaf node */ + haddr_t leaf_addr = HADDR_UNDEF; /* Leaf address on disk */ + unsigned leaf_flags = H5AC__NO_FLAGS_SET; /* Flags for unprotecting leaf node */ + unsigned idx = 0; /* Location of record which matches key */ + int cmp; /* Comparison value of records */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check arguments. */ + HDassert(hdr); + HDassert(curr_node_ptr); + HDassert(H5F_addr_defined(curr_node_ptr->addr)); + + /* Lock current B-tree node */ + if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, parent, curr_node_ptr, FALSE, H5AC__NO_FLAGS_SET))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") + leaf_addr = curr_node_ptr->addr; + + /* Sanity check number of records */ + HDassert(curr_node_ptr->all_nrec == curr_node_ptr->node_nrec); + HDassert(leaf->nrec == curr_node_ptr->node_nrec); + + /* Find correct location to remove this record */ + if(H5B2__locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx, &cmp) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records") + if(cmp != 0) + HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, FAIL, "record is not in B-tree") + + /* Check for invalidating the min/max record for the tree */ + if(H5B2_POS_MIDDLE != curr_pos) { + /* (Don't use 'else' for the idx check, to allow for root leaf node) */ + if(idx == 0) { + if(H5B2_POS_LEFT == curr_pos || H5B2_POS_ROOT == curr_pos) { + if(hdr->min_native_rec) + hdr->min_native_rec = H5MM_xfree(hdr->min_native_rec); + } /* end if */ + } /* end if */ + if(idx == (unsigned)(leaf->nrec - 1)) { + if(H5B2_POS_RIGHT == curr_pos || H5B2_POS_ROOT == curr_pos) { + if(hdr->max_native_rec) + hdr->max_native_rec = H5MM_xfree(hdr->max_native_rec); + } /* end if */ + } /* end if */ + } /* end if */ + + /* Make 'remove' callback if there is one */ + if(op) + if((op)(H5B2_LEAF_NREC(leaf, hdr, idx), op_data) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTDELETE, FAIL, "unable to remove record into leaf node") + + /* Update number of records in node */ + leaf->nrec--; + + if(leaf->nrec > 0) { + /* Shadow the node if doing SWMR writes */ + if(hdr->swmr_write) { + if(H5B2__shadow_leaf(leaf, dxpl_id, curr_node_ptr) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCOPY, FAIL, "unable to shadow leaf node") + leaf_addr = curr_node_ptr->addr; + } /* end if */ + + /* Pack record out of leaf */ + if(idx < leaf->nrec) + HDmemmove(H5B2_LEAF_NREC(leaf, hdr, idx), H5B2_LEAF_NREC(leaf, hdr, (idx + 1)), hdr->cls->nrec_size * (leaf->nrec - idx)); + + /* Mark leaf node as dirty also */ + leaf_flags |= H5AC__DIRTIED_FLAG; + } /* end if */ + else { + /* Let the cache know that the object is deleted */ + leaf_flags |= H5AC__DELETED_FLAG; + if(!hdr->swmr_write) + leaf_flags |= H5AC__DIRTIED_FLAG | H5AC__FREE_FILE_SPACE_FLAG; + + /* Reset address of parent node pointer */ + curr_node_ptr->addr = HADDR_UNDEF; + } /* end else */ + + /* Update record count for parent of leaf node */ + curr_node_ptr->node_nrec--; + +done: + /* Release the B-tree leaf node */ + if(leaf && H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_LEAF, leaf_addr, leaf, leaf_flags) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release leaf B-tree node") + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5B2__remove_leaf() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__remove_leaf_by_idx + * + * Purpose: Removes a record from a B-tree leaf node, according to the + * offset in the B-tree records. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Nov 14 2006 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2__remove_leaf_by_idx(H5B2_hdr_t *hdr, hid_t dxpl_id, + H5B2_node_ptr_t *curr_node_ptr, H5B2_nodepos_t curr_pos, void *parent, + unsigned idx, H5B2_remove_t op, void *op_data) +{ + H5B2_leaf_t *leaf; /* Pointer to leaf node */ + haddr_t leaf_addr = HADDR_UNDEF; /* Leaf address on disk */ + unsigned leaf_flags = H5AC__NO_FLAGS_SET; /* Flags for unprotecting leaf node */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check arguments. */ + HDassert(hdr); + HDassert(curr_node_ptr); + HDassert(H5F_addr_defined(curr_node_ptr->addr)); + + /* Lock B-tree leaf node */ + if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, parent, curr_node_ptr, FALSE, H5AC__NO_FLAGS_SET))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") + leaf_addr = curr_node_ptr->addr; + + /* Sanity check number of records */ + HDassert(curr_node_ptr->all_nrec == curr_node_ptr->node_nrec); + HDassert(leaf->nrec == curr_node_ptr->node_nrec); + HDassert(idx < leaf->nrec); + + /* Check for invalidating the min/max record for the tree */ + if(H5B2_POS_MIDDLE != curr_pos) { + /* (Don't use 'else' for the idx check, to allow for root leaf node) */ + if(idx == 0) { + if(H5B2_POS_LEFT == curr_pos || H5B2_POS_ROOT == curr_pos) { + if(hdr->min_native_rec) + hdr->min_native_rec = H5MM_xfree(hdr->min_native_rec); + } /* end if */ + } /* end if */ + if(idx == (unsigned)(leaf->nrec - 1)) { + if(H5B2_POS_RIGHT == curr_pos || H5B2_POS_ROOT == curr_pos) { + if(hdr->max_native_rec) + hdr->max_native_rec = H5MM_xfree(hdr->max_native_rec); + } /* end if */ + } /* end if */ + } /* end if */ + + /* Make 'remove' callback if there is one */ + if(op) + if((op)(H5B2_LEAF_NREC(leaf, hdr, idx), op_data) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTDELETE, FAIL, "unable to remove record into leaf node") + + /* Update number of records in node */ + leaf->nrec--; + + if(leaf->nrec > 0) { + /* Shadow the node if doing SWMR writes */ + if(hdr->swmr_write) { + if(H5B2__shadow_leaf(leaf, dxpl_id, curr_node_ptr) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCOPY, FAIL, "unable to shadow leaf node") + leaf_addr = curr_node_ptr->addr; + } /* end if */ + + /* Pack record out of leaf */ + if(idx < leaf->nrec) + HDmemmove(H5B2_LEAF_NREC(leaf, hdr, idx), H5B2_LEAF_NREC(leaf, hdr, (idx + 1)), hdr->cls->nrec_size * (leaf->nrec - idx)); + + /* Mark leaf node as dirty also */ + leaf_flags |= H5AC__DIRTIED_FLAG; + } /* end if */ + else { + /* Let the cache know that the object is deleted */ + leaf_flags |= H5AC__DELETED_FLAG; + if(!hdr->swmr_write) + leaf_flags |= H5AC__DIRTIED_FLAG | H5AC__FREE_FILE_SPACE_FLAG; + + /* Reset address of parent node pointer */ + curr_node_ptr->addr = HADDR_UNDEF; + } /* end else */ + + /* Update record count for parent of leaf node */ + curr_node_ptr->node_nrec--; + +done: + /* Release the B-tree leaf node */ + if(leaf && H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_LEAF, leaf_addr, leaf, leaf_flags) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release leaf B-tree node") + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5B2__remove_leaf_by_idx() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__leaf_free + * + * Purpose: Destroys a B-tree leaf node in memory. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@ncsa.uiuc.edu + * Feb 2 2005 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2__leaf_free(H5B2_leaf_t *leaf) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* + * Check arguments. + */ + HDassert(leaf); + + /* Release leaf's native key buffer */ + if(leaf->leaf_native) + leaf->leaf_native = (uint8_t *)H5FL_FAC_FREE(leaf->hdr->node_info[0].nat_rec_fac, leaf->leaf_native); + + /* Decrement ref. count on B-tree header */ + if(H5B2__hdr_decr(leaf->hdr) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTDEC, FAIL, "can't decrement ref. count on B-tree header") + + /* Sanity check */ + HDassert(NULL == leaf->top_proxy); + + /* Free B-tree leaf node info */ + leaf = H5FL_FREE(H5B2_leaf_t, leaf); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B2__leaf_free() */ + +#ifdef H5B2_DEBUG + +/*------------------------------------------------------------------------- + * Function: H5B2__assert_leaf + * + * Purpose: Verify than a leaf node is mostly sane + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Quincey Koziol + * koziol@ncsa.uiuc.edu + * Feb 19 2005 + * + *------------------------------------------------------------------------- + */ +H5_ATTR_PURE herr_t +H5B2__assert_leaf(const H5B2_hdr_t *hdr, const H5B2_leaf_t *leaf) +{ + /* General sanity checking on node */ + HDassert(leaf->nrec <= hdr->node_info->split_nrec); + + return(0); +} /* end H5B2__assert_leaf() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__assert_leaf2 + * + * Purpose: Verify than a leaf node is mostly sane + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Quincey Koziol + * koziol@ncsa.uiuc.edu + * Feb 19 2005 + * + *------------------------------------------------------------------------- + */ +H5_ATTR_PURE herr_t +H5B2__assert_leaf2(const H5B2_hdr_t *hdr, const H5B2_leaf_t *leaf, const H5B2_leaf_t H5_ATTR_UNUSED *leaf2) +{ + /* General sanity checking on node */ + HDassert(leaf->nrec <= hdr->node_info->split_nrec); + + return(0); +} /* end H5B2__assert_leaf2() */ +#endif /* H5B2_DEBUG */ + diff --git a/src/H5B2pkg.h b/src/H5B2pkg.h index edac296..7b1ec4d 100644 --- a/src/H5B2pkg.h +++ b/src/H5B2pkg.h @@ -125,6 +125,9 @@ #define H5B2_NUM_INT_REC(h, d) \ (((h)->node_size - (H5B2_INT_PREFIX_SIZE + H5B2_INT_POINTER_SIZE(h, d))) / ((h)->rrec_size + H5B2_INT_POINTER_SIZE(h, d))) +/* Uncomment this macro to enable extra sanity checking */ +/* #define H5B2_DEBUG */ + /****************************/ /* Package Private Typedefs */ @@ -185,6 +188,26 @@ typedef struct H5B2_hdr_t { void *min_native_rec; /* Pointer to minimum native record */ void *max_native_rec; /* Pointer to maximum native record */ + /* SWMR / Flush dependency information (not stored) */ + hbool_t swmr_write; /* Whether we are doing SWMR writes */ + H5AC_proxy_entry_t *top_proxy; /* 'Top' proxy cache entry for all B-tree entries */ + void *parent; /* Pointer to 'top' proxy flush dependency + * parent, if it exists, otherwise NULL. + * If the v2 B-tree is being used to index a + * chunked dataset and the dataset metadata is + * modified by a SWMR writer, this field will + * be set equal to the object header proxy + * that is the flush dependency parent + * of the v2 B-tree header. + * + * The field is used to avoid duplicate setups + * of the flush dependency relationship, and to + * allow the v2 B-tree header to destroy the + * flush dependency on receipt of an eviction + * notification from the metadata cache. + */ + uint64_t shadow_epoch; /* Epoch of header, for making shadow copies */ + /* Client information (not stored) */ const H5B2_class_t *cls; /* Class of B-tree client */ void *cb_ctx; /* Client callback context */ @@ -199,6 +222,11 @@ typedef struct H5B2_leaf_t { H5B2_hdr_t *hdr; /* Pointer to the [pinned] v2 B-tree header */ uint8_t *leaf_native; /* Pointer to native records */ uint16_t nrec; /* Number of records in node */ + + /* SWMR / Flush dependency information (not stored) */ + H5AC_proxy_entry_t *top_proxy; /* 'Top' proxy cache entry for all B-tree entries */ + void *parent; /* Flush dependency parent for leaf */ + uint64_t shadow_epoch; /* Epoch of node, for making shadow copies */ } H5B2_leaf_t; /* B-tree internal node information */ @@ -212,6 +240,11 @@ typedef struct H5B2_internal_t { H5B2_node_ptr_t *node_ptrs; /* Pointer to node pointers */ uint16_t nrec; /* Number of records in node */ uint16_t depth; /* Depth of this node in the B-tree */ + + /* SWMR / Flush dependency information (not stored) */ + H5AC_proxy_entry_t *top_proxy; /* 'Top' proxy cache entry for all B-tree entries */ + void *parent; /* Flush dependency parent for internal node */ + uint64_t shadow_epoch; /* Epoch of node, for making shadow copies */ } H5B2_internal_t; /* v2 B-tree */ @@ -232,37 +265,40 @@ typedef enum H5B2_nodepos_t { typedef enum H5B2_update_status_t { H5B2_UPDATE_UNKNOWN, /* Unknown update status (initial state) */ H5B2_UPDATE_MODIFY_DONE, /* Update successfully modified existing record */ + H5B2_UPDATE_SHADOW_DONE, /* Update modified existing record and modified node was shadowed */ H5B2_UPDATE_INSERT_DONE, /* Update inserted record successfully */ H5B2_UPDATE_INSERT_CHILD_FULL /* Update will insert record, but child is full */ } H5B2_update_status_t; -/* Callback info for loading a free space header into the cache */ +/* Callback info for loading a v2 B-tree header into the cache */ typedef struct H5B2_hdr_cache_ud_t { H5F_t *f; /* File that v2 b-tree header is within */ haddr_t addr; /* Address of B-tree header in the file */ void *ctx_udata; /* User-data for protecting */ } H5B2_hdr_cache_ud_t; -/* Callback info for loading a free space internal node into the cache */ +/* Callback info for loading a v2 B-tree internal node into the cache */ typedef struct H5B2_internal_cache_ud_t { H5F_t *f; /* File that v2 b-tree header is within */ H5B2_hdr_t *hdr; /* v2 B-tree header */ + void *parent; /* Flush dependency parent */ uint16_t nrec; /* Number of records in node to load */ uint16_t depth; /* Depth of node to load */ } H5B2_internal_cache_ud_t; -/* Callback info for loading a free space leaf node into the cache */ +/* Callback info for loading a v2 B-tree leaf node into the cache */ typedef struct H5B2_leaf_cache_ud_t { H5F_t *f; /* File that v2 b-tree header is within */ H5B2_hdr_t *hdr; /* v2 B-tree header */ + void *parent; /* Flush dependency parent */ uint16_t nrec; /* Number of records in node to load */ } H5B2_leaf_cache_ud_t; #ifdef H5B2_TESTING /* Node information for testing */ typedef struct H5B2_node_info_test_t { - unsigned depth; /* Depth of node */ - unsigned nrec; /* Number of records in node */ + uint16_t depth; /* Depth of node */ + uint16_t nrec; /* Number of records in node */ } H5B2_node_info_test_t; #endif /* H5B2_TESTING */ @@ -306,6 +342,30 @@ extern const H5B2_class_t *const H5B2_client_class_g[H5B2_NUM_BTREE_ID]; /* Package Private Prototypes */ /******************************/ +/* Generic routines */ +H5_DLL herr_t H5B2__create_flush_depend(H5AC_info_t *parent_entry, + H5AC_info_t *child_entry); +H5_DLL herr_t H5B2__update_flush_depend(H5B2_hdr_t *hdr, hid_t dxpl_id, + unsigned depth, const H5B2_node_ptr_t *node_ptr, void *old_parent, + void *new_parent); +H5_DLL herr_t H5B2__destroy_flush_depend(H5AC_info_t *parent_entry, + H5AC_info_t *child_entry); + +/* Internal node management routines */ +H5_DLL herr_t H5B2__split1(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, + H5B2_node_ptr_t *curr_node_ptr, unsigned *parent_cache_info_flags_ptr, + H5B2_internal_t *internal, unsigned *internal_flags_ptr, unsigned idx); +H5_DLL herr_t H5B2__redistribute2(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, + H5B2_internal_t *internal, unsigned idx); +H5_DLL herr_t H5B2__redistribute3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, + H5B2_internal_t *internal, unsigned *internal_flags_ptr, unsigned idx); +H5_DLL herr_t H5B2__merge2(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, + H5B2_node_ptr_t *curr_node_ptr, unsigned *parent_cache_info_flags_ptr, + H5B2_internal_t *internal, unsigned *internal_flags_ptr, unsigned idx); +H5_DLL herr_t H5B2__merge3(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, + H5B2_node_ptr_t *curr_node_ptr, unsigned *parent_cache_info_flags_ptr, + H5B2_internal_t *internal, unsigned *internal_flags_ptr, unsigned idx); + /* Routines for managing B-tree header info */ H5_DLL H5B2_hdr_t *H5B2__hdr_alloc(H5F_t *f); H5_DLL haddr_t H5B2__hdr_create(H5F_t *f, hid_t dxpl_id, @@ -324,17 +384,23 @@ H5_DLL herr_t H5B2__hdr_unprotect(H5B2_hdr_t *hdr, hid_t dxpl_id, H5_DLL herr_t H5B2__hdr_delete(H5B2_hdr_t *hdr, hid_t dxpl_id); /* Routines for operating on leaf nodes */ -H5B2_leaf_t *H5B2__protect_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, haddr_t addr, - uint16_t nrec, unsigned flags); +H5_DLL H5B2_leaf_t * H5B2__protect_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, + void *parent, H5B2_node_ptr_t *node_ptr, hbool_t shadow, unsigned flags); +H5_DLL herr_t H5B2__swap_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, + H5B2_internal_t *internal, unsigned *internal_flags_ptr, unsigned idx, + void *swap_loc); /* Routines for operating on internal nodes */ H5_DLL H5B2_internal_t *H5B2__protect_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, - haddr_t addr, uint16_t nrec, uint16_t depth, unsigned flags); + void *parent, H5B2_node_ptr_t *node_ptr, uint16_t depth, hbool_t shadow, + unsigned flags); /* Routines for allocating nodes */ H5_DLL herr_t H5B2__split_root(H5B2_hdr_t *hdr, hid_t dxpl_id); -H5_DLL herr_t H5B2__create_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, +H5_DLL herr_t H5B2__create_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, void *parent, H5B2_node_ptr_t *node_ptr); +H5_DLL herr_t H5B2__create_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, void *parent, + H5B2_node_ptr_t *node_ptr, uint16_t depth); /* Routines for releasing structures */ H5_DLL herr_t H5B2__hdr_free(H5B2_hdr_t *hdr); @@ -342,59 +408,64 @@ H5_DLL herr_t H5B2__leaf_free(H5B2_leaf_t *l); H5_DLL herr_t H5B2__internal_free(H5B2_internal_t *i); /* Routines for inserting records */ -H5_DLL herr_t H5B2__insert_hdr(H5B2_hdr_t *hdr, hid_t dxpl_id, void *udata); +H5_DLL herr_t H5B2__insert(H5B2_hdr_t *hdr, hid_t dxpl_id, void *udata); H5_DLL herr_t H5B2__insert_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, unsigned *parent_cache_info_flags_ptr, - H5B2_node_ptr_t *curr_node_ptr, H5B2_nodepos_t curr_pos, void *udata); + H5B2_node_ptr_t *curr_node_ptr, H5B2_nodepos_t curr_pos, void *parent, void *udata); H5_DLL herr_t H5B2__insert_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, - H5B2_node_ptr_t *curr_node_ptr, H5B2_nodepos_t curr_pos, void *udata); + H5B2_node_ptr_t *curr_node_ptr, H5B2_nodepos_t curr_pos, void *parent, void *udata); /* Routines for update records */ H5_DLL herr_t H5B2__update_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, unsigned *parent_cache_info_flags_ptr, H5B2_node_ptr_t *curr_node_ptr, H5B2_update_status_t *status, - H5B2_nodepos_t curr_pos, void *udata, H5B2_modify_t op, void *op_data); + H5B2_nodepos_t curr_pos, void *parent, void *udata, H5B2_modify_t op, + void *op_data); H5_DLL herr_t H5B2__update_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, H5B2_node_ptr_t *curr_node_ptr, H5B2_update_status_t *status, - H5B2_nodepos_t curr_pos, void *udata, H5B2_modify_t op, void *op_data); + H5B2_nodepos_t curr_pos, void *parent, void *udata, H5B2_modify_t op, + void *op_data); /* Routines for iterating over nodes/records */ H5_DLL herr_t H5B2__iterate_node(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, - const H5B2_node_ptr_t *curr_node, H5B2_operator_t op, void *op_data); + const H5B2_node_ptr_t *curr_node, void *parent, H5B2_operator_t op, void *op_data); H5_DLL herr_t H5B2__node_size(H5B2_hdr_t *hdr, hid_t dxpl_id, - uint16_t depth, const H5B2_node_ptr_t *curr_node, hsize_t *op_data); + uint16_t depth, const H5B2_node_ptr_t *curr_node, void *parent, + hsize_t *op_data); /* Routines for locating records */ H5_DLL herr_t H5B2__locate_record(const H5B2_class_t *type, unsigned nrec, size_t *rec_off, const uint8_t *native, const void *udata, unsigned *idx, int *result); H5_DLL herr_t H5B2__neighbor_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, H5B2_node_ptr_t *curr_node_ptr, void *neighbor_loc, - H5B2_compare_t comp, void *udata, H5B2_found_t op, void *op_data); + H5B2_compare_t comp, void *parent, void *udata, H5B2_found_t op, + void *op_data); H5_DLL herr_t H5B2__neighbor_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, - H5B2_node_ptr_t *curr_node_ptr, void *neighbor_loc, - H5B2_compare_t comp, void *udata, H5B2_found_t op, void *op_data); + H5B2_node_ptr_t *curr_node_ptr, void *neighbor_loc, H5B2_compare_t comp, + void *parent, void *udata, H5B2_found_t op, void *op_data); /* Routines for removing records */ H5_DLL herr_t H5B2__remove_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, - hbool_t *depth_decreased, void *swap_loc, uint16_t depth, + hbool_t *depth_decreased, void *swap_loc, void *swap_parent, uint16_t depth, H5AC_info_t *parent_cache_info, unsigned *parent_cache_info_flags_ptr, H5B2_nodepos_t curr_pos, H5B2_node_ptr_t *curr_node_ptr, void *udata, H5B2_remove_t op, void *op_data); H5_DLL herr_t H5B2__remove_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, - H5B2_node_ptr_t *curr_node_ptr, H5B2_nodepos_t curr_pos, + H5B2_node_ptr_t *curr_node_ptr, H5B2_nodepos_t curr_pos, void *parent, void *udata, H5B2_remove_t op, void *op_data); H5_DLL herr_t H5B2__remove_internal_by_idx(H5B2_hdr_t *hdr, hid_t dxpl_id, - hbool_t *depth_decreased, void *swap_loc, uint16_t depth, + hbool_t *depth_decreased, void *swap_loc, void *swap_parent, uint16_t depth, H5AC_info_t *parent_cache_info, unsigned *parent_cache_info_flags_ptr, H5B2_node_ptr_t *curr_node_ptr, H5B2_nodepos_t curr_pos, hsize_t n, H5B2_remove_t op, void *op_data); H5_DLL herr_t H5B2__remove_leaf_by_idx(H5B2_hdr_t *hdr, hid_t dxpl_id, - H5B2_node_ptr_t *curr_node_ptr, H5B2_nodepos_t curr_pos, + H5B2_node_ptr_t *curr_node_ptr, H5B2_nodepos_t curr_pos, void *parent, unsigned idx, H5B2_remove_t op, void *op_data); /* Routines for deleting nodes */ H5_DLL herr_t H5B2__delete_node(H5B2_hdr_t *hdr, hid_t dxpl_id, uint16_t depth, - const H5B2_node_ptr_t *curr_node, H5B2_remove_t op, void *op_data); + const H5B2_node_ptr_t *curr_node, void *parent, H5B2_remove_t op, + void *op_data); /* Debugging routines for dumping file structures */ H5_DLL herr_t H5B2__hdr_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, @@ -406,6 +477,15 @@ H5_DLL herr_t H5B2__leaf_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE *stream, int indent, int fwidth, const H5B2_class_t *type, haddr_t hdr_addr, unsigned nrec, haddr_t obj_addr); +/* Sanity checking routines */ +#ifdef H5B2_DEBUG +/* Don't label these with H5_ATTR_PURE or you'll get even more warnings... */ +H5_DLL herr_t H5B2__assert_internal(hsize_t parent_all_nrec, const H5B2_hdr_t *hdr, const H5B2_internal_t *internal); +H5_DLL herr_t H5B2__assert_internal2(hsize_t parent_all_nrec, const H5B2_hdr_t *hdr, const H5B2_internal_t *internal, const H5B2_internal_t *internal2); +H5_DLL herr_t H5B2__assert_leaf(const H5B2_hdr_t *hdr, const H5B2_leaf_t *leaf); +H5_DLL herr_t H5B2__assert_leaf2(const H5B2_hdr_t *hdr, const H5B2_leaf_t *leaf, const H5B2_leaf_t *leaf2); +#endif /* H5B2_DEBUG */ + /* Testing routines */ #ifdef H5B2_TESTING H5_DLL herr_t H5B2_get_root_addr_test(H5B2_t *bt2, haddr_t *root_addr); diff --git a/src/H5B2private.h b/src/H5B2private.h index 3caf41f..161e25e 100644 --- a/src/H5B2private.h +++ b/src/H5B2private.h @@ -31,7 +31,8 @@ #include "H5B2public.h" /* Private headers needed by this file */ -#include "H5Fprivate.h" /* File access */ +#include "H5ACprivate.h" /* Metadata cache */ +#include "H5Fprivate.h" /* File access */ /**************************/ /* Library Private Macros */ @@ -153,6 +154,7 @@ H5_DLL herr_t H5B2_size(H5B2_t *bt2, hid_t dxpl_id, H5_DLL herr_t H5B2_close(H5B2_t *bt2, hid_t dxpl_id); H5_DLL herr_t H5B2_delete(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *ctx_udata, H5B2_remove_t op, void *op_data); +H5_DLL herr_t H5B2_depend(H5B2_t *bt2, hid_t dxpl_id, H5AC_proxy_entry_t *parent); H5_DLL herr_t H5B2_patch_file(H5B2_t *fa, H5F_t *f); /* Statistics routines */ diff --git a/src/H5B2stat.c b/src/H5B2stat.c index 10c692e..da721c6 100644 --- a/src/H5B2stat.c +++ b/src/H5B2stat.c @@ -139,7 +139,7 @@ H5B2_size(H5B2_t *bt2, hid_t dxpl_id, hsize_t *btree_size) *btree_size += hdr->node_size; else /* Iterate through nodes */ - if(H5B2__node_size(hdr, dxpl_id, hdr->depth, &hdr->root, btree_size) < 0) + if(H5B2__node_size(hdr, dxpl_id, hdr->depth, &hdr->root, hdr, btree_size) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTLIST, FAIL, "node iteration failed") } /* end if */ diff --git a/src/H5B2test.c b/src/H5B2test.c index 24163e0..aec2aba 100644 --- a/src/H5B2test.c +++ b/src/H5B2test.c @@ -518,6 +518,7 @@ H5B2_get_node_info_test(H5B2_t *bt2, hid_t dxpl_id, void *udata, { H5B2_hdr_t *hdr; /* Pointer to the B-tree header */ H5B2_node_ptr_t curr_node_ptr; /* Node pointer info for current node */ + void *parent = NULL; /* Parent of current node */ uint16_t depth; /* Current depth of the tree */ int cmp; /* Comparison value of records */ unsigned idx; /* Location of record which matches key */ @@ -537,6 +538,10 @@ H5B2_get_node_info_test(H5B2_t *bt2, hid_t dxpl_id, void *udata, /* Make copy of the root node pointer to start search with */ curr_node_ptr = hdr->root; + /* Set initial parent, if doing swmr writes */ + if(hdr->swmr_write) + parent = hdr; + /* Current depth of the tree */ depth = hdr->depth; @@ -551,9 +556,16 @@ H5B2_get_node_info_test(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_node_ptr_t next_node_ptr; /* Node pointer info for next node */ /* Lock B-tree current node */ - if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, curr_node_ptr.addr, curr_node_ptr.node_nrec, depth, H5AC__READ_ONLY_FLAG))) + if(NULL == (internal = H5B2__protect_internal(hdr, dxpl_id, parent, &curr_node_ptr, depth, FALSE, H5AC__READ_ONLY_FLAG))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to load B-tree internal node") + /* Unpin parent if necessary */ + if(parent) { + if(parent != hdr && H5AC_unpin_entry(parent) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPIN, FAIL, "unable to unpin parent entry") + parent = NULL; + } /* end if */ + /* Locate node pointer for child */ if(H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx, &cmp) < 0) @@ -567,9 +579,13 @@ H5B2_get_node_info_test(H5B2_t *bt2, hid_t dxpl_id, void *udata, next_node_ptr = internal->node_ptrs[idx]; /* Unlock current node */ - if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr.addr, internal, H5AC__NO_FLAGS_SET) < 0) + if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr.addr, internal, (unsigned)(hdr->swmr_write ? H5AC__PIN_ENTRY_FLAG : H5AC__NO_FLAGS_SET)) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree node") + /* Keep track of parent if necessary */ + if(hdr->swmr_write) + parent = internal; + /* Set pointer to next node to load */ curr_node_ptr = next_node_ptr; } /* end if */ @@ -594,9 +610,16 @@ H5B2_get_node_info_test(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_leaf_t *leaf; /* Pointer to leaf node in B-tree */ /* Lock B-tree leaf node */ - if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, curr_node_ptr.addr, curr_node_ptr.node_nrec, H5AC__READ_ONLY_FLAG))) + if(NULL == (leaf = H5B2__protect_leaf(hdr, dxpl_id, parent, &curr_node_ptr, FALSE, H5AC__READ_ONLY_FLAG))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node") + /* Unpin parent if necessary */ + if(parent) { + if(parent != hdr && H5AC_unpin_entry(parent) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPIN, FAIL, "unable to unpin parent entry") + parent = NULL; + } /* end if */ + /* Locate record */ if(H5B2__locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx, &cmp) < 0) @@ -616,6 +639,12 @@ H5B2_get_node_info_test(H5B2_t *bt2, hid_t dxpl_id, void *udata, ninfo->nrec = curr_node_ptr.node_nrec; done: + if(parent) { + HDassert(ret_value < 0); + if(parent != hdr && H5AC_unpin_entry(parent) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPIN, FAIL, "unable to unpin parent entry") + } /* end if */ + FUNC_LEAVE_NOAPI(ret_value) } /* H5B2_get_node_info_test() */ diff --git a/src/H5Bcache.c b/src/H5Bcache.c index 8354e8e..b2be829 100644 --- a/src/H5Bcache.c +++ b/src/H5Bcache.c @@ -54,14 +54,13 @@ /********************/ /* Metadata cache callbacks */ -static herr_t H5B__get_load_size(const void *udata, size_t *image_len); -static void *H5B__deserialize(const void *image, size_t len, void *udata, +static herr_t H5B__cache_get_initial_load_size(void *udata, size_t *image_len); +static void *H5B__cache_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5B__image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); -static herr_t H5B__serialize(const H5F_t *f, void *image, size_t len, +static herr_t H5B__cache_image_len(const void *thing, size_t *image_len); +static herr_t H5B__cache_serialize(const H5F_t *f, void *image, size_t len, void *thing); -static herr_t H5B__free_icr(void *thing); +static herr_t H5B__cache_free_icr(void *thing); /*********************/ @@ -74,14 +73,15 @@ const H5AC_class_t H5AC_BT[1] = {{ "v1 B-tree", /* Metadata client name (for debugging) */ H5FD_MEM_BTREE, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5B__get_load_size, /* 'get_load_size' callback */ - H5B__deserialize, /* 'deserialize' callback */ - H5B__image_len, /* 'image_len' callback */ + H5B__cache_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + NULL, /* 'verify_chksum' callback */ + H5B__cache_deserialize, /* 'deserialize' callback */ + H5B__cache_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ - H5B__serialize, /* 'serialize' callback */ + H5B__cache_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ - H5B__free_icr, /* 'free_icr' callback */ - NULL, /* 'clear" callback */ + H5B__cache_free_icr, /* 'free_icr' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -92,7 +92,7 @@ const H5AC_class_t H5AC_BT[1] = {{ /*------------------------------------------------------------------------- - * Function: H5B__get_load_size + * Function: H5B__cache_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -105,9 +105,9 @@ const H5AC_class_t H5AC_BT[1] = {{ *------------------------------------------------------------------------- */ static herr_t -H5B__get_load_size(const void *_udata, size_t *image_len) +H5B__cache_get_initial_load_size(void *_udata, size_t *image_len) { - const H5B_cache_ud_t *udata = (const H5B_cache_ud_t *)_udata; /* User data for callback */ + H5B_cache_ud_t *udata = (H5B_cache_ud_t *)_udata; /* User data for callback */ H5B_shared_t *shared; /* Pointer to shared B-tree info */ FUNC_ENTER_STATIC_NOERR @@ -124,11 +124,11 @@ H5B__get_load_size(const void *_udata, size_t *image_len) *image_len = shared->sizeof_rnode; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5B__get_load_size() */ +} /* end H5B__cache_get_initial_load_size() */ /*------------------------------------------------------------------------- - * Function: H5B__deserialize + * Function: H5B__cache_deserialize * * Purpose: Deserialize the data structure from disk. * @@ -142,7 +142,7 @@ H5B__get_load_size(const void *_udata, size_t *image_len) *------------------------------------------------------------------------- */ static void * -H5B__deserialize(const void *_image, size_t H5_ATTR_UNUSED len, void *_udata, +H5B__cache_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, void *_udata, hbool_t H5_ATTR_UNUSED *dirty) { H5B_t *bt = NULL; /* Pointer to the deserialized B-tree node */ @@ -231,11 +231,11 @@ done: HDONE_ERROR(H5E_BTREE, H5E_CANTFREE, NULL, "unable to destroy B-tree node") FUNC_LEAVE_NOAPI(ret_value) -} /* end H5B__deserialize() */ +} /* end H5B__cache_deserialize() */ /*------------------------------------------------------------------------- - * Function: H5B__image_len + * Function: H5B__cache_image_len * * Purpose: Compute the size of the data structure on disk. * @@ -248,8 +248,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B__image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5B__cache_image_len(const void *_thing, size_t *image_len) { const H5B_t *bt = (const H5B_t *)_thing; /* Pointer to the B-tree node */ H5B_shared_t *shared; /* Pointer to shared B-tree info */ @@ -268,11 +267,11 @@ H5B__image_len(const void *_thing, size_t *image_len, *image_len = shared->sizeof_rnode; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5B__image_len() */ +} /* end H5B__cache_image_len() */ /*------------------------------------------------------------------------- - * Function: H5B__serialize + * Function: H5B__cache_serialize * * Purpose: Serialize the data structure for writing to disk. * @@ -285,7 +284,7 @@ H5B__image_len(const void *_thing, size_t *image_len, *------------------------------------------------------------------------- */ static herr_t -H5B__serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED len, +H5B__cache_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED len, void *_thing) { H5B_t *bt = (H5B_t *)_thing; /* Pointer to the B-tree node */ @@ -354,11 +353,11 @@ H5B__serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED len, done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5B__serialize() */ +} /* end H5B__cache_serialize() */ /*------------------------------------------------------------------------- - * Function: H5B__free_icr + * Function: H5B__cache_free_icr * * Purpose: Destroy/release an "in core representation" of a data structure * @@ -371,7 +370,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B__free_icr(void *thing) +H5B__cache_free_icr(void *thing) { herr_t ret_value = SUCCEED; /* Return value */ @@ -386,5 +385,5 @@ H5B__free_icr(void *thing) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5B__free_icr() */ +} /* end H5B__cache_free_icr() */ diff --git a/src/H5Bdbg.c b/src/H5Bdbg.c index d92a24b..b22264d 100644 --- a/src/H5Bdbg.c +++ b/src/H5Bdbg.c @@ -77,6 +77,9 @@ H5B_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE *stream, int indent, int f HDassert(fwidth >= 0); HDassert(type); + /* Currently does not support SWMR access */ + HDassert(!(H5F_INTENT(f) & H5F_ACC_SWMR_WRITE)); + /* Get shared info for B-tree */ if(NULL == (rc_shared = (type->get_shared)(f, udata))) HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree's shared ref. count object") @@ -82,9 +82,6 @@ /* Headers */ /***********/ #include "H5private.h" /* Generic Functions */ -#ifdef H5_HAVE_PARALLEL -#include "H5ACprivate.h" /* Metadata cache */ -#endif /* H5_HAVE_PARALLEL */ #include "H5Cpkg.h" /* Cache */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* Files */ @@ -170,8 +167,11 @@ static herr_t H5C__mark_flush_dep_dirty(H5C_cache_entry_t * entry); static herr_t H5C__mark_flush_dep_clean(H5C_cache_entry_t * entry); +static herr_t H5C__verify_len_eoa(H5F_t *f, const H5C_class_t * type, + haddr_t addr, size_t *len, hbool_t actual); + static herr_t H5C__generate_image(const H5F_t *f, H5C_t * cache_ptr, H5C_cache_entry_t *entry_ptr, - hid_t dxpl_id, int64_t *entry_size_change_ptr); + hid_t dxpl_id); #if H5C_DO_SLIST_SANITY_CHECKS static hbool_t H5C_entry_in_skip_list(H5C_t * cache_ptr, @@ -189,11 +189,6 @@ static void H5C__assert_flush_dep_nocycle(const H5C_cache_entry_t * entry, const H5C_cache_entry_t * base_entry); #endif /* NDEBUG */ -#if 0 /* debugging routines */ -herr_t H5C_dump_cache(H5C_t * cache_ptr, const char * cache_name); -herr_t H5C_dump_cache_skip_list(H5C_t * cache_ptr, char * calling_fcn); -#endif /* debugging routines */ - /*********************/ /* Package Variables */ @@ -202,6 +197,9 @@ herr_t H5C_dump_cache_skip_list(H5C_t * cache_ptr, char * calling_fcn); /* Package initialization variable */ hbool_t H5_PKG_INIT_VAR = FALSE; +/* Declare a free list to manage the tag info struct */ +H5FL_DEFINE(H5C_tag_info_t); + /*****************************/ /* Library Private Variables */ @@ -218,12 +216,6 @@ H5FL_DEFINE_STATIC(H5C_t); /* Declare a free list to manage flush dependency arrays */ H5FL_BLK_DEFINE_STATIC(parent); -/* Declare extern free list to manage the H5C_collective_write_t struct */ -H5FL_EXTERN(H5C_collective_write_t); - -/* Declare a free list to manage corked object addresses */ -H5FL_DEFINE_STATIC(haddr_t); - /*------------------------------------------------------------------------- @@ -275,26 +267,18 @@ H5C_create(size_t max_cache_size, HDassert( type_name_table_ptr ); for ( i = 0; i <= max_type_id; i++ ) { - HDassert( (type_name_table_ptr)[i] ); HDassert( HDstrlen(( type_name_table_ptr)[i]) > 0 ); } - if ( NULL == (cache_ptr = H5FL_CALLOC(H5C_t)) ) { - - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, \ - "memory allocation failed") - } - - if ( (cache_ptr->slist_ptr = H5SL_create(H5SL_TYPE_HADDR, NULL)) == NULL ) { + if(NULL == (cache_ptr = H5FL_CALLOC(H5C_t))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + if(NULL == (cache_ptr->slist_ptr = H5SL_create(H5SL_TYPE_HADDR, NULL))) HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, NULL, "can't create skip list.") - } - - if ( (cache_ptr->cork_list_ptr = H5SL_create(H5SL_TYPE_HADDR, NULL)) == NULL ) { - HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, NULL, "can't create skip list for corked object addresses.") - } + if(NULL == (cache_ptr->tag_list = H5SL_create(H5SL_TYPE_HADDR, NULL))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, NULL, "can't create skip list for tagged entry addresses.") /* If we get this far, we should succeed. Go ahead and initialize all * the fields. @@ -304,6 +288,12 @@ H5C_create(size_t max_cache_size, cache_ptr->flush_in_progress = FALSE; + cache_ptr->logging_enabled = FALSE; + + cache_ptr->currently_logging = FALSE; + + cache_ptr->log_file_ptr = NULL; + cache_ptr->trace_file_ptr = NULL; cache_ptr->aux_ptr = aux_ptr; @@ -341,8 +331,6 @@ H5C_create(size_t max_cache_size, cache_ptr->ignore_tags = FALSE; cache_ptr->slist_changed = FALSE; - cache_ptr->slist_change_in_pre_serialize = FALSE; - cache_ptr->slist_change_in_serialize = FALSE; cache_ptr->slist_len = 0; cache_ptr->slist_size = (size_t)0; @@ -356,6 +344,7 @@ H5C_create(size_t max_cache_size, cache_ptr->entries_removed_counter = 0; cache_ptr->last_entry_removed_ptr = NULL; + cache_ptr->entry_watched_for_removal = NULL; cache_ptr->pl_len = 0; cache_ptr->pl_size = (size_t)0; @@ -377,6 +366,7 @@ H5C_create(size_t max_cache_size, cache_ptr->coll_list_size = (size_t)0; cache_ptr->coll_head_ptr = NULL; cache_ptr->coll_tail_ptr = NULL; + cache_ptr->coll_write_list = NULL; #endif /* H5_HAVE_PARALLEL */ cache_ptr->cLRU_list_len = 0; @@ -459,26 +449,20 @@ H5C_create(size_t max_cache_size, ret_value = cache_ptr; done: - - if ( ret_value == 0 ) { - - if ( cache_ptr != NULL ) { - - if ( cache_ptr->slist_ptr != NULL ) + if(NULL == ret_value) { + if(cache_ptr != NULL) { + if(cache_ptr->slist_ptr != NULL) H5SL_close(cache_ptr->slist_ptr); - if ( cache_ptr->cork_list_ptr != NULL ) - H5SL_close(cache_ptr->cork_list_ptr); + if(cache_ptr->tag_list != NULL) + H5SL_close(cache_ptr->tag_list); cache_ptr->magic = 0; cache_ptr = H5FL_FREE(H5C_t, cache_ptr); - } /* end if */ - } /* end if */ FUNC_LEAVE_NOAPI(ret_value) - } /* H5C_create() */ @@ -669,31 +653,31 @@ H5C_def_auto_resize_rpt_fcn(H5C_t * cache_ptr, /*------------------------------------------------------------------------- - * Function: H5C_free_cork_list_cb + * Function: H5C_free_tag_list_cb * - * Purpose: Callback function to free the list of object addresses - * on the skip list. + * Purpose: Callback function to free tag nodes from the skip list. * * Return: Non-negative on success/Negative on failure * - * Programmer: Vailin Choi; January 2014 + * Programmer: Vailin Choi + * January 2014 * *------------------------------------------------------------------------- */ static herr_t -H5C_free_cork_list_cb(void *_item, void H5_ATTR_UNUSED *key, void H5_ATTR_UNUSED *op_data) +H5C_free_tag_list_cb(void *_item, void H5_ATTR_UNUSED *key, void H5_ATTR_UNUSED *op_data) { - haddr_t *addr = (haddr_t *)_item; + H5C_tag_info_t *tag_info = (H5C_tag_info_t *)_item; FUNC_ENTER_NOAPI_NOINIT_NOERR - HDassert(addr); + HDassert(tag_info); /* Release the item */ - addr = H5FL_FREE(haddr_t, addr); + tag_info = H5FL_FREE(H5C_tag_info_t, tag_info); FUNC_LEAVE_NOAPI(0) -} /* H5C_free_cork_list_cb() */ +} /* H5C_free_tag_list_cb() */ /*------------------------------------------------------------------------- @@ -741,9 +725,9 @@ H5C_dest(H5F_t * f, hid_t dxpl_id) cache_ptr->slist_ptr = NULL; } /* end if */ - if(cache_ptr->cork_list_ptr != NULL) { - H5SL_destroy(cache_ptr->cork_list_ptr, H5C_free_cork_list_cb, NULL); - cache_ptr->cork_list_ptr = NULL; + if(cache_ptr->tag_list != NULL) { + H5SL_destroy(cache_ptr->tag_list, H5C_free_tag_list_cb, NULL); + cache_ptr->tag_list = NULL; } /* end if */ #ifndef NDEBUG @@ -758,7 +742,39 @@ done: /*------------------------------------------------------------------------- + * Function: H5C_evict * + * Purpose: Evict all except pinned entries in the cache + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * Dec 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_evict(H5F_t * f, hid_t dxpl_id) +{ + H5C_t *cache_ptr = f->shared->cache; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + + /* Flush and invalidate all cache entries except the pinned entries */ + if(H5C_flush_invalidate_cache(f, dxpl_id, H5C__EVICT_ALLOW_LAST_PINS_FLAG) < 0 ) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to evict entries in the cache") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_evict() */ + + +/*------------------------------------------------------------------------- * Function: H5C_expunge_entry * * Purpose: Use this function to tell the cache to expunge an entry @@ -779,10 +795,6 @@ H5C_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, H5C_t * cache_ptr; H5C_cache_entry_t * entry_ptr = NULL; unsigned flush_flags = (H5C__FLUSH_INVALIDATE_FLAG | H5C__FLUSH_CLEAR_ONLY_FLAG); -#if H5C_DO_SANITY_CHECKS - hbool_t entry_was_dirty; - hsize_t entry_size; -#endif /* H5C_DO_SANITY_CHECKS */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -797,8 +809,7 @@ H5C_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, #if H5C_DO_EXTREME_SANITY_CHECKS if(H5C_validate_lru_list(cache_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "LRU extreme sanity check failed on entry.\n"); + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "LRU extreme sanity check failed on entry.\n"); #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ /* Look for entry in cache */ @@ -827,38 +838,19 @@ H5C_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, * This will clear the entry, and then delete it from the cache. */ - /* Pass along 'free file space' flag to cache client. */ + /* Pass along 'free file space' flag */ flush_flags |= (flags & H5C__FREE_FILE_SPACE_FLAG); -#if H5C_DO_SANITY_CHECKS - entry_was_dirty = entry_ptr->is_dirty; - entry_size = entry_ptr->size; -#endif /* H5C_DO_EXTREME_SANITY_CHECKS */ - /* Delete the entry from the skip list on destroy */ flush_flags |= H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG; - if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, flush_flags, NULL, NULL) < 0) + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, flush_flags) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTEXPUNGE, FAIL, "can't flush entry") -#if H5C_DO_SANITY_CHECKS - if ( entry_was_dirty ) - { - /* we have just removed an entry from the skip list. Thus - * we must touch up cache_ptr->slist_len_increase and - * cache_ptr->slist_size_increase to keep from skewing - * the sanity checks on flushes. - */ - cache_ptr->slist_len_increase -= 1; - cache_ptr->slist_size_increase -= (int64_t)(entry_size); - } -#endif /* H5C_DO_SANITY_CHECKS */ - done: #if H5C_DO_EXTREME_SANITY_CHECKS if(H5C_validate_lru_list(cache_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "LRU extreme sanity check failed on exit.\n"); + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "LRU extreme sanity check failed on exit.\n") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ FUNC_LEAVE_NOAPI(ret_value) @@ -1246,6 +1238,7 @@ H5C_insert_entry(H5F_t * f, HDassert( cache_ptr ); HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); HDassert( type ); + HDassert( type->image_len ); HDassert( H5F_addr_defined(addr) ); HDassert( thing ); @@ -1297,14 +1290,6 @@ H5C_insert_entry(H5F_t * f, entry_ptr->image_ptr = NULL; entry_ptr->image_up_to_date = FALSE; - /* Apply tag to newly inserted entry */ - if(H5C__tag_entry(cache_ptr, entry_ptr, dxpl_id) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "Cannot tag metadata entry") - - /* Set the entry's cork status */ - if(H5C_cork(cache_ptr, entry_ptr->tag, H5C__GET_CORKED, &entry_ptr->is_corked) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Cannot retrieve entry's cork status") - entry_ptr->is_protected = FALSE; entry_ptr->is_read_only = FALSE; entry_ptr->ro_ref_count = 0; @@ -1320,27 +1305,10 @@ H5C_insert_entry(H5F_t * f, /* not protected, so can't be dirtied */ entry_ptr->dirtied = FALSE; - /* Retrieve the size of the thing. Set the compressed field to FALSE - * and the compressed_size field to zero first, as they may not be - * initialized by the image_len call. - */ - entry_ptr->compressed = FALSE; - entry_ptr->compressed_size = 0; - if((type->image_len)(thing, &(entry_ptr->size), &(entry_ptr->compressed), - &(entry_ptr->compressed_size)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGETSIZE, FAIL, "Can't get size of thing") + /* Retrieve the size of the thing */ + if((type->image_len)(thing, &(entry_ptr->size)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGETSIZE, FAIL, "can't get size of thing") HDassert(entry_ptr->size > 0 && entry_ptr->size < H5C_MAX_ENTRY_SIZE); - HDassert(((type->flags & H5C__CLASS_COMPRESSED_FLAG) != 0) || - (entry_ptr->compressed == FALSE)); - - /* entry has just been inserted -- thus compressed size cannot have - * been computed yet. Thus if entry_ptr->compressed is TRUE, - * entry_ptr->size must equal entry_ptr->compressed_size. - */ - HDassert((entry_ptr->compressed == FALSE) || - (entry_ptr->size == entry_ptr->compressed_size)); - HDassert((entry_ptr->compressed == TRUE) || - (entry_ptr->compressed_size == 0)); entry_ptr->in_slist = FALSE; @@ -1375,27 +1343,26 @@ H5C_insert_entry(H5F_t * f, entry_ptr->coll_prev = NULL; #endif /* H5_HAVE_PARALLEL */ - H5C__RESET_CACHE_ENTRY_STATS(entry_ptr) + /* Apply tag to newly inserted entry */ + if(H5C__tag_entry(cache_ptr, entry_ptr, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "Cannot tag metadata entry") - if ( ( cache_ptr->flash_size_increase_possible ) && - ( entry_ptr->size > cache_ptr->flash_size_increase_threshold ) ) { + H5C__RESET_CACHE_ENTRY_STATS(entry_ptr) + if(cache_ptr->flash_size_increase_possible && + (entry_ptr->size > cache_ptr->flash_size_increase_threshold)) if(H5C__flash_increase_cache_size(cache_ptr, 0, entry_ptr->size) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTINS, FAIL, "H5C__flash_increase_cache_size failed.") - } if(cache_ptr->index_size >= cache_ptr->max_cache_size) - empty_space = 0; + empty_space = 0; else - empty_space = cache_ptr->max_cache_size - cache_ptr->index_size; + empty_space = cache_ptr->max_cache_size - cache_ptr->index_size; - if ( ( cache_ptr->evictions_enabled ) && - ( ( (cache_ptr->index_size + entry_ptr->size) > - cache_ptr->max_cache_size) + if(cache_ptr->evictions_enabled && + (((cache_ptr->index_size + entry_ptr->size) > cache_ptr->max_cache_size) || - ( ( ( empty_space + cache_ptr->clean_index_size ) < - cache_ptr->min_clean_size ) ) ) ) { - + (((empty_space + cache_ptr->clean_index_size) < cache_ptr->min_clean_size)))) { size_t space_needed; if(empty_space <= entry_ptr->size) @@ -1444,20 +1411,10 @@ H5C_insert_entry(H5F_t * f, H5C__INSERT_IN_INDEX(cache_ptr, entry_ptr, FAIL) - /* New entries are presumed to be dirty, so this if statement is - * unnecessary. Rework it once the rest of the code changes are - * in and tested. -- JRM - */ - if ( entry_ptr->is_dirty ) { - - entry_ptr->flush_marker = set_flush_marker; - H5C__INSERT_ENTRY_IN_SLIST(cache_ptr, entry_ptr, FAIL) - - } else { - - entry_ptr->flush_marker = FALSE; - } - + /* New entries are presumed to be dirty */ + HDassert(entry_ptr->is_dirty); + entry_ptr->flush_marker = set_flush_marker; + H5C__INSERT_ENTRY_IN_SLIST(cache_ptr, entry_ptr, FAIL) H5C__UPDATE_RP_FOR_INSERTION(cache_ptr, entry_ptr, FAIL) #if H5C_DO_EXTREME_SANITY_CHECKS @@ -1576,36 +1533,124 @@ H5C_mark_entry_dirty(void *thing) entry_ptr->dirtied = TRUE; } else if ( entry_ptr->is_pinned ) { - hbool_t was_clean; + hbool_t was_clean; /* Whether the entry was previously clean */ + /* Remember previous dirty status */ was_clean = !entry_ptr->is_dirty; - /* mark the entry as dirty if it isn't already */ + /* Mark the entry as dirty if it isn't already */ entry_ptr->is_dirty = TRUE; entry_ptr->image_up_to_date = FALSE; - /* Propagate the dirty flag up the flush dependency chain if appropriate */ + /* Modify cache data structures */ + if(was_clean) + H5C__UPDATE_INDEX_FOR_ENTRY_DIRTY(cache_ptr, entry_ptr) + if(!entry_ptr->in_slist) + H5C__INSERT_ENTRY_IN_SLIST(cache_ptr, entry_ptr, FAIL) + + /* Update stats for entry being marked dirty */ + H5C__UPDATE_STATS_FOR_DIRTY_PIN(cache_ptr, entry_ptr) + + /* Check for entry changing status and do notifications, etc. */ if(was_clean) { - H5C__UPDATE_INDEX_FOR_ENTRY_DIRTY(cache_ptr, entry_ptr); + /* If the entry's type has a 'notify' callback send a 'entry dirtied' + * notice now that the entry is fully integrated into the cache. + */ + if(entry_ptr->type->notify && + (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_DIRTIED, entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag set") + /* Propagate the dirty flag up the flush dependency chain if appropriate */ if(entry_ptr->flush_dep_nparents > 0) if(H5C__mark_flush_dep_dirty(entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep dirty flag") } /* end if */ + } /* end if */ + else + HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Entry is neither pinned nor protected??") - if(!entry_ptr->in_slist) { - H5C__INSERT_ENTRY_IN_SLIST(cache_ptr, entry_ptr, FAIL) - } +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_mark_entry_dirty() */ - H5C__UPDATE_STATS_FOR_DIRTY_PIN(cache_ptr, entry_ptr) + +/*------------------------------------------------------------------------- + * Function: H5C_mark_entry_clean + * + * Purpose: Mark a pinned entry as clean. The target entry MUST be pinned. + * + * If the entry is not + * already clean, the function places function marks the entry + * clean and removes it from the skip list. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * 7/23/16 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_mark_entry_clean(void *_thing) +{ + H5C_t * cache_ptr; + H5C_cache_entry_t * entry_ptr = (H5C_cache_entry_t *)_thing; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(entry_ptr); + HDassert(H5F_addr_defined(entry_ptr->addr)); + cache_ptr = entry_ptr->cache_ptr; + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + + /* Operate on pinned entry */ + if(entry_ptr->is_protected) + HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKCLEAN, FAIL, "entry is protected") + else if(entry_ptr->is_pinned) { + hbool_t was_dirty; /* Whether the entry was previously dirty */ + /* Remember previous dirty status */ + was_dirty = entry_ptr->is_dirty; + + /* Mark the entry as clean if it isn't already */ + entry_ptr->is_dirty = FALSE; + + /* Also reset the 'flush_marker' flag, since the entry shouldn't be flushed now */ + entry_ptr->flush_marker = FALSE; + + /* Modify cache data structures */ + if(was_dirty) + H5C__UPDATE_INDEX_FOR_ENTRY_CLEAN(cache_ptr, entry_ptr) + if(entry_ptr->in_slist) + H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, FALSE) + + /* Update stats for entry being marked clean */ + H5C__UPDATE_STATS_FOR_CLEAR(cache_ptr, entry_ptr) + + /* Check for entry changing status and do notifications, etc. */ + if(was_dirty) { + /* If the entry's type has a 'notify' callback send a 'entry cleaned' + * notice now that the entry is fully integrated into the cache. + */ + if(entry_ptr->type->notify && + (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_CLEANED, entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag cleared") + + /* Propagate the clean up the flush dependency chain, if appropriate */ + if(entry_ptr->flush_dep_nparents > 0) + if(H5C__mark_flush_dep_clean(entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKCLEAN, FAIL, "Can't propagate flush dep clean") + } /* end if */ } /* end if */ else - HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Entry is neither pinned nor protected??") + HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKCLEAN, FAIL, "Entry is not pinned??") done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5C_mark_entry_dirty() */ +} /* H5C_mark_entry_clean() */ /*------------------------------------------------------------------------- @@ -1630,63 +1675,48 @@ H5C_move_entry(H5C_t * cache_ptr, { H5C_cache_entry_t * entry_ptr = NULL; H5C_cache_entry_t * test_entry_ptr = NULL; -#if H5C_DO_SANITY_CHECKS - hbool_t removed_entry_from_slist = FALSE; -#endif /* H5C_DO_SANITY_CHECKS */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) - HDassert( cache_ptr ); - HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); - HDassert( type ); - HDassert( H5F_addr_defined(old_addr) ); - HDassert( H5F_addr_defined(new_addr) ); - HDassert( H5F_addr_ne(old_addr, new_addr) ); + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(type); + HDassert(H5F_addr_defined(old_addr)); + HDassert(H5F_addr_defined(new_addr)); + HDassert(H5F_addr_ne(old_addr, new_addr)); #if H5C_DO_EXTREME_SANITY_CHECKS - if ( ( H5C_validate_protected_entry_list(cache_ptr) < 0 ) || - ( H5C_validate_pinned_entry_list(cache_ptr) < 0 ) || - ( H5C_validate_lru_list(cache_ptr) < 0 ) ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "an extreme sanity check failed on entry.\n"); - } + if((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || + (H5C_validate_lru_list(cache_ptr) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry.\n") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ H5C__SEARCH_INDEX(cache_ptr, old_addr, entry_ptr, FAIL) - if ( ( entry_ptr == NULL ) || ( entry_ptr->type != type ) ) { - + if(entry_ptr == NULL || entry_ptr->type != type) /* the old item doesn't exist in the cache, so we are done. */ HGOTO_DONE(SUCCEED) - } - HDassert( entry_ptr->addr == old_addr ); - HDassert( entry_ptr->type == type ); - - if ( entry_ptr->is_protected ) { + HDassert(entry_ptr->addr == old_addr); + HDassert(entry_ptr->type == type); - HGOTO_ERROR(H5E_CACHE, H5E_CANTMOVE, FAIL, \ - "Target entry is protected.") - } + /* Check for R/W status, otherwise error */ + /* (Moving a R/O entry would mark it dirty, which shouldn't + * happen. QAK - 2016/12/02) + */ + if(entry_ptr->is_read_only) + HGOTO_ERROR(H5E_CACHE, H5E_CANTMOVE, FAIL, "can't move R/O entry") H5C__SEARCH_INDEX(cache_ptr, new_addr, test_entry_ptr, FAIL) - if ( test_entry_ptr != NULL ) { /* we are hosed */ - - if ( test_entry_ptr->type == type ) { - - HGOTO_ERROR(H5E_CACHE, H5E_CANTMOVE, FAIL, \ - "Target already moved & reinserted???.") - - } else { - - HGOTO_ERROR(H5E_CACHE, H5E_CANTMOVE, FAIL, \ - "New address already in use?.") - - } - } + if(test_entry_ptr != NULL) { /* we are hosed */ + if(test_entry_ptr->type == type) + HGOTO_ERROR(H5E_CACHE, H5E_CANTMOVE, FAIL, "target already moved & reinserted???") + else + HGOTO_ERROR(H5E_CACHE, H5E_CANTMOVE, FAIL, "new address already in use?") + } /* end if */ /* If we get this far we have work to do. Remove *entry_ptr from * the hash table (and skip list if necessary), change its address to the @@ -1703,77 +1733,64 @@ H5C_move_entry(H5C_t * cache_ptr, * change the addr. If the entry is only in the process of being flushed, * don't mark it as dirty either, lest we confuse the flush call back. */ - - if ( ! ( entry_ptr->destroy_in_progress ) ) { - + if(!entry_ptr->destroy_in_progress) { H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr) - if ( entry_ptr->in_slist ) { - - HDassert( cache_ptr->slist_ptr ); - - H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr) - -#if H5C_DO_SANITY_CHECKS - - removed_entry_from_slist = TRUE; + if(entry_ptr->in_slist) { + HDassert(cache_ptr->slist_ptr); -#endif /* H5C_DO_SANITY_CHECKS */ - } - } + H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, FALSE) + } /* end if */ + } /* end if */ entry_ptr->addr = new_addr; if(!entry_ptr->destroy_in_progress) { - hbool_t was_dirty; + hbool_t was_dirty; /* Whether the entry was previously dirty */ + /* Remember previous dirty status */ was_dirty = entry_ptr->is_dirty; + + /* Mark the entry as dirty if it isn't already */ entry_ptr->is_dirty = TRUE; /* This shouldn't be needed, but it keeps the test code happy */ entry_ptr->image_up_to_date = FALSE; - /* Propagate the dirty flag up the flush dependency chain if - * appropriate */ - if(!entry_ptr->flush_in_progress) { - if(!was_dirty && entry_ptr->flush_dep_nparents > 0) - if(H5C__mark_flush_dep_dirty(entry_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep dirty flag") - } /* end if */ - + /* Modify cache data structures */ H5C__INSERT_IN_INDEX(cache_ptr, entry_ptr, FAIL) - H5C__INSERT_ENTRY_IN_SLIST(cache_ptr, entry_ptr, FAIL) -#if H5C_DO_SANITY_CHECKS - if(removed_entry_from_slist) { - /* we just removed the entry from the slist. Thus we - * must touch up cache_ptr->slist_len_increase and - * cache_ptr->slist_size_increase to keep from skewing - * the sanity checks. - */ - cache_ptr->slist_len_increase -= 1; - cache_ptr->slist_size_increase -= (int64_t)(entry_ptr->size); - } /* end if */ -#endif /* H5C_DO_SANITY_CHECKS */ - + /* Skip some actions if we're in the middle of flushing the entry */ if(!entry_ptr->flush_in_progress) { - /* skip the update if a flush is in progress */ + /* Update the replacement policy for the entry */ H5C__UPDATE_RP_FOR_MOVE(cache_ptr, entry_ptr, was_dirty, FAIL) - } + + /* Check for entry changing status and do notifications, etc. */ + if(!was_dirty) { + /* If the entry's type has a 'notify' callback send a 'entry dirtied' + * notice now that the entry is fully integrated into the cache. + */ + if(entry_ptr->type->notify && + (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_DIRTIED, entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag set") + + /* Propagate the dirty flag up the flush dependency chain if appropriate */ + if(entry_ptr->flush_dep_nparents > 0) + if(H5C__mark_flush_dep_dirty(entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep dirty flag") + } /* end if */ + } /* end if */ } /* end if */ H5C__UPDATE_STATS_FOR_MOVE(cache_ptr, entry_ptr) done: #if H5C_DO_EXTREME_SANITY_CHECKS - if ( ( H5C_validate_protected_entry_list(cache_ptr) < 0 ) || - ( H5C_validate_pinned_entry_list(cache_ptr) < 0 ) || - ( H5C_validate_lru_list(cache_ptr) < 0 ) ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "an extreme sanity check failed on exit.\n"); - } + if((H5C_validate_protected_entry_list(cache_ptr) < 0) || + (H5C_validate_pinned_entry_list(cache_ptr) < 0) || + (H5C_validate_lru_list(cache_ptr) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on exit.\n") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ FUNC_LEAVE_NOAPI(ret_value) @@ -1842,11 +1859,6 @@ H5C_resize_entry(void *thing, size_t new_size) if(entry_ptr->image_ptr) entry_ptr->image_ptr = H5MM_xfree(entry_ptr->image_ptr); - /* Propagate the dirty flag up the flush dependency chain if appropriate */ - if(was_clean && entry_ptr->flush_dep_nparents > 0) - if(H5C__mark_flush_dep_dirty(entry_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep dirty flag") - /* do a flash cache size increase if appropriate */ if ( cache_ptr->flash_size_increase_possible ) { @@ -1882,29 +1894,39 @@ H5C_resize_entry(void *thing, size_t new_size) } /* end if */ #endif /* H5_HAVE_PARALLEL */ + /* update statistics just before changing the entry size */ + H5C__UPDATE_STATS_FOR_ENTRY_SIZE_CHANGE(cache_ptr, entry_ptr, new_size); + /* update the hash table */ - H5C__UPDATE_INDEX_FOR_SIZE_CHANGE((cache_ptr), (entry_ptr->size),\ - (new_size), (entry_ptr), (was_clean)); + H5C__UPDATE_INDEX_FOR_SIZE_CHANGE(cache_ptr, entry_ptr->size, \ + new_size, entry_ptr, was_clean); /* if the entry is in the skip list, update that too */ - if ( entry_ptr->in_slist ) { - H5C__UPDATE_SLIST_FOR_SIZE_CHANGE((cache_ptr), (entry_ptr->size),\ - (new_size)); - } /* end if */ - - /* update statistics just before changing the entry size */ - H5C__UPDATE_STATS_FOR_ENTRY_SIZE_CHANGE((cache_ptr), (entry_ptr), \ - (new_size)); + if(entry_ptr->in_slist) + H5C__UPDATE_SLIST_FOR_SIZE_CHANGE(cache_ptr, entry_ptr->size, new_size); /* finally, update the entry size proper */ entry_ptr->size = new_size; - if(!entry_ptr->in_slist) { + if(!entry_ptr->in_slist) H5C__INSERT_ENTRY_IN_SLIST(cache_ptr, entry_ptr, FAIL) - } /* end if */ - if(entry_ptr->is_pinned) { + if(entry_ptr->is_pinned) H5C__UPDATE_STATS_FOR_DIRTY_PIN(cache_ptr, entry_ptr) + + /* Check for entry changing status and do notifications, etc. */ + if(was_clean) { + /* If the entry's type has a 'notify' callback send a 'entry dirtied' + * notice now that the entry is fully integrated into the cache. + */ + if(entry_ptr->type->notify && + (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_DIRTIED, entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag set") + + /* Propagate the dirty flag up the flush dependency chain if appropriate */ + if(entry_ptr->flush_dep_nparents > 0) + if(H5C__mark_flush_dep_dirty(entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep dirty flag") } /* end if */ } /* end if */ @@ -2112,6 +2134,7 @@ H5C_protect(H5F_t * f, hbool_t coll_access = FALSE; /* whether access to the cache entry is done collectively */ #endif /* H5_HAVE_PARALLEL */ hbool_t write_permitted; + hbool_t was_loaded = FALSE; /* Whether the entry was loaded as a result of the protect */ size_t empty_space; void * thing; H5C_cache_entry_t * entry_ptr; @@ -2197,25 +2220,17 @@ H5C_protect(H5F_t * f, if(entry_ptr->image_ptr == NULL) { int mpi_rank; - size_t image_size; if((mpi_rank = H5F_mpi_get_rank(f)) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "Can't get MPI rank") - if(entry_ptr->compressed) - image_size = entry_ptr->compressed_size; - else - image_size = entry_ptr->size; - HDassert(image_size > 0); - - if(NULL == (entry_ptr->image_ptr = H5MM_malloc(image_size + H5C_IMAGE_EXTRA_SPACE))) + if(NULL == (entry_ptr->image_ptr = H5MM_malloc(entry_ptr->size + H5C_IMAGE_EXTRA_SPACE))) HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, NULL, "memory allocation failed for on disk image buffer") #if H5C_DO_MEMORY_SANITY_CHECKS - HDmemcpy(((uint8_t *)entry_ptr->image_ptr) + image_size, - H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); + HDmemcpy(((uint8_t *)entry_ptr->image_ptr) + entry_ptr->size, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); #endif /* H5C_DO_MEMORY_SANITY_CHECKS */ if(0 == mpi_rank) - if(H5C__generate_image(f, cache_ptr, entry_ptr, dxpl_id, NULL) < 0) + if(H5C__generate_image(f, cache_ptr, entry_ptr, dxpl_id) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, NULL, "can't generate entry's image") } /* end if */ HDassert(entry_ptr->image_ptr); @@ -2236,22 +2251,21 @@ H5C_protect(H5F_t * f, #if H5C_DO_TAGGING_SANITY_CHECKS { - H5C_tag_t tag; /* Tag structure */ + /* Verify tag value */ + if(cache_ptr->ignore_tags != TRUE) { + haddr_t tag; /* Tag value */ - /* The entry is already in the cache, but make sure that the tag value - being passed in via dxpl is still legal. This will ensure that had - the entry NOT been in the cache, tagging was still set up correctly - and it would have received a legal tag value after getting loaded - from disk. */ + /* The entry is already in the cache, but make sure that the tag value + being passed in via dxpl is still legal. This will ensure that had + the entry NOT been in the cache, tagging was still set up correctly + and it would have received a legal tag value after getting loaded + from disk. */ - /* Get the tag from the DXPL */ - if((H5P_get(dxpl, "H5C_tag", &tag)) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "unable to query property value"); + /* Get the tag from the DXPL */ + if((H5P_get(dxpl, H5AC_TAG_NAME, &tag)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "unable to query property value"); - /* Verify tag value */ - if(cache_ptr->ignore_tags != TRUE) { - /* Verify legal tag value */ - if(H5C_verify_tag(entry_ptr->type->id, tag.value, tag.globality) < 0) + if(H5C_verify_tag(entry_ptr->type->id, tag) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, NULL, "tag verification failed") } /* end if */ } @@ -2284,10 +2298,6 @@ H5C_protect(H5F_t * f, if(H5C__tag_entry(cache_ptr, entry_ptr, dxpl_id) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, NULL, "Cannot tag metadata entry") - /* Set the entry's cork status */ - if(H5C_cork(cache_ptr, entry_ptr->tag, H5C__GET_CORKED, &entry_ptr->is_corked) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, NULL, "Cannot retrieve entry's cork status") - /* If the entry is very large, and we are configured to allow it, * we may wish to perform a flash cache size increase. */ @@ -2409,12 +2419,9 @@ H5C_protect(H5F_t * f, */ H5C__UPDATE_RP_FOR_INSERTION(cache_ptr, entry_ptr, NULL) - /* If the entry's type has a 'notify' callback send a 'after load' - * notice now that the entry is fully integrated into the cache. - */ - if(entry_ptr->type->notify && - (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_AFTER_LOAD, entry_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, NULL, "can't notify client about entry inserted into cache") + /* Record that the entry was loaded, to trigger a notify callback later */ + /* (After the entry is fully added to the cache) */ + was_loaded = TRUE; } HDassert( entry_ptr->addr == addr ); @@ -2516,6 +2523,20 @@ H5C_protect(H5F_t * f, } } + /* If we loaded the entry and the entry's type has a 'notify' callback, send + * an 'after load' notice now that the entry is fully integrated into + * the cache and protected. We must wait until it is protected so it is not + * evicted during the notify callback. + */ + if(was_loaded) { + /* If the entry's type has a 'notify' callback send a 'after load' + * notice now that the entry is fully integrated into the cache. + */ + if(entry_ptr->type->notify && + (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_AFTER_LOAD, entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, NULL, "can't notify client about entry inserted into cache") + } /* end if */ + #ifdef H5_HAVE_PARALLEL if(H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) { /* Make sure the size of the collective entries in the cache remain in check */ @@ -2535,7 +2556,6 @@ H5C_protect(H5F_t * f, #endif /* H5_HAVE_PARALLEL */ done: - #if H5C_DO_EXTREME_SANITY_CHECKS if ( ( H5C_validate_protected_entry_list(cache_ptr) < 0 ) || ( H5C_validate_pinned_entry_list(cache_ptr) < 0 ) || @@ -2544,7 +2564,6 @@ done: #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ FUNC_LEAVE_NOAPI(ret_value) - } /* H5C_protect() */ @@ -2567,18 +2586,14 @@ H5C_reset_cache_hit_rate_stats(H5C_t * cache_ptr) FUNC_ENTER_NOAPI(FAIL) - if ( ( cache_ptr == NULL ) || ( cache_ptr->magic != H5C__H5C_T_MAGIC ) ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Bad cache_ptr on entry.") - } + if((cache_ptr == NULL) || (cache_ptr->magic != H5C__H5C_T_MAGIC)) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "bad cache_ptr on entry.") cache_ptr->cache_hits = 0; cache_ptr->cache_accesses = 0; done: - FUNC_LEAVE_NOAPI(ret_value) - } /* H5C_reset_cache_hit_rate_stats() */ @@ -2605,137 +2620,95 @@ herr_t H5C_set_cache_auto_resize_config(H5C_t *cache_ptr, H5C_auto_size_ctl_t *config_ptr) { - herr_t result; size_t new_max_cache_size; size_t new_min_clean_size; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) - if ( ( cache_ptr == NULL ) || ( cache_ptr->magic != H5C__H5C_T_MAGIC ) ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Bad cache_ptr on entry.") - } - - if ( config_ptr == NULL ) { - + if((cache_ptr == NULL) || (cache_ptr->magic != H5C__H5C_T_MAGIC)) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "bad cache_ptr on entry.") + if(config_ptr == NULL) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "NULL config_ptr on entry.") - } - - if ( config_ptr->version != H5C__CURR_AUTO_SIZE_CTL_VER ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Unknown config version.") - } + if(config_ptr->version != H5C__CURR_AUTO_SIZE_CTL_VER) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "unknown config version.") /* check general configuration section of the config: */ - if ( SUCCEED != H5C_validate_resize_config(config_ptr, - H5C_RESIZE_CFG__VALIDATE_GENERAL) ) { - - HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, \ - "error in general configuration fields of new config.") - } + if(H5C_validate_resize_config(config_ptr, H5C_RESIZE_CFG__VALIDATE_GENERAL) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "error in general configuration fields of new config.") /* check size increase control fields of the config: */ - if ( SUCCEED != H5C_validate_resize_config(config_ptr, - H5C_RESIZE_CFG__VALIDATE_INCREMENT) ) { - - HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, \ - "error in the size increase control fields of new config.") - } + if(H5C_validate_resize_config(config_ptr, H5C_RESIZE_CFG__VALIDATE_INCREMENT) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "error in the size increase control fields of new config.") /* check size decrease control fields of the config: */ - if ( SUCCEED != H5C_validate_resize_config(config_ptr, - H5C_RESIZE_CFG__VALIDATE_DECREMENT) ) { - - HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, \ - "error in the size decrease control fields of new config.") - } + if(H5C_validate_resize_config(config_ptr, H5C_RESIZE_CFG__VALIDATE_DECREMENT) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "error in the size decrease control fields of new config.") /* check for conflicts between size increase and size decrease controls: */ - if ( SUCCEED != H5C_validate_resize_config(config_ptr, - H5C_RESIZE_CFG__VALIDATE_INTERACTIONS) ) { - - HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, \ - "conflicting threshold fields in new config.") - } + if(H5C_validate_resize_config(config_ptr, H5C_RESIZE_CFG__VALIDATE_INTERACTIONS) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "conflicting threshold fields in new config.") /* will set the increase possible fields to FALSE later if needed */ cache_ptr->size_increase_possible = TRUE; cache_ptr->flash_size_increase_possible = TRUE; cache_ptr->size_decrease_possible = TRUE; - switch ( config_ptr->incr_mode ) - { + switch(config_ptr->incr_mode) { case H5C_incr__off: cache_ptr->size_increase_possible = FALSE; break; case H5C_incr__threshold: - if ( ( config_ptr->lower_hr_threshold <= (double)0.0f ) || - ( config_ptr->increment <= (double)1.0f ) || - ( ( config_ptr->apply_max_increment ) && - ( config_ptr->max_increment <= 0 ) ) ) { - + if((config_ptr->lower_hr_threshold <= (double)0.0f) || + (config_ptr->increment <= (double)1.0f) || + ((config_ptr->apply_max_increment) && (config_ptr->max_increment <= 0))) cache_ptr->size_increase_possible = FALSE; - } break; default: /* should be unreachable */ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Unknown incr_mode?!?!?.") - } + } /* end switch */ /* logically, this is were configuration for flash cache size increases * should go. However, this configuration depends on max_cache_size, so * we wait until the end of the function, when this field is set. */ - switch ( config_ptr->decr_mode ) - { + switch(config_ptr->decr_mode) { case H5C_decr__off: cache_ptr->size_decrease_possible = FALSE; break; case H5C_decr__threshold: - if ( ( config_ptr->upper_hr_threshold >= (double)1.0f ) || - ( config_ptr->decrement >= (double)1.0f ) || - ( ( config_ptr->apply_max_decrement ) && - ( config_ptr->max_decrement <= 0 ) ) ) { - + if((config_ptr->upper_hr_threshold >= (double)1.0f) || + (config_ptr->decrement >= (double)1.0f) || + ((config_ptr->apply_max_decrement) && (config_ptr->max_decrement <= 0))) cache_ptr->size_decrease_possible = FALSE; - } break; case H5C_decr__age_out: - if ( ( ( config_ptr->apply_empty_reserve ) && - ( config_ptr->empty_reserve >= (double)1.0f ) ) || - ( ( config_ptr->apply_max_decrement ) && - ( config_ptr->max_decrement <= 0 ) ) ) { - + if(((config_ptr->apply_empty_reserve) && (config_ptr->empty_reserve >= (double)1.0f)) || + ((config_ptr->apply_max_decrement) && (config_ptr->max_decrement <= 0))) cache_ptr->size_decrease_possible = FALSE; - } break; case H5C_decr__age_out_with_threshold: - if ( ( ( config_ptr->apply_empty_reserve ) && - ( config_ptr->empty_reserve >= (double)1.0f ) ) || - ( ( config_ptr->apply_max_decrement ) && - ( config_ptr->max_decrement <= 0 ) ) || - ( config_ptr->upper_hr_threshold >= (double)1.0f ) ) { - + if(((config_ptr->apply_empty_reserve) && (config_ptr->empty_reserve >= (double)1.0f)) || + ((config_ptr->apply_max_decrement) && (config_ptr->max_decrement <= 0)) || + (config_ptr->upper_hr_threshold >= (double)1.0f)) cache_ptr->size_decrease_possible = FALSE; - } break; default: /* should be unreachable */ HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Unknown decr_mode?!?!?.") - } - - if ( config_ptr->max_size == config_ptr->min_size ) { + } /* end switch */ + if(config_ptr->max_size == config_ptr->min_size) { cache_ptr->size_increase_possible = FALSE; cache_ptr->flash_size_increase_possible = FALSE; cache_ptr->size_decrease_possible = FALSE; - } + } /* end if */ /* flash_size_increase_possible is intentionally omitted from the * following: @@ -2753,25 +2726,16 @@ H5C_set_cache_auto_resize_config(H5C_t *cache_ptr, * go through the exercise even if the current size is within * range and an initial size has not been provided. */ - if ( (cache_ptr->resize_ctl).set_initial_size ) { - - new_max_cache_size = (cache_ptr->resize_ctl).initial_size; - } - else if ( cache_ptr->max_cache_size > (cache_ptr->resize_ctl).max_size ) { - - new_max_cache_size = (cache_ptr->resize_ctl).max_size; - } - else if ( cache_ptr->max_cache_size < (cache_ptr->resize_ctl).min_size ) { - - new_max_cache_size = (cache_ptr->resize_ctl).min_size; - - } else { - + if(cache_ptr->resize_ctl.set_initial_size) + new_max_cache_size = cache_ptr->resize_ctl.initial_size; + else if(cache_ptr->max_cache_size > cache_ptr->resize_ctl.max_size) + new_max_cache_size = cache_ptr->resize_ctl.max_size; + else if(cache_ptr->max_cache_size < cache_ptr->resize_ctl.min_size) + new_max_cache_size = cache_ptr->resize_ctl.min_size; + else new_max_cache_size = cache_ptr->max_cache_size; - } - new_min_clean_size = (size_t) - ((double)new_max_cache_size * + new_min_clean_size = (size_t)((double)new_max_cache_size * ((cache_ptr->resize_ctl).min_clean_fraction)); @@ -2781,50 +2745,30 @@ H5C_set_cache_auto_resize_config(H5C_t *cache_ptr, * * by definition. */ - HDassert( new_min_clean_size <= new_max_cache_size ); - HDassert( (cache_ptr->resize_ctl).min_size <= new_max_cache_size ); - HDassert( new_max_cache_size <= (cache_ptr->resize_ctl).max_size ); - - if ( new_max_cache_size < cache_ptr->max_cache_size ) { + HDassert(new_min_clean_size <= new_max_cache_size); + HDassert(cache_ptr->resize_ctl.min_size <= new_max_cache_size); + HDassert(new_max_cache_size <= cache_ptr->resize_ctl.max_size); + if(new_max_cache_size < cache_ptr->max_cache_size) cache_ptr->size_decreased = TRUE; - } cache_ptr->max_cache_size = new_max_cache_size; cache_ptr->min_clean_size = new_min_clean_size; - if ( H5C_reset_cache_hit_rate_stats(cache_ptr) != SUCCEED ) { - + if(H5C_reset_cache_hit_rate_stats(cache_ptr) < 0) /* this should be impossible... */ - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "H5C_reset_cache_hit_rate_stats failed.") - } + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_reset_cache_hit_rate_stats failed.") /* remove excess epoch markers if any */ - if ( ( config_ptr->decr_mode == H5C_decr__age_out_with_threshold ) || - ( config_ptr->decr_mode == H5C_decr__age_out ) ) { - - if ( cache_ptr->epoch_markers_active > - (cache_ptr->resize_ctl).epochs_before_eviction ) { - - result = - H5C__autoadjust__ageout__remove_excess_markers(cache_ptr); - - if ( result != SUCCEED ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "can't remove excess epoch markers.") - } - } - } else if ( cache_ptr->epoch_markers_active > 0 ) { - - result = H5C__autoadjust__ageout__remove_all_markers(cache_ptr); - - if ( result != SUCCEED ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "error removing all epoch markers.") - } + if((config_ptr->decr_mode == H5C_decr__age_out_with_threshold) || + (config_ptr->decr_mode == H5C_decr__age_out)) { + if(cache_ptr->epoch_markers_active > cache_ptr->resize_ctl.epochs_before_eviction) + if(H5C__autoadjust__ageout__remove_excess_markers(cache_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "can't remove excess epoch markers.") + } /* end if */ + else if(cache_ptr->epoch_markers_active > 0) { + if(H5C__autoadjust__ageout__remove_all_markers(cache_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "error removing all epoch markers.") } /* configure flash size increase facility. We wait until the @@ -2835,33 +2779,26 @@ H5C_set_cache_auto_resize_config(H5C_t *cache_ptr, * go ahead and configure it. */ - if ( cache_ptr->flash_size_increase_possible ) { - - switch ( config_ptr->flash_incr_mode ) - { + if(cache_ptr->flash_size_increase_possible) { + switch(config_ptr->flash_incr_mode) { case H5C_flash_incr__off: cache_ptr->flash_size_increase_possible = FALSE; break; case H5C_flash_incr__add_space: cache_ptr->flash_size_increase_possible = TRUE; - cache_ptr->flash_size_increase_threshold = - (size_t) - (((double)(cache_ptr->max_cache_size)) * + cache_ptr->flash_size_increase_threshold = (size_t)(((double)(cache_ptr->max_cache_size)) * ((cache_ptr->resize_ctl).flash_threshold)); break; default: /* should be unreachable */ - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "Unknown flash_incr_mode?!?!?.") + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Unknown flash_incr_mode?!?!?.") break; - } - } + } /* end switch */ + } /* end if */ done: - FUNC_LEAVE_NOAPI(ret_value) - } /* H5C_set_cache_auto_resize_config() */ @@ -2907,922 +2844,6 @@ done: /*------------------------------------------------------------------------- - * Function: H5C_set_prefix - * - * Purpose: Set the values of the prefix field of H5C_t. This - * filed is used to label some debugging output. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: John Mainzer - * 1/20/06 - * - *------------------------------------------------------------------------- - */ -herr_t -H5C_set_prefix(H5C_t * cache_ptr, char * prefix) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - if ( ( cache_ptr == NULL ) || - ( cache_ptr->magic != H5C__H5C_T_MAGIC ) || - ( prefix == NULL ) || - ( HDstrlen(prefix) >= H5C__PREFIX_LEN ) ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Bad param(s) on entry.") - } - - HDstrncpy(&(cache_ptr->prefix[0]), prefix, (size_t)(H5C__PREFIX_LEN)); - - cache_ptr->prefix[H5C__PREFIX_LEN - 1] = '\0'; - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5C_set_prefix() */ - - -/*------------------------------------------------------------------------- - * Function: H5C_set_trace_file_ptr - * - * Purpose: Set the trace_file_ptr field for the cache. - * - * This field must either be NULL (which turns of trace - * file logging), or be a pointer to an open file to which - * trace file data is to be written. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: John Mainzer - * 1/20/06 - * - *------------------------------------------------------------------------- - */ -herr_t -H5C_set_trace_file_ptr(H5C_t * cache_ptr, - FILE * trace_file_ptr) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - /* This would normally be an assert, but we need to use an HGOTO_ERROR - * call to shut up the compiler. - */ - if ( ( ! cache_ptr ) || ( cache_ptr->magic != H5C__H5C_T_MAGIC ) ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Bad cache_ptr") - } - - cache_ptr->trace_file_ptr = trace_file_ptr; - -done: - FUNC_LEAVE_NOAPI(ret_value) - -} /* H5C_set_trace_file_ptr() */ - - -/*------------------------------------------------------------------------- - * Function: H5C_stats - * - * Purpose: Prints statistics about the cache. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: John Mainzer - * 6/2/04 - * - * JRM -- 11/13/08 - * Added code displaying the max_clean_index_size and - * max_dirty_index_size. - * - * MAM -- 01/06/09 - * Added code displaying the calls_to_msic, - * total_entries_skipped_in_msic, total_entries_scanned_in_msic, - * and max_entries_skipped_in_msic fields. - * - * JRM -- 4/11/15 - * Added code displaying the new slist_scan_restarts, - * LRU_scan_restarts, and hash_bucket_scan_restarts fields; - * - *------------------------------------------------------------------------- - */ -herr_t -H5C_stats(H5C_t * cache_ptr, - const char * cache_name, - hbool_t -#if !H5C_COLLECT_CACHE_STATS - H5_ATTR_UNUSED -#endif /* H5C_COLLECT_CACHE_STATS */ - display_detailed_stats) -{ -#if H5C_COLLECT_CACHE_STATS - int i; - int64_t total_hits = 0; - int64_t total_misses = 0; - int64_t total_write_protects = 0; - int64_t total_read_protects = 0; - int64_t max_read_protects = 0; - int64_t total_insertions = 0; - int64_t total_pinned_insertions = 0; - int64_t total_clears = 0; - int64_t total_flushes = 0; - int64_t total_evictions = 0; - int64_t total_take_ownerships = 0; - int64_t total_moves = 0; - int64_t total_entry_flush_moves = 0; - int64_t total_cache_flush_moves = 0; - int64_t total_size_increases = 0; - int64_t total_size_decreases = 0; - int64_t total_entry_flush_size_changes = 0; - int64_t total_cache_flush_size_changes = 0; - int64_t total_pins = 0; - int64_t total_unpins = 0; - int64_t total_dirty_pins = 0; - int64_t total_pinned_flushes = 0; - int64_t total_pinned_clears = 0; - int32_t aggregate_max_accesses = 0; - int32_t aggregate_min_accesses = 1000000; - int32_t aggregate_max_clears = 0; - int32_t aggregate_max_flushes = 0; - size_t aggregate_max_size = 0; - int32_t aggregate_max_pins = 0; - double hit_rate; - double average_successful_search_depth = 0.0f; - double average_failed_search_depth = 0.0f; - double average_entries_skipped_per_calls_to_msic = 0.0f; - double average_entries_scanned_per_calls_to_msic = 0.0f; -#endif /* H5C_COLLECT_CACHE_STATS */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); - - /* This would normally be an assert, but we need to use an HGOTO_ERROR - * call to shut up the compiler. - */ - if ( ( ! cache_ptr ) || - ( cache_ptr->magic != H5C__H5C_T_MAGIC ) || - ( !cache_name ) ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Bad cache_ptr or cache_name") - } - -#if H5C_COLLECT_CACHE_STATS - - for ( i = 0; i <= cache_ptr->max_type_id; i++ ) { - - total_hits += cache_ptr->hits[i]; - total_misses += cache_ptr->misses[i]; - total_write_protects += cache_ptr->write_protects[i]; - total_read_protects += cache_ptr->read_protects[i]; - if ( max_read_protects < cache_ptr->max_read_protects[i] ) { - max_read_protects = cache_ptr->max_read_protects[i]; - } - total_insertions += cache_ptr->insertions[i]; - total_pinned_insertions += cache_ptr->pinned_insertions[i]; - total_clears += cache_ptr->clears[i]; - total_flushes += cache_ptr->flushes[i]; - total_evictions += cache_ptr->evictions[i]; - total_take_ownerships += cache_ptr->take_ownerships[i]; - total_moves += cache_ptr->moves[i]; - total_entry_flush_moves += cache_ptr->entry_flush_moves[i]; - total_cache_flush_moves += cache_ptr->cache_flush_moves[i]; - total_size_increases += cache_ptr->size_increases[i]; - total_size_decreases += cache_ptr->size_decreases[i]; - total_entry_flush_size_changes - += cache_ptr->entry_flush_size_changes[i]; - total_cache_flush_size_changes - += cache_ptr->cache_flush_size_changes[i]; - total_pins += cache_ptr->pins[i]; - total_unpins += cache_ptr->unpins[i]; - total_dirty_pins += cache_ptr->dirty_pins[i]; - total_pinned_flushes += cache_ptr->pinned_flushes[i]; - total_pinned_clears += cache_ptr->pinned_clears[i]; -#if H5C_COLLECT_CACHE_ENTRY_STATS - if ( aggregate_max_accesses < cache_ptr->max_accesses[i] ) - aggregate_max_accesses = cache_ptr->max_accesses[i]; - if ( aggregate_min_accesses > aggregate_max_accesses ) - aggregate_min_accesses = aggregate_max_accesses; - if ( aggregate_min_accesses > cache_ptr->min_accesses[i] ) - aggregate_min_accesses = cache_ptr->min_accesses[i]; - if ( aggregate_max_clears < cache_ptr->max_clears[i] ) - aggregate_max_clears = cache_ptr->max_clears[i]; - if ( aggregate_max_flushes < cache_ptr->max_flushes[i] ) - aggregate_max_flushes = cache_ptr->max_flushes[i]; - if ( aggregate_max_size < cache_ptr->max_size[i] ) - aggregate_max_size = cache_ptr->max_size[i]; - if ( aggregate_max_pins < cache_ptr->max_pins[i] ) - aggregate_max_pins = cache_ptr->max_pins[i]; -#endif /* H5C_COLLECT_CACHE_ENTRY_STATS */ - } - - if ( ( total_hits > 0 ) || ( total_misses > 0 ) ) { - - hit_rate = (double)100.0f * ((double)(total_hits)) / - ((double)(total_hits + total_misses)); - } else { - hit_rate = 0.0f; - } - - if ( cache_ptr->successful_ht_searches > 0 ) { - - average_successful_search_depth = - ((double)(cache_ptr->total_successful_ht_search_depth)) / - ((double)(cache_ptr->successful_ht_searches)); - } - - if ( cache_ptr->failed_ht_searches > 0 ) { - - average_failed_search_depth = - ((double)(cache_ptr->total_failed_ht_search_depth)) / - ((double)(cache_ptr->failed_ht_searches)); - } - - - HDfprintf(stdout, "\n%sH5C: cache statistics for %s\n", - cache_ptr->prefix, cache_name); - - HDfprintf(stdout, "\n"); - - HDfprintf(stdout, - "%s hash table insertion / deletions = %ld / %ld\n", - cache_ptr->prefix, - (long)(cache_ptr->total_ht_insertions), - (long)(cache_ptr->total_ht_deletions)); - - HDfprintf(stdout, - "%s HT successful / failed searches = %ld / %ld\n", - cache_ptr->prefix, - (long)(cache_ptr->successful_ht_searches), - (long)(cache_ptr->failed_ht_searches)); - - HDfprintf(stdout, - "%s Av. HT suc / failed search depth = %f / %f\n", - cache_ptr->prefix, - average_successful_search_depth, - average_failed_search_depth); - - HDfprintf(stdout, - "%s current (max) index size / length = %ld (%ld) / %ld (%ld)\n", - cache_ptr->prefix, - (long)(cache_ptr->index_size), - (long)(cache_ptr->max_index_size), - (long)(cache_ptr->index_len), - (long)(cache_ptr->max_index_len)); - - HDfprintf(stdout, - "%s current (max) clean/dirty idx size = %ld (%ld) / %ld (%ld)\n", - cache_ptr->prefix, - (long)(cache_ptr->clean_index_size), - (long)(cache_ptr->max_clean_index_size), - (long)(cache_ptr->dirty_index_size), - (long)(cache_ptr->max_dirty_index_size)); - - HDfprintf(stdout, - "%s current (max) slist size / length = %ld (%ld) / %ld (%ld)\n", - cache_ptr->prefix, - (long)(cache_ptr->slist_size), - (long)(cache_ptr->max_slist_size), - (long)(cache_ptr->slist_len), - (long)(cache_ptr->max_slist_len)); - - HDfprintf(stdout, - "%s current (max) PL size / length = %ld (%ld) / %ld (%ld)\n", - cache_ptr->prefix, - (long)(cache_ptr->pl_size), - (long)(cache_ptr->max_pl_size), - (long)(cache_ptr->pl_len), - (long)(cache_ptr->max_pl_len)); - - HDfprintf(stdout, - "%s current (max) PEL size / length = %ld (%ld) / %ld (%ld)\n", - cache_ptr->prefix, - (long)(cache_ptr->pel_size), - (long)(cache_ptr->max_pel_size), - (long)(cache_ptr->pel_len), - (long)(cache_ptr->max_pel_len)); - - HDfprintf(stdout, - "%s current LRU list size / length = %ld / %ld\n", - cache_ptr->prefix, - (long)(cache_ptr->LRU_list_size), - (long)(cache_ptr->LRU_list_len)); - - HDfprintf(stdout, - "%s current clean LRU size / length = %ld / %ld\n", - cache_ptr->prefix, - (long)(cache_ptr->cLRU_list_size), - (long)(cache_ptr->cLRU_list_len)); - - HDfprintf(stdout, - "%s current dirty LRU size / length = %ld / %ld\n", - cache_ptr->prefix, - (long)(cache_ptr->dLRU_list_size), - (long)(cache_ptr->dLRU_list_len)); - - HDfprintf(stdout, - "%s Total hits / misses / hit_rate = %ld / %ld / %f\n", - cache_ptr->prefix, - (long)total_hits, - (long)total_misses, - hit_rate); - - HDfprintf(stdout, - "%s Total write / read (max) protects = %ld / %ld (%ld)\n", - cache_ptr->prefix, - (long)total_write_protects, - (long)total_read_protects, - (long)max_read_protects); - - HDfprintf(stdout, - "%s Total clears / flushes = %ld / %ld\n", - cache_ptr->prefix, - (long)total_clears, - (long)total_flushes); - - HDfprintf(stdout, - "%s Total evictions / take ownerships = %ld / %ld\n", - cache_ptr->prefix, - (long)total_evictions, - (long)total_take_ownerships); - - HDfprintf(stdout, - "%s Total insertions(pinned) / moves = %ld(%ld) / %ld\n", - cache_ptr->prefix, - (long)total_insertions, - (long)total_pinned_insertions, - (long)total_moves); - - HDfprintf(stdout, - "%s Total entry / cache flush moves = %ld / %ld\n", - cache_ptr->prefix, - (long)total_entry_flush_moves, - (long)total_cache_flush_moves); - - HDfprintf(stdout, "%s Total entry size incrs / decrs = %ld / %ld\n", - cache_ptr->prefix, - (long)total_size_increases, - (long)total_size_decreases); - - HDfprintf(stdout, "%s Ttl entry/cache flush size changes = %ld / %ld\n", - cache_ptr->prefix, - (long)total_entry_flush_size_changes, - (long)total_cache_flush_size_changes); - - HDfprintf(stdout, - "%s Total entry pins (dirty) / unpins = %ld (%ld) / %ld\n", - cache_ptr->prefix, - (long)total_pins, - (long)total_dirty_pins, - (long)total_unpins); - - HDfprintf(stdout, "%s Total pinned flushes / clears = %ld / %ld\n", - cache_ptr->prefix, - (long)total_pinned_flushes, - (long)total_pinned_clears); - - HDfprintf(stdout, "%s MSIC: (make space in cache) calls = %lld\n", - cache_ptr->prefix, - (long long)(cache_ptr->calls_to_msic)); - - if (cache_ptr->calls_to_msic > 0) { - average_entries_skipped_per_calls_to_msic = - (((double)(cache_ptr->total_entries_skipped_in_msic)) / - ((double)(cache_ptr->calls_to_msic))); - } - - HDfprintf(stdout, "%s MSIC: Average/max entries skipped = %lf / %ld\n", - cache_ptr->prefix, - (double)average_entries_skipped_per_calls_to_msic, - (long)(cache_ptr->max_entries_skipped_in_msic)); - - if (cache_ptr->calls_to_msic > 0) { - average_entries_scanned_per_calls_to_msic = - (((double)(cache_ptr->total_entries_scanned_in_msic)) / - ((double)(cache_ptr->calls_to_msic))); - } - - HDfprintf(stdout, "%s MSIC: Average/max entries scanned = %lf / %ld\n", - cache_ptr->prefix, - (double)average_entries_scanned_per_calls_to_msic, - (long)(cache_ptr->max_entries_scanned_in_msic)); - - HDfprintf(stdout, "%s MSIC: Scanned to make space(evict) = %lld\n", - cache_ptr->prefix, - (long long)(cache_ptr->entries_scanned_to_make_space)); - - HDfprintf(stdout, "%s MSIC: Scanned to satisfy min_clean = %lld\n", - cache_ptr->prefix, - (long long)(cache_ptr->total_entries_scanned_in_msic - - cache_ptr->entries_scanned_to_make_space)); - - HDfprintf(stdout, - "%s slist/LRU/hash bkt scan restarts = %lld / %lld / %lld.\n", - cache_ptr->prefix, - (long long)(cache_ptr->slist_scan_restarts), - (long long)(cache_ptr->LRU_scan_restarts), - (long long)(cache_ptr->hash_bucket_scan_restarts)); - -#if H5C_COLLECT_CACHE_ENTRY_STATS - - HDfprintf(stdout, "%s aggregate max / min accesses = %d / %d\n", - cache_ptr->prefix, - (int)aggregate_max_accesses, - (int)aggregate_min_accesses); - - HDfprintf(stdout, "%s aggregate max_clears / max_flushes = %d / %d\n", - cache_ptr->prefix, - (int)aggregate_max_clears, - (int)aggregate_max_flushes); - - HDfprintf(stdout, "%s aggregate max_size / max_pins = %d / %d\n", - cache_ptr->prefix, - (int)aggregate_max_size, - (int)aggregate_max_pins); - -#endif /* H5C_COLLECT_CACHE_ENTRY_STATS */ - - if ( display_detailed_stats ) - { - - for ( i = 0; i <= cache_ptr->max_type_id; i++ ) { - - HDfprintf(stdout, "\n"); - - HDfprintf(stdout, "%s Stats on %s:\n", - cache_ptr->prefix, - ((cache_ptr->type_name_table_ptr))[i]); - - if ( ( cache_ptr->hits[i] > 0 ) || ( cache_ptr->misses[i] > 0 ) ) { - - hit_rate = (double)100.0f * ((double)(cache_ptr->hits[i])) / - ((double)(cache_ptr->hits[i] + cache_ptr->misses[i])); - } else { - hit_rate = 0.0f; - } - - HDfprintf(stdout, - "%s hits / misses / hit_rate = %ld / %ld / %f\n", - cache_ptr->prefix, - (long)(cache_ptr->hits[i]), - (long)(cache_ptr->misses[i]), - hit_rate); - - HDfprintf(stdout, - "%s write / read (max) protects = %ld / %ld (%d)\n", - cache_ptr->prefix, - (long)(cache_ptr->write_protects[i]), - (long)(cache_ptr->read_protects[i]), - (int)(cache_ptr->max_read_protects[i])); - - HDfprintf(stdout, - "%s clears / flushes = %ld / %ld\n", - cache_ptr->prefix, - (long)(cache_ptr->clears[i]), - (long)(cache_ptr->flushes[i])); - - HDfprintf(stdout, - "%s evictions / take ownerships = %ld / %ld\n", - cache_ptr->prefix, - (long)(cache_ptr->evictions[i]), - (long)(cache_ptr->take_ownerships[i])); - - HDfprintf(stdout, - "%s insertions(pinned) / moves = %ld(%ld) / %ld\n", - cache_ptr->prefix, - (long)(cache_ptr->insertions[i]), - (long)(cache_ptr->pinned_insertions[i]), - (long)(cache_ptr->moves[i])); - - HDfprintf(stdout, - "%s entry / cache flush moves = %ld / %ld\n", - cache_ptr->prefix, - (long)(cache_ptr->entry_flush_moves[i]), - (long)(cache_ptr->cache_flush_moves[i])); - - HDfprintf(stdout, - "%s size increases / decreases = %ld / %ld\n", - cache_ptr->prefix, - (long)(cache_ptr->size_increases[i]), - (long)(cache_ptr->size_decreases[i])); - - HDfprintf(stdout, - "%s entry/cache flush size changes = %ld / %ld\n", - cache_ptr->prefix, - (long)(cache_ptr->entry_flush_size_changes[i]), - (long)(cache_ptr->cache_flush_size_changes[i])); - - - HDfprintf(stdout, - "%s entry pins / unpins = %ld / %ld\n", - cache_ptr->prefix, - (long)(cache_ptr->pins[i]), - (long)(cache_ptr->unpins[i])); - - HDfprintf(stdout, - "%s entry dirty pins/pin'd flushes = %ld / %ld\n", - cache_ptr->prefix, - (long)(cache_ptr->dirty_pins[i]), - (long)(cache_ptr->pinned_flushes[i])); - -#if H5C_COLLECT_CACHE_ENTRY_STATS - - HDfprintf(stdout, - "%s entry max / min accesses = %d / %d\n", - cache_ptr->prefix, - cache_ptr->max_accesses[i], - cache_ptr->min_accesses[i]); - - HDfprintf(stdout, - "%s entry max_clears / max_flushes = %d / %d\n", - cache_ptr->prefix, - cache_ptr->max_clears[i], - cache_ptr->max_flushes[i]); - - HDfprintf(stdout, - "%s entry max_size / max_pins = %d / %d\n", - cache_ptr->prefix, - (int)(cache_ptr->max_size[i]), - (int)(cache_ptr->max_pins[i])); - - -#endif /* H5C_COLLECT_CACHE_ENTRY_STATS */ - - } - } - - HDfprintf(stdout, "\n"); - -#endif /* H5C_COLLECT_CACHE_STATS */ - -done: - FUNC_LEAVE_NOAPI(ret_value) - -} /* H5C_stats() */ - - -/*------------------------------------------------------------------------- - * - * Function: H5C_stats__reset - * - * Purpose: Reset the stats fields to their initial values. - * - * Return: void - * - * Programmer: John Mainzer, 4/28/04 - * - * JRM 11/13/08 - * Added initialization for the new max_clean_index_size and - * max_dirty_index_size fields. - * - * MAM -- 01/06/09 - * Added code to initalize the calls_to_msic, - * total_entries_skipped_in_msic, total_entries_scanned_in_msic, - * and max_entries_skipped_in_msic fields. - * - * JRM 4/11/15 - * Added code to initialize the new slist_scan_restarts, - * LRU_scan_restarts, hash_bucket_scan_restarts, and - * take_ownerships fields. - * - *------------------------------------------------------------------------- - */ -void -#ifndef NDEBUG -H5C_stats__reset(H5C_t * cache_ptr) -#else /* NDEBUG */ -#if H5C_COLLECT_CACHE_STATS -H5C_stats__reset(H5C_t * cache_ptr) -#else /* H5C_COLLECT_CACHE_STATS */ -H5C_stats__reset(H5C_t H5_ATTR_UNUSED * cache_ptr) -#endif /* H5C_COLLECT_CACHE_STATS */ -#endif /* NDEBUG */ -{ -#if H5C_COLLECT_CACHE_STATS - int i; -#endif /* H5C_COLLECT_CACHE_STATS */ - - HDassert( cache_ptr ); - HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); - -#if H5C_COLLECT_CACHE_STATS - for ( i = 0; i <= cache_ptr->max_type_id; i++ ) - { - cache_ptr->hits[i] = 0; - cache_ptr->misses[i] = 0; - cache_ptr->write_protects[i] = 0; - cache_ptr->read_protects[i] = 0; - cache_ptr->max_read_protects[i] = 0; - cache_ptr->insertions[i] = 0; - cache_ptr->pinned_insertions[i] = 0; - cache_ptr->clears[i] = 0; - cache_ptr->flushes[i] = 0; - cache_ptr->evictions[i] = 0; - cache_ptr->take_ownerships[i] = 0; - cache_ptr->moves[i] = 0; - cache_ptr->entry_flush_moves[i] = 0; - cache_ptr->cache_flush_moves[i] = 0; - cache_ptr->pins[i] = 0; - cache_ptr->unpins[i] = 0; - cache_ptr->dirty_pins[i] = 0; - cache_ptr->pinned_flushes[i] = 0; - cache_ptr->pinned_clears[i] = 0; - cache_ptr->size_increases[i] = 0; - cache_ptr->size_decreases[i] = 0; - cache_ptr->entry_flush_size_changes[i] = 0; - cache_ptr->cache_flush_size_changes[i] = 0; - } - - cache_ptr->total_ht_insertions = 0; - cache_ptr->total_ht_deletions = 0; - cache_ptr->successful_ht_searches = 0; - cache_ptr->total_successful_ht_search_depth = 0; - cache_ptr->failed_ht_searches = 0; - cache_ptr->total_failed_ht_search_depth = 0; - - cache_ptr->max_index_len = 0; - cache_ptr->max_index_size = (size_t)0; - cache_ptr->max_clean_index_size = (size_t)0; - cache_ptr->max_dirty_index_size = (size_t)0; - - cache_ptr->max_slist_len = 0; - cache_ptr->max_slist_size = (size_t)0; - - cache_ptr->max_pl_len = 0; - cache_ptr->max_pl_size = (size_t)0; - - cache_ptr->max_pel_len = 0; - cache_ptr->max_pel_size = (size_t)0; - - cache_ptr->calls_to_msic = 0; - cache_ptr->total_entries_skipped_in_msic = 0; - cache_ptr->total_entries_scanned_in_msic = 0; - cache_ptr->max_entries_skipped_in_msic = 0; - cache_ptr->max_entries_scanned_in_msic = 0; - cache_ptr->entries_scanned_to_make_space = 0; - - cache_ptr->slist_scan_restarts = 0; - cache_ptr->LRU_scan_restarts = 0; - cache_ptr->hash_bucket_scan_restarts = 0; - -#if H5C_COLLECT_CACHE_ENTRY_STATS - - for ( i = 0; i <= cache_ptr->max_type_id; i++ ) - { - cache_ptr->max_accesses[i] = 0; - cache_ptr->min_accesses[i] = 1000000; - cache_ptr->max_clears[i] = 0; - cache_ptr->max_flushes[i] = 0; - cache_ptr->max_size[i] = (size_t)0; - cache_ptr->max_pins[i] = 0; - } - -#endif /* H5C_COLLECT_CACHE_ENTRY_STATS */ -#endif /* H5C_COLLECT_CACHE_STATS */ - - return; - -} /* H5C_stats__reset() */ - - -/*------------------------------------------------------------------------- - * Function: H5C_dump_cache - * - * Purpose: Print a summary of the contents of the metadata cache for - * debugging purposes. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: John Mainzer - * 10/10/10 - * - *------------------------------------------------------------------------- - */ -herr_t -H5C_dump_cache(H5C_t * cache_ptr, - const char * cache_name) -{ - herr_t ret_value = SUCCEED; /* Return value */ - int i; - H5C_cache_entry_t * entry_ptr = NULL; - H5SL_t * slist_ptr = NULL; - H5SL_node_t * node_ptr = NULL; - - FUNC_ENTER_NOAPI(FAIL) - - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - HDassert(cache_name != NULL ); - - /* First, create a skip list */ - slist_ptr = H5SL_create(H5SL_TYPE_HADDR, NULL); - - if ( slist_ptr == NULL ) { - - HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, FAIL, "can't create skip list.") - } - - /* Next, scan the index, and insert all entries in the skip list. - * Do this, as we want to display cache entries in increasing address - * order. - */ - for ( i = 0; i < H5C__HASH_TABLE_LEN; i++ ) { - - entry_ptr = cache_ptr->index[i]; - - while ( entry_ptr != NULL ) { - - HDassert( entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC ); - - if ( H5SL_insert(slist_ptr, entry_ptr, &(entry_ptr->addr)) < 0 ) { - - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, \ - "Can't insert entry in skip list") - } - - entry_ptr = entry_ptr->ht_next; - } - } - - /* If we get this far, all entries in the cache are listed in the - * skip list -- scan the skip list generating the desired output. - */ - - HDfprintf(stdout, "\n\nDump of metadata cache \"%s\".\n", cache_name); - HDfprintf(stdout, - "Num: Addr: Tag: Len: Type: Prot: Pinned: Dirty: Corked:\n"); - - i = 0; - - node_ptr = H5SL_first(slist_ptr); - - if ( node_ptr != NULL ) { - - entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); - - } else { - - entry_ptr = NULL; - } - - while ( entry_ptr != NULL ) { - - HDassert( entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC ); - - HDfprintf(stdout, - "%s%d 0x%16llx 0x%3llx 0x%3llx %2d %d %d %d %d\n", - cache_ptr->prefix, i, - (long long)(entry_ptr->addr), - (long long)(entry_ptr->tag), - (long long)(entry_ptr->size), - (int)(entry_ptr->type->id), - (int)(entry_ptr->is_protected), - (int)(entry_ptr->is_pinned), - (int)(entry_ptr->is_dirty), - (int)(entry_ptr->is_corked)); - - /* increment node_ptr before we delete its target */ - node_ptr = H5SL_next(node_ptr); - - /* remove the first item in the skip list */ - if ( H5SL_remove(slist_ptr, &(entry_ptr->addr)) != entry_ptr ) { - - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, \ - "Can't delete entry from skip list.") - } - - if ( node_ptr != NULL ) { - - entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); - - } else { - - entry_ptr = NULL; - } - - i++; - } - - HDfprintf(stdout, "\n\n"); - - /* Finally, discard the skip list */ - - HDassert( H5SL_count(slist_ptr) == 0 ); - - H5SL_close(slist_ptr); - -done: - - FUNC_LEAVE_NOAPI(ret_value) - -} /* H5C_dump_cache() */ - - -/*------------------------------------------------------------------------- - * Function: H5C_dump_cache_skip_list - * - * Purpose: Debugging routine that prints a summary of the contents of - * the skip list used by the metadata cache metadata cache to - * maintain an address sorted list of dirty entries. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: John Mainzer - * 11/15/14 - * - *------------------------------------------------------------------------- - */ -#if 0 /* debugging routine */ -herr_t -H5C_dump_cache_skip_list(H5C_t * cache_ptr, char * calling_fcn) -{ - herr_t ret_value = SUCCEED; /* Return value */ - int i; - H5C_cache_entry_t * entry_ptr = NULL; - H5SL_node_t * node_ptr = NULL; - - FUNC_ENTER_NOAPI(FAIL) - - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - HDassert(calling_fcn != NULL); - - HDfprintf(stdout, "\n\nDumping metadata cache skip list from %s.\n", - calling_fcn); - HDfprintf(stdout, " slist len = %d.\n", cache_ptr->slist_len); - HDfprintf(stdout, " slist size = %lld.\n", - (long long)(cache_ptr->slist_size)); - - if ( cache_ptr->slist_len > 0 ) - { - /* If we get this far, all entries in the cache are listed in the - * skip list -- scan the skip list generating the desired output. - */ - - HDfprintf(stdout, - "Num: Addr: Len: Prot/Pind: Dirty: Type:\n"); - - i = 0; - - node_ptr = H5SL_first(cache_ptr->slist_ptr); - - if ( node_ptr != NULL ) { - - entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); - - } else { - - entry_ptr = NULL; - } - - while ( entry_ptr != NULL ) { - - HDassert( entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC ); - - HDfprintf(stdout, - "%s%d 0x%016llx %4lld %d/%d %d %s\n", - cache_ptr->prefix, i, - (long long)(entry_ptr->addr), - (long long)(entry_ptr->size), - (int)(entry_ptr->is_protected), - (int)(entry_ptr->is_pinned), - (int)(entry_ptr->is_dirty), - entry_ptr->type->name); - - HDfprintf(stdout, " node_ptr = 0x%llx, item = 0x%llx\n", - (unsigned long long)node_ptr, - (unsigned long long)H5SL_item(node_ptr)); - - /* increment node_ptr before we delete its target */ - node_ptr = H5SL_next(node_ptr); - - if ( node_ptr != NULL ) { - - entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); - - } else { - - entry_ptr = NULL; - } - - i++; - } - } - - HDfprintf(stdout, "\n\n"); - -done: - - FUNC_LEAVE_NOAPI(ret_value) - -} /* H5C_dump_cache_skip_list() */ -#endif /* debugging routine */ - - -/*------------------------------------------------------------------------- * Function: H5C_unpin_entry_from_client() * * Purpose: Internal routine to unpin a cache entry from a client action. @@ -4116,17 +3137,31 @@ H5C_unprotect(H5F_t * f, /* Check for newly dirtied entry */ if(was_clean && entry_ptr->is_dirty) { + /* Update index for newly dirtied entry */ + H5C__UPDATE_INDEX_FOR_ENTRY_DIRTY(cache_ptr, entry_ptr) + + /* If the entry's type has a 'notify' callback send a 'entry dirtied' + * notice now that the entry is fully integrated into the cache. + */ + if(entry_ptr->type->notify && + (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_DIRTIED, entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag set") + /* Propagate the flush dep dirty flag up the flush dependency chain * if appropriate */ if(entry_ptr->flush_dep_nparents > 0) if(H5C__mark_flush_dep_dirty(entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep dirty flag") - - /* Update index for newly dirtied entry */ - H5C__UPDATE_INDEX_FOR_ENTRY_DIRTY(cache_ptr, entry_ptr) } /* end if */ /* Check for newly clean entry */ else if(!was_clean && !entry_ptr->is_dirty) { + /* If the entry's type has a 'notify' callback send a 'entry cleaned' + * notice now that the entry is fully integrated into the cache. + */ + if(entry_ptr->type->notify && + (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_CLEANED, entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag cleared") + /* Propagate the flush dep clean flag up the flush dependency chain * if appropriate */ if(entry_ptr->flush_dep_nparents > 0) @@ -4174,9 +3209,6 @@ H5C_unprotect(H5F_t * f, unsigned flush_flags = (H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__FLUSH_INVALIDATE_FLAG); - /* we can't delete a pinned entry */ - HDassert ( ! (entry_ptr->is_pinned ) ); - /* verify that the target entry is in the cache. */ H5C__SEARCH_INDEX(cache_ptr, addr, test_entry_ptr, FAIL) if(test_entry_ptr == NULL) @@ -4195,21 +3227,9 @@ H5C_unprotect(H5F_t * f, /* Delete the entry from the skip list on destroy */ flush_flags |= H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG; - if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, flush_flags, NULL, NULL) < 0) + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, flush_flags) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "Can't flush entry") -#if H5C_DO_SANITY_CHECKS - if ( ( take_ownership ) && ( ! was_clean ) ) - { - /* we have just removed an entry from the skip list. Thus - * we must touch up cache_ptr->slist_len_increase and - * cache_ptr->slist_size_increase to keep from skewing - * the sanity checks on flushes. - */ - cache_ptr->slist_len_increase -= 1; - cache_ptr->slist_size_increase -= (int64_t)(entry_ptr->size); - } -#endif /* H5C_DO_SANITY_CHECKS */ } #ifdef H5_HAVE_PARALLEL else if ( clear_entry ) { @@ -4221,7 +3241,7 @@ H5C_unprotect(H5F_t * f, else if(test_entry_ptr != entry_ptr) HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "hash table contains multiple entries for addr?!?.") - if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, NULL, NULL) < 0) + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "Can't clear entry") } #endif /* H5_HAVE_PARALLEL */ @@ -4572,6 +3592,11 @@ H5C_create_flush_dependency(void * parent_thing, void * child_thing) HDassert(parent_entry->flush_dep_ndirty_children < parent_entry->flush_dep_nchildren); parent_entry->flush_dep_ndirty_children++; + + /* If the parent has a 'notify' callback, send a 'child entry dirtied' notice */ + if(parent_entry->type->notify && + (parent_entry->type->notify)(H5C_NOTIFY_ACTION_CHILD_DIRTIED, parent_entry) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify parent about child entry dirty flag set") } /* end if */ /* Post-conditions, for successful operation */ @@ -4678,6 +3703,11 @@ H5C_destroy_flush_dependency(void *parent_thing, void * child_thing) HDassert(parent_entry->flush_dep_ndirty_children > 0); parent_entry->flush_dep_ndirty_children--; + + /* If the parent has a 'notify' callback, send a 'child entry cleaned' notice */ + if(parent_entry->type->notify && + (parent_entry->type->notify)(H5C_NOTIFY_ACTION_CHILD_CLEANED, parent_entry) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify parent about child entry dirty flag reset") } /* end if */ /* Shrink or free the parent array if apporpriate */ @@ -5385,40 +4415,37 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f, next_ptr = entry_ptr->next; prev_ptr = entry_ptr->prev; - if ( prev_ptr != NULL ) { - + if(prev_ptr != NULL) prev_is_dirty = prev_ptr->is_dirty; - } - - /* dirty corked entry is skipped */ - if(entry_ptr->is_corked && entry_ptr->is_dirty) - corked = TRUE; - else if ( entry_ptr->is_dirty ) { - - /* reset entries_removed_counter and - * last_entry_removed_ptr prior to the call to - * H5C__flush_single_entry() so that we can spot - * unexpected removals of entries from the cache, - * and set the restart_scan flag if proceeding - * would be likely to cause us to scan an entry - * that is no longer in the cache. - */ - cache_ptr->entries_removed_counter = 0; - cache_ptr->last_entry_removed_ptr = NULL; - - if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__NO_FLAGS_SET, NULL, NULL) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry") - if ( ( cache_ptr->entries_removed_counter > 1 ) || - ( cache_ptr->last_entry_removed_ptr == prev_ptr ) ) + if(entry_ptr->is_dirty ) { + /* dirty corked entry is skipped */ + if(entry_ptr->tag_info && entry_ptr->tag_info->corked) + corked = TRUE; + else { + /* reset entries_removed_counter and + * last_entry_removed_ptr prior to the call to + * H5C__flush_single_entry() so that we can spot + * unexpected removals of entries from the cache, + * and set the restart_scan flag if proceeding + * would be likely to cause us to scan an entry + * that is no longer in the cache. + */ + cache_ptr->entries_removed_counter = 0; + cache_ptr->last_entry_removed_ptr = NULL; - restart_scan = TRUE; + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__NO_FLAGS_SET) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry") - } else { + if(cache_ptr->entries_removed_counter > 1 || cache_ptr->last_entry_removed_ptr == prev_ptr) + restart_scan = TRUE; + } /* end else */ + } /* end if */ + else { bytes_evicted += entry_ptr->size; - if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, NULL, NULL) < 0 ) + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0 ) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry") } @@ -5501,7 +4528,7 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f, prev_ptr = entry_ptr->prev; if ( ! (entry_ptr->is_dirty) ) { - if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, NULL, NULL) < 0) + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush clean entry") } /* just skip the entry if it is dirty, as we can't do @@ -6043,13 +5070,37 @@ H5C_flush_invalidate_cache(const H5F_t * f, hid_t dxpl_id, unsigned flags) } /* end while */ /* Invariants, after destroying all entries in the hash table */ - HDassert(cache_ptr->index_size == 0); - HDassert(cache_ptr->clean_index_size == 0); + if(!(flags & H5C__EVICT_ALLOW_LAST_PINS_FLAG)) { + HDassert(cache_ptr->index_size == 0); + HDassert(cache_ptr->clean_index_size == 0); + HDassert(cache_ptr->pel_len == 0); + HDassert(cache_ptr->pel_size == 0); + } /* end if */ + else { + H5C_cache_entry_t *entry_ptr; /* Cache entry */ + unsigned u; /* Local index variable */ + + /* All rings except ring 4 should be empty now */ + /* (Ring 4 has the superblock) */ + for(u = H5C_RING_USER; u < H5C_RING_SB; u++) { + HDassert(cache_ptr->index_ring_len[u] == 0); + HDassert(cache_ptr->index_ring_size[u] == 0); + HDassert(cache_ptr->clean_index_ring_size[u] == 0); + } /* end for */ + + /* Check that any remaining pinned entries are in the superblock ring */ + entry_ptr = cache_ptr->pel_head_ptr; + while(entry_ptr) { + /* Check ring */ + HDassert(entry_ptr->ring == H5C_RING_SB); + + /* Advance to next entry in pinned entry list */ + entry_ptr = entry_ptr->next; + } /* end while */ + } /* end else */ HDassert(cache_ptr->dirty_index_size == 0); HDassert(cache_ptr->slist_len == 0); HDassert(cache_ptr->slist_size == 0); - HDassert(cache_ptr->pel_len == 0); - HDassert(cache_ptr->pel_size == 0); HDassert(cache_ptr->pl_len == 0); HDassert(cache_ptr->pl_size == 0); HDassert(cache_ptr->LRU_list_len == 0); @@ -6112,14 +5163,8 @@ H5C_flush_invalidate_ring(const H5F_t * f, hid_t dxpl_id, H5C_ring_t ring, H5C_cache_entry_t * entry_ptr = NULL; H5C_cache_entry_t * next_entry_ptr = NULL; #if H5C_DO_SANITY_CHECKS - int64_t flushed_slist_len = 0; - int64_t initial_slist_len = 0; - int64_t flushed_slist_size = 0; + int64_t initial_slist_len = 0; size_t initial_slist_size = 0; - int64_t entry_size_change; - int64_t * entry_size_change_ptr = &entry_size_change; -#else /* H5C_DO_SANITY_CHECKS */ - int64_t * entry_size_change_ptr = NULL; #endif /* H5C_DO_SANITY_CHECKS */ herr_t ret_value = SUCCEED; @@ -6211,29 +5256,18 @@ H5C_flush_invalidate_ring(const H5F_t * f, hid_t dxpl_id, H5C_ring_t ring, */ cache_ptr->slist_len_increase = 0; cache_ptr->slist_size_increase = 0; - - /* Finally, reset the flushed_slist_len and flushed_slist_size - * fields to zero, as these fields are used to accumulate - * the slist lenght and size that we see as we scan through - * the slist. - */ - flushed_slist_len = 0; - flushed_slist_size = 0; #endif /* H5C_DO_SANITY_CHECKS */ - /* set the cache_ptr->slist_change_in_pre_serialize and - * cache_ptr->slist_change_in_serialize to false. + /* Set the cache_ptr->slist_changed to false. * - * These flags are set to TRUE by H5C__flush_single_entry if the - * slist is modified by a pre_serialize or serialize call - * respectively. + * This flag is set to TRUE by H5C__flush_single_entry if the slist + * is modified by a pre_serialize, serialize, or notify callback. * - * H5C_flush_invalidate_cache() uses these flags to detect any + * H5C_flush_invalidate_ring() uses this flag to detect any * modifications to the slist that might corrupt the scan of * the slist -- and restart the scan in this event. */ - cache_ptr->slist_change_in_pre_serialize = FALSE; - cache_ptr->slist_change_in_serialize = FALSE; + cache_ptr->slist_changed = FALSE; /* this done, start the scan of the slist */ restart_slist_scan = TRUE; @@ -6271,6 +5305,7 @@ H5C_flush_invalidate_ring(const H5F_t * f, hid_t dxpl_id, H5C_ring_t ring, * slist if we find them. However, best we do some extra * sanity checking just in case. */ + HDassert(entry_ptr != NULL); HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(entry_ptr->in_slist); HDassert(entry_ptr->is_dirty); @@ -6303,9 +5338,6 @@ H5C_flush_invalidate_ring(const H5F_t * f, hid_t dxpl_id, H5C_ring_t ring, * flush, we must keep the slist in cannonical form at all * times. */ - HDassert(entry_ptr != NULL); - HDassert(entry_ptr->in_slist); - if(((!entry_ptr->flush_me_last) || ((entry_ptr->flush_me_last) && (cache_ptr->num_last_entries >= cache_ptr->slist_len))) && @@ -6317,32 +5349,10 @@ H5C_flush_invalidate_ring(const H5F_t * f, hid_t dxpl_id, H5C_ring_t ring, */ protected_entries++; } else if(entry_ptr->is_pinned) { - -#if H5C_DO_SANITY_CHECKS - /* update flushed_slist_len & flushed_slist_size - * before the flush. Note that the entry will - * be removed from the slist after the flush, - * and thus may be resized by the flush callback. - * This is OK, as we will catch the size delta in - * cache_ptr->slist_size_increase. - * - */ - flushed_slist_len++; - flushed_slist_size += (int64_t)entry_ptr->size; - entry_size_change = 0; -#endif /* H5C_DO_SANITY_CHECKS */ - - if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__NO_FLAGS_SET, entry_size_change_ptr, NULL) < 0) + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__DURING_FLUSH_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "dirty pinned entry flush failed.") -#if H5C_DO_SANITY_CHECKS - /* entry size may have changed during the flush. - * Update flushed_slist_size to account for this. - */ - flushed_slist_size += entry_size_change; -#endif /* H5C_DO_SANITY_CHECKS */ - if((cache_ptr->slist_change_in_serialize) || - (cache_ptr->slist_change_in_pre_serialize)) { + if(cache_ptr->slist_changed) { /* The slist has been modified by something * other than the simple removal of the * of the flushed entry after the flush. @@ -6351,39 +5361,15 @@ H5C_flush_invalidate_ring(const H5F_t * f, hid_t dxpl_id, H5C_ring_t ring, * scan through the slist, so restart it. */ restart_slist_scan = TRUE; - cache_ptr->slist_change_in_pre_serialize = FALSE; - cache_ptr->slist_change_in_serialize = FALSE; + cache_ptr->slist_changed = FALSE; H5C__UPDATE_STATS_FOR_SLIST_SCAN_RESTART(cache_ptr); } /* end if */ } /* end if */ else { -#if H5C_DO_SANITY_CHECKS - /* update flushed_slist_len & flushed_slist_size - * before the flush. Note that the entry will - * be removed from the slist after the flush, - * and thus may be resized by the flush callback. - * This is OK, as we will catch the size delta in - * cache_ptr->slist_size_increase. - * - */ - flushed_slist_len++; - flushed_slist_size += (int64_t)entry_ptr->size; - entry_size_change = 0; -#endif /* H5C_DO_SANITY_CHECKS */ - - if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, - (cooked_flags | H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG), - entry_size_change_ptr, NULL) < 0) + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, (cooked_flags | H5C__DURING_FLUSH_FLAG | H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG)) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "dirty entry flush destroy failed.") -#if H5C_DO_SANITY_CHECKS - /* entry size may have changed during the flush. - * Update flushed_slist_size to account for this. - */ - flushed_slist_size += entry_size_change; -#endif /* H5C_DO_SANITY_CHECKS */ - if((cache_ptr->slist_change_in_serialize) || - (cache_ptr->slist_change_in_pre_serialize)) { + if(cache_ptr->slist_changed) { /* The slist has been modified by something * other than the simple removal of the * of the flushed entry after the flush. @@ -6392,8 +5378,7 @@ H5C_flush_invalidate_ring(const H5F_t * f, hid_t dxpl_id, H5C_ring_t ring, * scan through the slist, so restart it. */ restart_slist_scan = TRUE; - cache_ptr->slist_change_in_pre_serialize = FALSE; - cache_ptr->slist_change_in_serialize = FALSE; + cache_ptr->slist_changed = FALSE; H5C__UPDATE_STATS_FOR_SLIST_SCAN_RESTART(cache_ptr) } /* end if */ } /* end else */ @@ -6411,10 +5396,8 @@ H5C_flush_invalidate_ring(const H5F_t * f, hid_t dxpl_id, H5C_ring_t ring, */ if(node_ptr == NULL) { - HDassert((flushed_slist_len + cache_ptr->slist_len) == - (initial_slist_len + cache_ptr->slist_len_increase)); - HDassert((flushed_slist_size + (int64_t)cache_ptr->slist_size) == - ((int64_t)initial_slist_size + cache_ptr->slist_size_increase)); + HDassert(cache_ptr->slist_len == (initial_slist_len + cache_ptr->slist_len_increase)); + HDassert((int64_t)cache_ptr->slist_size == ((int64_t)initial_slist_size + cache_ptr->slist_size_increase)); } /* end if */ #endif /* H5C_DO_SANITY_CHECKS */ @@ -6425,7 +5408,7 @@ H5C_flush_invalidate_ring(const H5F_t * f, hid_t dxpl_id, H5C_ring_t ring, * It used to be that all entries remaining in the cache at * this point had to be clean, but with the fractal heap mods * this may not be the case. If so, we will flush entries out - * of increasing address order. + * in increasing address order. * * Writes to disk are possible here. */ @@ -6463,75 +5446,53 @@ H5C_flush_invalidate_ring(const H5F_t * f, hid_t dxpl_id, H5C_ring_t ring, * into the cache, or moved to a new location * in the file as a side effect of the flush. * - * If this happens, and one of the target - * entries happens to be the next entry in - * the hash bucket, we could find ourselves - * either find ourselves either scanning a - * non-existant entry, scanning through a - * different bucket, or skipping an entry. + * It's also possible that removing a clean + * entry will remove the last child of a proxy + * entry, allowing it to be removed also and + * invalidating the next_entry_ptr. + * + * If either of these happen, and one of the target + * or proxy entries happens to be the next entry in + * the hash bucket, we could either find ourselves + * either scanning a non-existant entry, scanning + * through a different bucket, or skipping an entry. * * Neither of these are good, so restart the * the scan at the head of the hash bucket - * after the flush if *entry_ptr was dirty, - * on the off chance that the next entry was - * a target. + * after the flush if we detect that the next_entry_ptr + * becomes invalid. * * This is not as inefficient at it might seem, * as hash buckets typically have at most two * or three entries. */ - hbool_t entry_was_dirty; - - entry_was_dirty = entry_ptr->is_dirty; - - if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, - (cooked_flags | H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG), - NULL, NULL) < 0) + cache_ptr->entry_watched_for_removal = next_entry_ptr; + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, (cooked_flags | H5C__DURING_FLUSH_FLAG | H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG)) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Entry flush destroy failed.") - if(entry_was_dirty) { - /* update stats for hash bucket scan - * restart here. + /* Check for the next entry getting removed */ + if(NULL != next_entry_ptr && NULL == cache_ptr->entry_watched_for_removal) { + /* update stats for hash bucket scan restart here. * -- JRM */ next_entry_ptr = cache_ptr->index[i]; H5C__UPDATE_STATS_FOR_HASH_BUCKET_SCAN_RESTART(cache_ptr) } /* end if */ + else + cache_ptr->entry_watched_for_removal = NULL; } /* end if */ } /* end if */ - - /* We can't do anything if the entry is pinned. The - * hope is that the entry will be unpinned as the - * result of destroys of entries that reference it. - * - * We detect this by noting the change in the number - * of pinned entries from pass to pass. If it stops - * shrinking before it hits zero, we scream and die. - */ - /* if the serialize function on the entry we last evicted - * loaded an entry into cache (as Quincey has promised me - * it never will), and if the cache was full, it is - * possible that *next_entry_ptr was flushed or evicted. - * - * Test to see if this happened here. Note that if this - * test is triggred, we are accessing a deallocated piece - * of dynamically allocated memory, so we just scream and - * die. - * - * Update: The code to restart the scan after flushes - * of dirty entries should make it impossible - * to satisfy the following test. Leave it in - * in case I am wrong. - * -- JRM - */ - if((next_entry_ptr != NULL) && (next_entry_ptr->magic != H5C__H5C_CACHE_ENTRY_T_MAGIC)) - /* Something horrible has happened to - * *next_entry_ptr -- scream and die. - */ - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "next_entry_ptr->magic is invalid?!?!?.") } /* end while loop scanning hash table bin */ } /* end for loop scanning hash table */ + /* We can't do anything if entries are pinned. The + * hope is that the entries will be unpinned as the + * result of destroys of entries that reference them. + * + * We detect this by noting the change in the number + * of pinned entries from pass to pass. If it stops + * shrinking before it hits zero, we scream and die. + */ old_ring_pel_len = cur_ring_pel_len; entry_ptr = cache_ptr->pel_head_ptr; cur_ring_pel_len = 0; @@ -6612,7 +5573,6 @@ herr_t H5C_flush_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring, unsigned flags) { H5C_t * cache_ptr = f->shared->cache; - hbool_t destroy; hbool_t flushed_entries_last_pass; hbool_t flush_marked_entries; hbool_t ignore_protected; @@ -6623,14 +5583,8 @@ H5C_flush_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring, unsigned flags) H5C_cache_entry_t * entry_ptr = NULL; H5C_cache_entry_t * next_entry_ptr = NULL; #if H5C_DO_SANITY_CHECKS - int64_t flushed_entries_count = 0; - int64_t flushed_entries_size = 0; int64_t initial_slist_len = 0; size_t initial_slist_size = 0; - int64_t entry_size_change; - int64_t * entry_size_change_ptr = &entry_size_change; -#else /* H5C_DO_SANITY_CHECKS */ - int64_t * entry_size_change_ptr = NULL; #endif /* H5C_DO_SANITY_CHECKS */ int i; herr_t ret_value = SUCCEED; @@ -6652,7 +5606,6 @@ H5C_flush_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring, unsigned flags) #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ ignore_protected = ( (flags & H5C__FLUSH_IGNORE_PROTECTED_FLAG) != 0 ); - destroy = ( (flags & H5C__FLUSH_INVALIDATE_FLAG) != 0 ); flush_marked_entries = ( (flags & H5C__FLUSH_MARKED_ENTRIES_FLAG) != 0 ); if(!flush_marked_entries) @@ -6668,17 +5621,15 @@ H5C_flush_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring, unsigned flags) */ flushed_entries_last_pass = TRUE; - /* set the cache_ptr->slist_change_in_pre_serialize and - * cache_ptr->slist_change_in_serialize to false. + /* Set the cache_ptr->slist_changed to false. * - * These flags are set to TRUE by H5C__flush_single_entry if the - * slist is modified by a pre_serialize or serialize call respectively. - * H5C_flush_cache uses these flags to detect any modifications + * This flag is set to TRUE by H5C__flush_single_entry if the + * slist is modified by a pre_serialize, serialize, or notify callback. + * H5C_flush_cache uses this flag to detect any modifications * to the slist that might corrupt the scan of the slist -- and * restart the scan in this event. */ - cache_ptr->slist_change_in_pre_serialize = FALSE; - cache_ptr->slist_change_in_serialize = FALSE; + cache_ptr->slist_changed = FALSE; while((cache_ptr->slist_ring_len[ring] > 0) && (protected_entries == 0) && @@ -6705,12 +5656,6 @@ H5C_flush_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring, unsigned flags) initial_slist_len = cache_ptr->slist_len; initial_slist_size = cache_ptr->slist_size; - /* We then zero counters that we use to track the number - * and total size of entries flushed: - */ - flushed_entries_count = 0; - flushed_entries_size = 0; - /* As mentioned above, there is the possibility that * entries will be dirtied, resized, flushed, or removed * from the cache via the take ownership flag during @@ -6779,7 +5724,7 @@ H5C_flush_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring, unsigned flags) if(!flush_marked_entries || entry_ptr->flush_marker) HDassert(entry_ptr->ring >= ring); - /* increment node pointer now, before we delete its target + /* advance node pointer now, before we delete its target * from the slist. */ node_ptr = H5SL_next(node_ptr); @@ -6788,28 +5733,18 @@ H5C_flush_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring, unsigned flags) if(NULL == next_entry_ptr) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "next_entry_ptr == NULL ?!?!") - HDassert(next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); - HDassert(next_entry_ptr->is_dirty); - HDassert(next_entry_ptr->in_slist); - if(!flush_marked_entries || next_entry_ptr->flush_marker) - HDassert(next_entry_ptr->ring >= ring); - HDassert(entry_ptr != next_entry_ptr); } /* end if */ else next_entry_ptr = NULL; - HDassert(entry_ptr != NULL); - HDassert(entry_ptr->in_slist); - if(((!flush_marked_entries) || (entry_ptr->flush_marker)) && ((!entry_ptr->flush_me_last) || (entry_ptr->flush_me_last && ((cache_ptr->num_last_entries >= cache_ptr->slist_len) || (flush_marked_entries && entry_ptr->flush_marker)))) && ( ( entry_ptr->flush_dep_nchildren == 0 ) || - ( ( ! destroy ) && - ( entry_ptr->flush_dep_ndirty_children == 0 ) ) ) && + ( entry_ptr->flush_dep_ndirty_children == 0 ) ) && (entry_ptr->ring == ring)) { if(entry_ptr->is_protected) { /* we probably have major problems -- but lets @@ -6820,24 +5755,10 @@ H5C_flush_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring, unsigned flags) protected_entries++; } /* end if */ else { -#if H5C_DO_SANITY_CHECKS - flushed_entries_count++; - flushed_entries_size += (int64_t)entry_ptr->size; - entry_size_change = 0; -#endif /* H5C_DO_SANITY_CHECKS */ - if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, flags, entry_size_change_ptr, NULL) < 0) + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, (flags | H5C__DURING_FLUSH_FLAG)) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush entry.") -#if H5C_DO_SANITY_CHECKS - /* it is possible that the entry size changed - * during flush -- update flushed_entries_size - * to account for this. - */ - flushed_entries_size += entry_size_change; -#endif /* H5C_DO_SANITY_CHECKS */ - - if((cache_ptr->slist_change_in_serialize) || - (cache_ptr->slist_change_in_pre_serialize)) { + if(cache_ptr->slist_changed) { /* The slist has been modified by something * other than the simple removal of the * of the flushed entry after the flush. @@ -6846,9 +5767,7 @@ H5C_flush_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring, unsigned flags) * scan through the slist, so restart it. */ restart_slist_scan = TRUE; - cache_ptr->slist_change_in_pre_serialize = FALSE; - cache_ptr->slist_change_in_serialize = FALSE; - + cache_ptr->slist_changed = FALSE; H5C__UPDATE_STATS_FOR_SLIST_SCAN_RESTART(cache_ptr) } /* end if */ @@ -6859,11 +5778,8 @@ H5C_flush_ring(H5F_t *f, hid_t dxpl_id, H5C_ring_t ring, unsigned flags) #if H5C_DO_SANITY_CHECKS /* Verify that the slist size and length are as expected. */ - HDassert((initial_slist_len + cache_ptr->slist_len_increase - - flushed_entries_count) == cache_ptr->slist_len); - HDassert((size_t)((int64_t)initial_slist_size + - cache_ptr->slist_size_increase - - flushed_entries_size) == cache_ptr->slist_size); + HDassert((initial_slist_len + cache_ptr->slist_len_increase) == cache_ptr->slist_len); + HDassert((size_t)((int64_t)initial_slist_size + cache_ptr->slist_size_increase) == cache_ptr->slist_size); #endif /* H5C_DO_SANITY_CHECKS */ } /* while */ @@ -6947,11 +5863,7 @@ done: */ herr_t H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, - unsigned flags, int64_t *entry_size_change_ptr, H5SL_t -#ifndef H5_HAVE_PARALLEL - H5_ATTR_UNUSED -#endif /* NDEBUG */ - *collective_write_list) + unsigned flags) { H5C_t * cache_ptr; /* Cache for file */ hbool_t destroy; /* external flag */ @@ -6959,8 +5871,10 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ hbool_t free_file_space; /* external flag */ hbool_t take_ownership; /* external flag */ hbool_t del_from_slist_on_destroy; /* external flag */ + hbool_t during_flush; /* external flag */ hbool_t write_entry; /* internal flag */ hbool_t destroy_entry; /* internal flag */ + hbool_t generate_image; /* internal flag */ hbool_t was_dirty; haddr_t entry_addr = HADDR_UNDEF; herr_t ret_value = SUCCEED; /* Return value */ @@ -6974,16 +5888,14 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ HDassert(entry_ptr); HDassert(entry_ptr->ring != H5C_RING_UNDEFINED); - /* If defined, initialize *entry_size_change_ptr to 0 */ - if(entry_size_change_ptr != NULL) - *entry_size_change_ptr = 0; - /* setup external flags from the flags parameter */ destroy = ((flags & H5C__FLUSH_INVALIDATE_FLAG) != 0); clear_only = ((flags & H5C__FLUSH_CLEAR_ONLY_FLAG) != 0); free_file_space = ((flags & H5C__FREE_FILE_SPACE_FLAG) != 0); take_ownership = ((flags & H5C__TAKE_OWNERSHIP_FLAG) != 0); del_from_slist_on_destroy = ((flags & H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) != 0); + during_flush = ((flags & H5C__DURING_FLUSH_FLAG) != 0); + generate_image = ((flags & H5C__GENERATE_IMAGE_FLAG) != 0); /* Set the flag for destroying the entry, based on the 'take ownership' * and 'destroy' flags @@ -7007,8 +5919,6 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ /* run initial sanity checks */ #if H5C_DO_SANITY_CHECKS - HDassert( ! ( destroy && entry_ptr->is_pinned ) ); - if(entry_ptr->in_slist) { HDassert(entry_ptr->is_dirty); @@ -7045,102 +5955,59 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ entry_ptr->flush_in_progress = TRUE; entry_ptr->flush_marker = FALSE; - /* serialize the entry if necessary, and then write it to disk. */ - if(write_entry) { - - /* The entry is dirty, and we are doing either a flush, - * or a flush destroy. In either case, serialize the - * entry and write it to disk. - * - * Note that this may cause the entry to be re-sized and/or - * moved in the cache. - * - * As we will not update the metadata cache's data structures - * until we we finish the write, we must touch up these - * data structures for size and location changes even if we - * are about to delete the entry from the cache (i.e. on a - * flush destroy). - */ + /* The entry is dirty, and we are doing a flush, a flush destroy or have + * been requested to generate an image. In those cases, serialize the + * entry. + */ + if(write_entry || generate_image) { HDassert(entry_ptr->is_dirty); -#if H5C_DO_SANITY_CHECKS - if(cache_ptr->check_write_permitted && !(cache_ptr->write_permitted)) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Write when writes are always forbidden!?!?!") -#endif /* H5C_DO_SANITY_CHECKS */ - if(NULL == entry_ptr->image_ptr) { - size_t image_size; - - if(entry_ptr->compressed) - image_size = entry_ptr->compressed_size; - else - image_size = entry_ptr->size; - HDassert(image_size > 0); - - - if(NULL == (entry_ptr->image_ptr = H5MM_malloc(image_size + H5C_IMAGE_EXTRA_SPACE))) + if(NULL == (entry_ptr->image_ptr = H5MM_malloc(entry_ptr->size + H5C_IMAGE_EXTRA_SPACE))) HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for on disk image buffer") #if H5C_DO_MEMORY_SANITY_CHECKS - HDmemcpy(((uint8_t *)entry_ptr->image_ptr) + image_size, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); + HDmemcpy(((uint8_t *)entry_ptr->image_ptr) + entry_ptr->size, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); #endif /* H5C_DO_MEMORY_SANITY_CHECKS */ } /* end if */ if(!(entry_ptr->image_up_to_date)) { /* Generate the entry's image */ - if(H5C__generate_image(f, cache_ptr, entry_ptr, dxpl_id, entry_size_change_ptr) < 0) + if(H5C__generate_image(f, cache_ptr, entry_ptr, dxpl_id) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "can't generate entry's image") } /* end if ( ! (entry_ptr->image_up_to_date) ) */ + } /* end if */ - /* Finally, write the image to disk. - * - * Note that if either the H5C__CLASS_NO_IO_FLAG or the - * the H5AC__CLASS_SKIP_WRITES flag is set in the - * in the entry's type, we silently skip the write. This - * flag should only be used in test code. - */ - if ( ( ((entry_ptr->type->flags) & H5C__CLASS_NO_IO_FLAG) == 0 ) && - ( ((entry_ptr->type->flags) & H5C__CLASS_SKIP_WRITES) == 0 ) ) - { - /* If compression is not enabled, the size of the entry on - * disk is entry_prt->size. However if entry_ptr->compressed - * is TRUE, the on disk size is entry_ptr->compressed_size. - */ - size_t image_size; + /* Finally, write the image to disk. + * + * Note that if the H5AC__CLASS_SKIP_WRITES flag is set in the + * in the entry's type, we silently skip the write. This + * flag should only be used in test code. + */ + if(write_entry) { + HDassert(entry_ptr->is_dirty); - if(entry_ptr->compressed) - image_size = entry_ptr->compressed_size; - else - image_size = entry_ptr->size; +#if H5C_DO_SANITY_CHECKS + if(cache_ptr->check_write_permitted && !(cache_ptr->write_permitted)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Write when writes are always forbidden!?!?!") +#endif /* H5C_DO_SANITY_CHECKS */ + if(((entry_ptr->type->flags) & H5C__CLASS_SKIP_WRITES) == 0) { #ifdef H5_HAVE_PARALLEL - if(collective_write_list) { - H5C_collective_write_t *item; - - if(NULL == (item = (H5C_collective_write_t *)H5FL_MALLOC(H5C_collective_write_t))) - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "unable to allocate skip list item") - - item->length = image_size; - item->free_buf = FALSE; - item->buf = entry_ptr->image_ptr; - item->offset = entry_ptr->addr; - - if(H5SL_insert(collective_write_list, item, &item->offset) < 0) { - H5MM_free(item); + if(cache_ptr->coll_write_list) { + if(H5SL_insert(cache_ptr->coll_write_list, entry_ptr, &entry_ptr->addr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "unable to insert skip list item") - } /* end if */ } /* end if */ else #endif /* H5_HAVE_PARALLEL */ - if(H5F_block_write(f, entry_ptr->type->mem_type, entry_ptr->addr, - image_size, dxpl_id, entry_ptr->image_ptr) < 0) + if(H5F_block_write(f, entry_ptr->type->mem_type, entry_ptr->addr, entry_ptr->size, dxpl_id, entry_ptr->image_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't write image to file.") - } + } /* end if */ /* if the entry has a notify callback, notify it that we have * just flushed the entry. */ if(entry_ptr->type->notify && - (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_AFTER_FLUSH, entry_ptr) < 0 ) + (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_AFTER_FLUSH, entry_ptr) < 0 ) HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client of entry flush") } /* if ( write_entry ) */ @@ -7165,25 +6032,30 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ H5C__UPDATE_STATS_FOR_FLUSH(cache_ptr, entry_ptr) } /* end else if */ + /* Note that the algorithm below is (very) similar to the set of operations + * in H5C_remove_entry() and should be kept in sync with changes + * to that code. - QAK, 2016/11/30 + */ + + /* Update the cache internal data structures. */ if(destroy) { + /* Sanity checks */ if(take_ownership) HDassert(!destroy_entry); else HDassert(destroy_entry); + HDassert(!entry_ptr->is_pinned); + /* Update stats, while entry is still in the cache */ H5C__UPDATE_STATS_FOR_EVICTION(cache_ptr, entry_ptr, take_ownership) - } /* end if */ - /* If the entry's type has a 'notify' callback and the entry is about - * to be removed from the cache, send a 'before eviction' notice while - * the entry is still fully integrated in the cache. - */ - if(destroy) + /* If the entry's type has a 'notify' callback and the entry is about + * to be removed from the cache, send a 'before eviction' notice while + * the entry is still fully integrated in the cache. + */ if(entry_ptr->type->notify && (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_BEFORE_EVICT, entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry to evict") - /* Update the cache internal data structures. */ - if(destroy) { /* Update the cache internal data structures as appropriate * for a destroy. Specifically: * @@ -7193,6 +6065,8 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ * * 3) Update the replacement policy for eviction * + * 4) Remove it from the tag list for this object + * * Finally, if the destroy_entry flag is set, discard the * entry. */ @@ -7200,14 +6074,18 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr) if(entry_ptr->in_slist && del_from_slist_on_destroy) - H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr) + H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush) H5C__UPDATE_RP_FOR_EVICTION(cache_ptr, entry_ptr, FAIL) + /* Remove entry from tag list */ + if(H5C__untag_entry(cache_ptr, entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove entry from tag list") + /* verify that the entry is no longer part of any flush dependencies */ HDassert(entry_ptr->flush_dep_nparents == 0); HDassert(entry_ptr->flush_dep_nchildren == 0); - } + } /* end if */ else { HDassert(clear_only || write_entry); HDassert(entry_ptr->is_dirty); @@ -7224,7 +6102,7 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ H5C__UPDATE_RP_FOR_FLUSH(cache_ptr, entry_ptr, FAIL) - H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr) + H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush) /* mark the entry as clean and update the index for * entry clean. Also, call the clear callback @@ -7234,19 +6112,22 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ H5C__UPDATE_INDEX_FOR_ENTRY_CLEAN(cache_ptr, entry_ptr); - if(entry_ptr->type->clear && (entry_ptr->type->clear)(f, (void *)entry_ptr, FALSE) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to clear entry") - - /* Propagate the clean flag up the flush dependency chain if - * appropriate */ + /* Check for entry changing status and do notifications, etc. */ if(was_dirty) { - HDassert(entry_ptr->flush_dep_ndirty_children == 0); + /* If the entry's type has a 'notify' callback send a 'entry cleaned' + * notice now that the entry is fully integrated into the cache. + */ + if(entry_ptr->type->notify && + (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_CLEANED, entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag cleared") + /* Propagate the clean flag up the flush dependency chain if appropriate */ + HDassert(entry_ptr->flush_dep_ndirty_children == 0); if(entry_ptr->flush_dep_nparents > 0) if(H5C__mark_flush_dep_clean(entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep clean flag") } /* end if */ - } + } /* end else */ /* reset the flush_in progress flag */ entry_ptr->flush_in_progress = FALSE; @@ -7261,7 +6142,10 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ * Now discard the entry if appropriate. */ if(destroy) { - /* start by freeing the buffer for the on disk image */ + /* Sanity check */ + HDassert(0 == entry_ptr->flush_dep_nparents); + + /* Start by freeing the buffer for the on disk image */ if(entry_ptr->image_ptr != NULL) entry_ptr->image_ptr = H5MM_xfree(entry_ptr->image_ptr); @@ -7275,31 +6159,23 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ HDassert(H5F_addr_defined(entry_ptr->addr)); HDassert(!H5F_IS_TMP_ADDR(f, entry_ptr->addr)); #ifndef NDEBUG -{ - hbool_t curr_compressed = FALSE; - size_t curr_len; - size_t curr_compressed_len = 0; - - /* Get the actual image size for the thing again */ - entry_ptr->type->image_len((void *)entry_ptr, &curr_len, &curr_compressed, &curr_compressed_len); - HDassert(curr_len == entry_ptr->size); - HDassert(curr_compressed == entry_ptr->compressed); - HDassert(curr_compressed_len == entry_ptr->compressed_size); -} + { + size_t curr_len; + + /* Get the actual image size for the thing again */ + entry_ptr->type->image_len((void *)entry_ptr, &curr_len); + HDassert(curr_len == entry_ptr->size); + } #endif /* NDEBUG */ - /* if the file space free size callback is defined, use + /* If the file space free size callback is defined, use * it to get the size of the block of file space to free. - * Otherwise use entry_ptr->compressed_size if - * entry_ptr->compressed == TRUE, and entry_ptr->size - * if entry_ptr->compressed == FALSE. + * Otherwise use entry_ptr->size. */ if(entry_ptr->type->fsf_size) { if((entry_ptr->type->fsf_size)((void *)entry_ptr, &fsf_size) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "unable to get file space free size") } /* end if */ - else if(entry_ptr->compressed) /* use compressed size */ - fsf_size = entry_ptr->compressed_size; else /* no file space free size callback -- use entry size */ fsf_size = entry_ptr->size; @@ -7321,26 +6197,30 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ * unexpected removal of entries (via expunge, eviction, * or take ownership at present), so that they can re-start * their scans if necessary. + * + * Also check if the entry we are watching for removal is being + * removed (usually the 'next' entry for an iteration) and reset + * it to indicate that it was removed. */ - cache_ptr->last_entry_removed_ptr++; + cache_ptr->entries_removed_counter++; cache_ptr->last_entry_removed_ptr = entry_ptr; + if(entry_ptr == cache_ptr->entry_watched_for_removal) + cache_ptr->entry_watched_for_removal = NULL; /* Check for actually destroying the entry in memory */ /* (As opposed to taking ownership of it) */ if(destroy_entry) { - /* if the entry is dirty and it has a clear callback, - * call this callback now. Since this callback exists, - * it follows tht the client maintains its own dirty bits, - * which must be cleared before the entry is freed to avoid - * sanity check failures. Also clear the dirty flag for - * the same reason. - */ if(entry_ptr->is_dirty) { + /* Reset dirty flag */ entry_ptr->is_dirty = FALSE; - if(entry_ptr->type->clear && (entry_ptr->type->clear)(f, (void *)entry_ptr, TRUE) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to clear entry") - } + /* If the entry's type has a 'notify' callback send a 'entry cleaned' + * notice now that the entry is fully integrated into the cache. + */ + if(entry_ptr->type->notify && + (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_CLEANED, entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag cleared") + } /* end if */ /* we are about to discard the in core representation -- * set the magic field to bad magic so we can detect a @@ -7353,7 +6233,7 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ if(entry_ptr->type->free_icr((void *)entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "free_icr callback failed.") - } + } /* end if */ else { HDassert(take_ownership); @@ -7362,7 +6242,7 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ * unless the entry is re-inserted properly */ entry_ptr->magic = H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC; - } + } /* end else */ } /* if (destroy) */ if(cache_ptr->log_flush) @@ -7382,6 +6262,70 @@ done: /*------------------------------------------------------------------------- * + * Function: H5C__verify_len_eoa + * + * Purpose: Verify that 'len' does not exceed eoa when 'actual' is + * false i.e. 'len" is the initial speculative length from + * get_load_size callback with null image pointer. + * If exceed, adjust 'len' accordingly. + * + * Verify that 'len' should not exceed eoa when 'actual' is + * true i.e. 'len' is the actual length from get_load_size + * callback with non-null image pointer. + * If exceed, return error. + * + * Return: FAIL if error is detected, SUCCEED otherwise. + * + * Programmer: Vailin Choi + * 9/6/15 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5C__verify_len_eoa(H5F_t *f, const H5C_class_t *type, haddr_t addr, + size_t *len, hbool_t actual) +{ + H5FD_mem_t cooked_type; /* Modified type, accounting for switching global heaps */ + haddr_t eoa; /* End-of-allocation in the file */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* if type == H5FD_MEM_GHEAP, H5F_block_read() forces + * type to H5FD_MEM_DRAW via its call to H5F__accum_read(). + * Thus we do the same for purposes of computing the EOA + * for sanity checks. + */ + cooked_type = (type->mem_type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type->mem_type; + + /* Get the file's end-of-allocation value */ + eoa = H5F_get_eoa(f, cooked_type); + if(!H5F_addr_defined(eoa)) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "invalid EOA address for file") + + /* Check for bad address in general */ + if(H5F_addr_gt(addr, eoa)) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "address of object past end of allocation") + + /* Check if the amount of data to read will be past the EOA */ + if(H5F_addr_gt((addr + *len), eoa)) { + if(actual) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "actual len exceeds EOA.") + else + /* Trim down the length of the metadata */ + *len = (size_t)(eoa - addr); + } /* end if */ + + if(*len <= 0) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "len not positive after adjustment for EOA.") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C__verify_len_eoa() */ + + +/*------------------------------------------------------------------------- + * * Function: H5C_load_entry * * Purpose: Attempt to load the entry at the specified disk address @@ -7409,15 +6353,7 @@ H5C_load_entry(H5F_t * f, void * udata) { hbool_t dirty = FALSE; /* Flag indicating whether thing was dirtied during deserialize */ - hbool_t compressed = FALSE; /* flag indicating whether thing */ - /* will be run through filters on */ - /* on read and write. Usually FALSE */ - /* set to true if appropriate. */ - size_t compressed_size = 0; /* entry compressed size if */ - /* known -- otherwise uncompressed. */ - /* Zero indicates compression not */ - /* enabled. */ - void * image = NULL; /* Buffer for disk image */ + uint8_t * image = NULL; /* Buffer for disk image */ void * thing = NULL; /* Pointer to thing loaded */ H5C_cache_entry_t *entry = NULL; /* Alias for thing loaded, as cache entry */ size_t len; /* Size of image in file */ @@ -7430,79 +6366,38 @@ H5C_load_entry(H5F_t * f, FUNC_ENTER_NOAPI_NOINIT + /* Sanity checks */ HDassert(f); HDassert(f->shared); HDassert(f->shared->cache); HDassert(type); - - /* verify absence of prohibited or unsupported type flag combinations */ - HDassert(!(type->flags & H5C__CLASS_NO_IO_FLAG)); - - /* for now, we do not combine the speculative load and compressed flags */ - HDassert(!((type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG) && - (type->flags & H5C__CLASS_COMPRESSED_FLAG))); + HDassert(H5F_addr_defined(addr)); + HDassert(type->get_initial_load_size); + if(type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG) + HDassert(type->get_final_load_size); + else + HDassert(NULL == type->get_final_load_size); + HDassert(type->deserialize); /* Can't see how skip reads could be usefully combined with - * either the speculative read or compressed flags. Hence disallow. + * the speculative read flag. Hence disallow. */ HDassert(!((type->flags & H5C__CLASS_SKIP_READS) && (type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG))); - HDassert(!((type->flags & H5C__CLASS_SKIP_READS) && - (type->flags & H5C__CLASS_COMPRESSED_FLAG))); - HDassert(H5F_addr_defined(addr)); - HDassert(type->get_load_size); - HDassert(type->deserialize); - - /* Call the get_load_size callback, to retrieve the initial - * size of image - */ - if(type->get_load_size(udata, &len) < 0) + /* Call the get_initial_load_size callback, to retrieve the initial size of image */ + if(type->get_initial_load_size(udata, &len) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, NULL, "can't retrieve image size") - HDassert(len > 0); /* Check for possible speculative read off the end of the file */ - if(type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG) { - - haddr_t eoa; /* End-of-allocation in the file */ - H5FD_mem_t cooked_type; - - /* if type == H5FD_MEM_GHEAP, H5F_block_read() forces - * type to H5FD_MEM_DRAW via its call to H5F__accum_read(). - * Thus we do the same for purposes of computing the eoa - * for sanity checks. - */ - cooked_type = (type->mem_type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type->mem_type; - - /* Get the file's end-of-allocation value */ - eoa = H5F_get_eoa(f, cooked_type); - - HDassert(H5F_addr_defined(eoa)); - - /* Check for bad address in general */ - if ( H5F_addr_gt(addr, eoa) ) - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "address of object past end of allocation") - - /* Check if the amount of data to read will be past the eoa */ - if( H5F_addr_gt((addr + len), eoa) ) { - - /* Trim down the length of the metadata */ - /* Note that for some cache clients, this will cause an - * assertion failure. JRM -- 8/29/14 - */ - len = (size_t)(eoa - addr); - } /* end if */ - - if ( len <= 0 ) - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "len not positive after adjustment for EOA.") - - } /* end if */ + if(type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG) + if(H5C__verify_len_eoa(f, type, addr, &len, FALSE) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "invalid len with respect to EOA") /* Allocate the buffer for reading the on-disk entry image */ - if(NULL == (image = H5MM_malloc(len + H5C_IMAGE_EXTRA_SPACE))) + if(NULL == (image = (uint8_t *)H5MM_malloc(len + H5C_IMAGE_EXTRA_SPACE))) HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, NULL, "memory allocation failed for on disk image buffer.") - #if H5C_DO_MEMORY_SANITY_CHECKS HDmemcpy(image + len, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); #endif /* H5C_DO_MEMORY_SANITY_CHECKS */ @@ -7518,194 +6413,144 @@ H5C_load_entry(H5F_t * f, /* Get the on-disk entry image */ if(0 == (type->flags & H5C__CLASS_SKIP_READS)) { + unsigned tries, max_tries; /* The # of read attempts */ + unsigned retries; /* The # of retries */ + htri_t chk_ret; /* return from verify_chksum callback */ + size_t actual_len = len; /* The actual length, after speculative reads have been resolved */ + uint64_t nanosec = 1; /* # of nanoseconds to sleep between retries */ + void *new_image; /* Pointer to image */ + hbool_t len_changed = TRUE; /* Whether to re-check speculative entries */ + + /* Get the # of read attempts */ + max_tries = tries = H5F_GET_READ_ATTEMPTS(f); + + /* + * This do/while loop performs the following till the metadata checksum + * is correct or the file's number of allowed read attempts are reached. + * --read the metadata + * --determine the actual size of the metadata + * --perform checksum verification + */ + do { + if(actual_len != len) { + if(NULL == (new_image = H5MM_realloc(image, len + H5C_IMAGE_EXTRA_SPACE))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, NULL, "image null after H5MM_realloc()") + image = (uint8_t *)new_image; +#if H5C_DO_MEMORY_SANITY_CHECKS + HDmemcpy(image + len, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); +#endif /* H5C_DO_MEMORY_SANITY_CHECKS */ + } /* end if */ + #ifdef H5_HAVE_PARALLEL - if(!coll_access || 0 == mpi_rank) { + if(!coll_access || 0 == mpi_rank) { #endif /* H5_HAVE_PARALLEL */ - - if(H5F_block_read(f, type->mem_type, addr, len, dxpl_id, image) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_READERROR, NULL, "Can't read image*") - + if(H5F_block_read(f, type->mem_type, addr, len, dxpl_id, image) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_READERROR, NULL, "Can't read image*") #ifdef H5_HAVE_PARALLEL - } /* end if */ - - /* if the collective metadata read optimization is turned on, - * bcast the metadata read from process 0 to all ranks in the file - * communicator - */ - if(coll_access) { - int buf_size; + } /* end if */ + /* if the collective metadata read optimization is turned on, + * bcast the metadata read from process 0 to all ranks in the file + * communicator + */ + if(coll_access) { + int buf_size; - H5_CHECKED_ASSIGN(buf_size, int, len, size_t); - if(MPI_SUCCESS != (mpi_code = MPI_Bcast(image, buf_size, MPI_BYTE, 0, comm))) - HMPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mpi_code) - } /* end if */ + H5_CHECKED_ASSIGN(buf_size, int, len, size_t); + if(MPI_SUCCESS != (mpi_code = MPI_Bcast(image, buf_size, MPI_BYTE, 0, comm))) + HMPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mpi_code) + } /* end if */ #endif /* H5_HAVE_PARALLEL */ - } /* end if */ - - /* Deserialize the on-disk image into the native memory form */ - if(NULL == (thing = type->deserialize(image, len, udata, &dirty))) - HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "Can't deserialize image") - - /* If the client's cache has an image_len callback, check it */ - if(type->image_len) { - size_t new_len; /* New size of on-disk image */ - - /* set magic and type field in *entry_ptr. While the image_len - * callback shouldn't touch the cache specific fields, it may check - * these fields to ensure that it it has received the expected - * value. - * - * Note that this initialization is repeated below on the off - * chance that we had to re-try the deserialization. - */ - entry = (H5C_cache_entry_t *)thing; - entry->magic = H5C__H5C_CACHE_ENTRY_T_MAGIC; - entry->type = type; - - /* verify that compressed and compressed_len are initialized */ - HDassert(compressed == FALSE); - HDassert(compressed_size == 0); - - /* Get the actual image size for the thing */ - if(type->image_len(thing, &new_len, &compressed, &compressed_size) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, NULL, "can't retrieve image length") - - if(new_len == 0) - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "image length is 0") - - HDassert(((type->flags & H5C__CLASS_COMPRESSED_FLAG) != 0) || - ((compressed == FALSE) && (compressed_size == 0))); - HDassert((compressed == TRUE) || (compressed_size == 0)); - - if(new_len != len) { - - if(type->flags & H5C__CLASS_COMPRESSED_FLAG) { - - /* if new_len != len, then compression must be - * enabled on the entry. In this case, the image_len - * callback should have set compressed to TRUE, set - * new_len equal to the uncompressed size of the - * entry, and compressed_len equal to the compressed - * size -- which must equal len. - * - * We can't verify the uncompressed size, but we can - * verify the rest with the following assertions. - */ - HDassert(compressed); - HDassert(compressed_size == len); - - /* new_len should contain the uncompressed size. Set len - * equal to new_len, so that the cache will use the - * uncompressed size for purposes of space allocation, etc. - */ - len = new_len; - - } else if (type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG) { - - void *new_image; /* Buffer for disk image */ - - /* compressed must be FALSE, and compressed_size - * must be zero. - */ - HDassert(!compressed); - HDassert(compressed_size == 0); - - /* Adjust the size of the image to match new_len */ - if(NULL == (new_image = H5MM_realloc(image, - new_len + H5C_IMAGE_EXTRA_SPACE))) - - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, NULL, "image null after H5MM_realloc()") - - image = new_image; + /* If the entry could be read speculatively and the length is still + * changing, check for updating the actual size + */ + if((type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG) && len_changed) { + /* Retrieve the actual length */ + actual_len = len; + if(type->get_final_load_size(image, len, udata, &actual_len) < 0) + continue; /* Transfer control to while() and count towards retries */ + + /* Check for the length changing */ + if(actual_len != len) { + /* Verify that the length isn't past the EOA for the file */ + if(H5C__verify_len_eoa(f, type, addr, &actual_len, TRUE) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "actual_len exceeds EOA.") + + /* Expand buffer to new size */ + if(NULL == (new_image = H5MM_realloc(image, actual_len + H5C_IMAGE_EXTRA_SPACE))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, NULL, "image null after H5MM_realloc()") + image = (uint8_t *)new_image; #if H5C_DO_MEMORY_SANITY_CHECKS - HDmemcpy(((uint8_t *)image) + new_len, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); + HDmemcpy(image + actual_len, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); #endif /* H5C_DO_MEMORY_SANITY_CHECKS */ - /* If the thing's image needs to be bigger for a speculatively - * loaded thing, free the thing and retry with new length - */ - if (new_len > len) { - - /* Release previous (possibly partially initialized) - * thing. Note that we must set entry->magic to - * H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC and set one or - * two other fields before the call to free_icr - * so as to avoid sanity check failures. - */ - entry->magic = H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC; - - entry->addr = addr; - - if ( type->free_icr(thing) < 0 ) - - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, NULL, "free_icr callback failed") - + if(actual_len > len) { #ifdef H5_HAVE_PARALLEL - if(!coll_access || 0 == mpi_rank) { + if(!coll_access || 0 == mpi_rank) { #endif /* H5_HAVE_PARALLEL */ - - /* Go get the on-disk image again */ - if(H5F_block_read(f, type->mem_type, addr, - new_len, dxpl_id, image) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "Can't read image") - + /* If the thing's image needs to be bigger for a speculatively + * loaded thing, go get the on-disk image again (the extra portion). + */ + if(H5F_block_read(f, type->mem_type, addr + len, actual_len - len, dxpl_id, image + len) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "can't read image") #ifdef H5_HAVE_PARALLEL - } - /* if the collective metadata read optimization is turned on, - bcast the metadata read from process 0 to all ranks in the file - communicator */ - if(coll_access) { - int buf_size; - - H5_CHECKED_ASSIGN(buf_size, int, new_len, size_t); - if(MPI_SUCCESS != (mpi_code = MPI_Bcast(image, buf_size, MPI_BYTE, 0, comm))) - HMPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mpi_code) - } /* end if */ + } + /* If the collective metadata read optimization is turned on, + * Bcast the metadata read from process 0 to all ranks in the file + * communicator */ + if(coll_access) { + int buf_size; + + H5_CHECKED_ASSIGN(buf_size, int, actual_len - len, size_t); + if(MPI_SUCCESS != (mpi_code = MPI_Bcast(image + len, buf_size, MPI_BYTE, 0, comm))) + HMPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mpi_code) + } /* end if */ #endif /* H5_HAVE_PARALLEL */ + } /* end if */ + } /* end if (actual_len != len) */ + else { + /* The length has stabilized */ + len_changed = FALSE; - /* Deserialize on-disk image into native memory form again */ - if(NULL == (thing = type->deserialize(image, new_len, udata, &dirty))) - HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "Can't deserialize image") + /* Set the final length */ + len = actual_len; + } /* else */ + } /* end if */ -#ifndef NDEBUG - { - /* new_compressed and new_compressed_size must be - * initialize to FALSE / 0 respectively, as clients - * that don't use compression may ignore these two - * parameters. - */ - hbool_t new_compressed = FALSE; - size_t new_compressed_size = 0; - size_t new_new_len; - - /* Get the actual image size for the thing again. Note - * that since this is a new thing, we have to set - * the magic and type fields again so as to avoid - * failing sanity checks. - */ - entry = (H5C_cache_entry_t *)thing; - entry->magic = H5C__H5C_CACHE_ENTRY_T_MAGIC; - entry->type = type; - - type->image_len(thing, &new_new_len, &new_compressed, &new_compressed_size); - HDassert(new_new_len == new_len); - HDassert(!new_compressed); - HDassert(new_compressed_size == 0); - } /* end block */ -#endif /* NDEBUG */ - } /* end if (new_len > len) */ + /* If there's no way to verify the checksum for a piece of metadata + * (usually because there's no checksum in the file), leave now + */ + if(type->verify_chksum == NULL) + break; - /* Retain adjusted size */ - len = new_len; + /* Verify the checksum for the metadata image */ + if((chk_ret = type->verify_chksum(image, actual_len, udata)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, NULL, "failure from verify_chksum callback") + if(chk_ret == TRUE) + break; - } else { /* throw an error */ + /* Sleep for some time */ + H5_nanosleep(nanosec); + nanosec *= 2; /* Double the sleep time next time */ + } while(--tries); - HGOTO_ERROR(H5E_CACHE, H5E_UNSUPPORTED, NULL, \ - "size of non-speculative, non-compressed object changed") - } - } /* end if (new_len != len) */ - } /* end if */ + /* Check for too many tries */ + if(tries == 0) + HGOTO_ERROR(H5E_CACHE, H5E_READERROR, NULL, "incorrect metadatda checksum after all read attempts") + + /* Calculate and track the # of retries */ + retries = max_tries - tries; + if(retries) /* Does not track 0 retry */ + if(H5F_track_metadata_read_retries(f, (unsigned)type->mem_type, retries) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "cannot track read tries = %u ", retries) + + /* Set the final length (in case it wasn't set earlier) */ + len = actual_len; + } /* end if !H5C__CLASS_SKIP_READS */ + + /* Deserialize the on-disk image into the native memory form */ + if(NULL == (thing = type->deserialize(image, len, udata, &dirty))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "Can't deserialize image") entry = (H5C_cache_entry_t *)thing; @@ -7735,8 +6580,6 @@ H5C_load_entry(H5F_t * f, entry->addr = addr; entry->size = len; HDassert(entry->size < H5C_MAX_ENTRY_SIZE); - entry->compressed = compressed; - entry->compressed_size = compressed_size; entry->image_ptr = image; entry->image_up_to_date = TRUE; entry->type = type; @@ -7789,7 +6632,7 @@ done: if(thing && type->free_icr(thing) < 0) HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, NULL, "free_icr callback failed") if(image) - image = H5MM_xfree(image); + image = (uint8_t *)H5MM_xfree(image); } /* end if */ FUNC_LEAVE_NOAPI(ret_value) @@ -7935,12 +6778,11 @@ H5C_make_space_in_cache(H5F_t * f, next_ptr = entry_ptr->next; prev_ptr = entry_ptr->prev; - if ( prev_ptr != NULL ) { - + if(prev_ptr != NULL) prev_is_dirty = prev_ptr->is_dirty; - } - if (entry_ptr->is_corked && entry_ptr->is_dirty) { + if(entry_ptr->is_dirty && + (entry_ptr->tag_info && entry_ptr->tag_info->corked)) { /* Skip "dirty" corked entries. */ ++num_corked_entries; @@ -7980,7 +6822,7 @@ H5C_make_space_in_cache(H5F_t * f, } /* end if */ #endif - if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__NO_FLAGS_SET, NULL, NULL) < 0) + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__NO_FLAGS_SET) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry") if ( ( cache_ptr->entries_removed_counter > 1 ) || @@ -7997,7 +6839,7 @@ H5C_make_space_in_cache(H5F_t * f, cache_ptr->entries_scanned_to_make_space++; #endif /* H5C_COLLECT_CACHE_STATS */ - if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, NULL, NULL) < 0) + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry") } else { /* We have enough space so don't flush clean entry. */ @@ -8138,7 +6980,7 @@ H5C_make_space_in_cache(H5F_t * f, #ifdef H5_HAVE_PARALLEL if(!(entry_ptr->coll_access)) { #endif /* H5_HAVE_PARALLEL */ - if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, NULL, NULL) < 0) + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry") #ifdef H5_HAVE_PARALLEL } /* end if */ @@ -8657,15 +7499,16 @@ done: * Return: Success: Non-negative * Failure: Negative * - * Programmer: Vailin Choi; January 2014 + * Programmer: Vailin Choi + * January 2014 * *------------------------------------------------------------------------- */ herr_t -H5C_cork(H5C_t * cache_ptr, haddr_t obj_addr, unsigned action, hbool_t *corked) +H5C_cork(H5C_t *cache_ptr, haddr_t obj_addr, unsigned action, hbool_t *corked) { - haddr_t *ptr; /* Points to an address */ - herr_t ret_value = SUCCEED; /* Return value */ + H5C_tag_info_t *tag_info; /* Points to a tag info struct */ + herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT @@ -8675,61 +7518,69 @@ H5C_cork(H5C_t * cache_ptr, haddr_t obj_addr, unsigned action, hbool_t *corked) HDassert(action == H5C__SET_CORK || action == H5C__UNCORK || action == H5C__GET_CORKED); /* Search the list of corked object addresses in the cache */ - ptr = (haddr_t *)H5SL_search(cache_ptr->cork_list_ptr, &obj_addr); + tag_info = (H5C_tag_info_t *)H5SL_search(cache_ptr->tag_list, &obj_addr); if(H5C__GET_CORKED == action) { HDassert(corked); - if(ptr != NULL && *ptr == obj_addr) + if(tag_info != NULL && tag_info->corked) *corked = TRUE; else *corked = FALSE; } /* end if */ else { - hbool_t is_corked; /* Cork status for an entry */ - /* Sanity check */ HDassert(H5C__SET_CORK == action || H5C__UNCORK == action); /* Perform appropriate action */ if(H5C__SET_CORK == action) { - haddr_t *addr_ptr = NULL; /* Points to an address */ - - if(ptr != NULL && *ptr == obj_addr) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't cork an already corked object") - - /* Allocate address */ - if(NULL == (addr_ptr = H5FL_MALLOC(haddr_t))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") - - /* Insert into the list */ - *addr_ptr = obj_addr; - if(H5SL_insert(cache_ptr->cork_list_ptr, addr_ptr, addr_ptr) < 0) { - addr_ptr = H5FL_FREE(haddr_t, addr_ptr); - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't insert address into cork list") + /* Check if this is the first entry for this tagged object */ + if(NULL == tag_info) { + /* Allocate new tag info struct */ + if(NULL == (tag_info = H5FL_CALLOC(H5C_tag_info_t))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "can't allocate tag info for cache entry") + + /* Set the tag for all entries */ + tag_info->tag = obj_addr; + + /* Insert tag info into skip list */ + if(H5SL_insert(cache_ptr->tag_list, tag_info, &(tag_info->tag)) < 0 ) + HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "can't insert tag info in skip list") } /* end if */ + else { + /* Check for object already corked */ + if(tag_info->corked) + HGOTO_ERROR(H5E_CACHE, H5E_CANTCORK, FAIL, "object already corked") + HDassert(tag_info->entry_cnt > 0 && tag_info->head); + } /* end else */ - /* Set the entry's cork status */ - is_corked = TRUE; + /* Set the corked status for the entire object */ + tag_info->corked = TRUE; } /* end if */ else { - if(ptr == NULL) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't uncork an object that is not corked ") + /* Sanity check */ + HDassert(tag_info); - /* Remove the object address from the list */ - ptr = (haddr_t *)H5SL_remove(cache_ptr->cork_list_ptr, &obj_addr); - if(ptr == NULL || *ptr != obj_addr) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't remove address from list") + /* Check for already uncorked */ + if(!tag_info->corked) + HGOTO_ERROR(H5E_CACHE, H5E_CANTUNCORK, FAIL, "object already uncorked") - /* Free address */ - ptr = H5FL_FREE(haddr_t, ptr); + /* Set the corked status for the entire object */ + tag_info->corked = FALSE; - /* Set the entry's cork status */ - is_corked = FALSE; - } /* end else */ + /* Remove the tag info from the tag list, if there's no more entries with this tag */ + if(0 == tag_info->entry_cnt) { + /* Sanity check */ + HDassert(NULL == tag_info->head); - /* Mark existing cache entries with tag (obj_addr) to the cork status */ - if(H5C__mark_tagged_entries_cork(cache_ptr, obj_addr, is_corked) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "can't mark cork status on entry") + if(H5SL_remove(cache_ptr->tag_list, &(tag_info->tag)) != tag_info) + HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove tag info from list") + + /* Release the tag info */ + tag_info = H5FL_FREE(H5C_tag_info_t, tag_info); + } /* end if */ + else + HDassert(NULL != tag_info->head); + } /* end else */ } /* end else */ done: @@ -8756,8 +7607,9 @@ static herr_t H5C__mark_flush_dep_dirty(H5C_cache_entry_t * entry) { unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC_NOERR + FUNC_ENTER_STATIC /* Sanity checks */ HDassert(entry); @@ -8769,9 +7621,15 @@ H5C__mark_flush_dep_dirty(H5C_cache_entry_t * entry) /* Adjust the parent's number of dirty children */ entry->flush_dep_parent[u]->flush_dep_ndirty_children++; + + /* If the parent has a 'notify' callback, send a 'child entry dirtied' notice */ + if(entry->flush_dep_parent[u]->type->notify && + (entry->flush_dep_parent[u]->type->notify)(H5C_NOTIFY_ACTION_CHILD_DIRTIED, entry->flush_dep_parent[u]) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify parent about child entry dirty flag set") } /* end for */ - FUNC_LEAVE_NOAPI(SUCCEED) +done: + FUNC_LEAVE_NOAPI(ret_value) } /* H5C__mark_flush_dep_dirty() */ @@ -8794,8 +7652,9 @@ static herr_t H5C__mark_flush_dep_clean(H5C_cache_entry_t * entry) { unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC_NOERR + FUNC_ENTER_STATIC /* Sanity checks */ HDassert(entry); @@ -8807,9 +7666,15 @@ H5C__mark_flush_dep_clean(H5C_cache_entry_t * entry) /* Adjust the parent's number of dirty children */ entry->flush_dep_parent[u]->flush_dep_ndirty_children--; + + /* If the parent has a 'notify' callback, send a 'child entry cleaned' notice */ + if(entry->flush_dep_parent[u]->type->notify && + (entry->flush_dep_parent[u]->type->notify)(H5C_NOTIFY_ACTION_CHILD_CLEANED, entry->flush_dep_parent[u]) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify parent about child entry dirty flag reset") } /* end for */ - FUNC_LEAVE_NOAPI(SUCCEED) +done: + FUNC_LEAVE_NOAPI(ret_value) } /* H5C__mark_flush_dep_clean() */ #ifndef NDEBUG @@ -8858,6 +7723,15 @@ H5C__assert_flush_dep_nocycle(const H5C_cache_entry_t * entry, * * Purpose: Serialize an entry and generate its image. * + * Note: This may cause the entry to be re-sized and/or moved in + * the cache. + * + * As we will not update the metadata cache's data structures + * until we we finish the write, we must touch up these + * data structures for size and location changes even if we + * are about to delete the entry from the cache (i.e. on a + * flush destroy). + * * Return: Non-negative on success/Negative on failure * * Programmer: Mohamad Chaarawi @@ -8867,12 +7741,11 @@ H5C__assert_flush_dep_nocycle(const H5C_cache_entry_t * entry, */ static herr_t H5C__generate_image(const H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr, - hid_t dxpl_id, int64_t *entry_size_change_ptr) + hid_t dxpl_id) { haddr_t new_addr = HADDR_UNDEF; haddr_t old_addr = HADDR_UNDEF; size_t new_len = 0; - size_t new_compressed_len = 0; unsigned serialize_flags = H5C__SERIALIZE_NO_FLAGS_SET; herr_t ret_value = SUCCEED; @@ -8881,34 +7754,20 @@ H5C__generate_image(const H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_p /* Sanity check */ HDassert(!entry_ptr->image_up_to_date); - /* reset cache_ptr->slist_changed so we can detect slist - * modifications in the pre_serialize call. - */ - cache_ptr->slist_changed = FALSE; - /* make note of the entry's current address */ old_addr = entry_ptr->addr; /* Call client's pre-serialize callback, if there's one */ if(entry_ptr->type->pre_serialize && - (entry_ptr->type->pre_serialize)(f, dxpl_id, - (void *)entry_ptr, entry_ptr->addr, entry_ptr->size, - entry_ptr->compressed_size, &new_addr, &new_len, - &new_compressed_len, &serialize_flags) < 0) + (entry_ptr->type->pre_serialize)(f, dxpl_id, (void *)entry_ptr, + entry_ptr->addr, entry_ptr->size, &new_addr, &new_len, &serialize_flags) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to pre-serialize entry") - /* set cache_ptr->slist_change_in_pre_serialize if the - * slist was modified. - */ - if(cache_ptr->slist_changed) - cache_ptr->slist_change_in_pre_serialize = TRUE; - /* Check for any flags set in the pre-serialize callback */ if(serialize_flags != H5C__SERIALIZE_NO_FLAGS_SET) { /* Check for unexpected flags from serialize callback */ if(serialize_flags & ~(H5C__SERIALIZE_RESIZED_FLAG | - H5C__SERIALIZE_MOVED_FLAG | - H5C__SERIALIZE_COMPRESSED_FLAG)) + H5C__SERIALIZE_MOVED_FLAG)) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unknown serialize flag(s)") #ifdef H5_HAVE_PARALLEL @@ -8942,61 +7801,37 @@ H5C__generate_image(const H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_p HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "resize/move in serialize occured in parallel case.") #endif - /* Resize the buffer if required */ - if(((!entry_ptr->compressed) && (serialize_flags & H5C__SERIALIZE_RESIZED_FLAG)) || - ((entry_ptr->compressed) && (serialize_flags & H5C__SERIALIZE_COMPRESSED_FLAG))) { - size_t new_image_size; - - if(entry_ptr->compressed) - new_image_size = new_compressed_len; - else - new_image_size = new_len; - HDassert(new_image_size > 0); - - /* Release the current image */ - if(entry_ptr->image_ptr) - entry_ptr->image_ptr = H5MM_xfree(entry_ptr->image_ptr); - + /* If required, resize the buffer and update the entry and the cache + * data structures */ + if(serialize_flags & H5C__SERIALIZE_RESIZED_FLAG) { /* Allocate a new image buffer */ - if(NULL == (entry_ptr->image_ptr = H5MM_malloc(new_image_size + H5C_IMAGE_EXTRA_SPACE))) + if(NULL == (entry_ptr->image_ptr = H5MM_realloc(entry_ptr->image_ptr, new_len + H5C_IMAGE_EXTRA_SPACE))) HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for on disk image buffer") - #if H5C_DO_MEMORY_SANITY_CHECKS - HDmemcpy(((uint8_t *)entry_ptr->image_ptr) + new_image_size, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); + HDmemcpy(((uint8_t *)entry_ptr->image_ptr) + new_len, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); #endif /* H5C_DO_MEMORY_SANITY_CHECKS */ - } /* end if */ - /* If required, update the entry and the cache data structures - * for a resize. - */ - if(serialize_flags & H5C__SERIALIZE_RESIZED_FLAG) { + /* Update statistics for resizing the entry */ H5C__UPDATE_STATS_FOR_ENTRY_SIZE_CHANGE(cache_ptr, entry_ptr, new_len); - /* update the hash table for the size change*/ + /* update the hash table for the size change */ H5C__UPDATE_INDEX_FOR_SIZE_CHANGE(cache_ptr, entry_ptr->size, \ new_len, entry_ptr, !(entry_ptr->is_dirty)); - /* The entry can't be protected since we are - * in the process of flushing it. Thus we must - * update the replacement policy data - * structures for the size change. The macro - * deals with the pinned case. + /* The entry can't be protected since we are in the process of + * flushing it. Thus we must update the replacement policy data + * structures for the size change. The macro deals with the pinned + * case. */ H5C__UPDATE_RP_FOR_SIZE_CHANGE(cache_ptr, entry_ptr, new_len); - /* as we haven't updated the cache data structures for + /* As we haven't updated the cache data structures for * for the flush or flush destroy yet, the entry should * be in the slist. Thus update it for the size change. */ HDassert(entry_ptr->in_slist); H5C__UPDATE_SLIST_FOR_SIZE_CHANGE(cache_ptr, entry_ptr->size, new_len); - /* if defined, update *entry_size_change_ptr for the - * change in entry size. - */ - if(entry_size_change_ptr != NULL) - *entry_size_change_ptr = (int64_t)new_len - (int64_t)(entry_ptr->size); - /* finally, update the entry for its new size */ entry_ptr->size = new_len; } /* end if */ @@ -9005,92 +7840,152 @@ H5C__generate_image(const H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_p * for a move */ if(serialize_flags & H5C__SERIALIZE_MOVED_FLAG) { -#if H5C_DO_SANITY_CHECKS - int64_t saved_slist_len_increase; - int64_t saved_slist_size_increase; -#endif /* H5C_DO_SANITY_CHECKS */ - H5C__UPDATE_STATS_FOR_MOVE(cache_ptr, entry_ptr); + /* We must update cache data structures for the change in address */ if(entry_ptr->addr == old_addr) { - /* we must update cache data structures for the - * change in address. - */ - - /* delete the entry from the hash table and the slist */ + /* Delete the entry from the hash table and the slist */ H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr); - H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr); + H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, FALSE); /* update the entry for its new address */ entry_ptr->addr = new_addr; /* and then reinsert in the index and slist */ H5C__INSERT_IN_INDEX(cache_ptr, entry_ptr, FAIL); - -#if H5C_DO_SANITY_CHECKS - /* save cache_ptr->slist_len_increase and - * cache_ptr->slist_size_increase before the - * reinsertion into the slist, and restore - * them afterwards to avoid skewing our sanity - * checking. - */ - saved_slist_len_increase = cache_ptr->slist_len_increase; - saved_slist_size_increase = cache_ptr->slist_size_increase; -#endif /* H5C_DO_SANITY_CHECKS */ - H5C__INSERT_ENTRY_IN_SLIST(cache_ptr, entry_ptr, FAIL); - -#if H5C_DO_SANITY_CHECKS - cache_ptr->slist_len_increase = saved_slist_len_increase; - cache_ptr->slist_size_increase = saved_slist_size_increase; -#endif /* H5C_DO_SANITY_CHECKS */ } /* end if */ else /* move is already done for us -- just do sanity checks */ HDassert(entry_ptr->addr == new_addr); } /* end if */ - - if(serialize_flags & H5C__SERIALIZE_COMPRESSED_FLAG) { - /* just save the new compressed entry size in - * entry_ptr->compressed_size. We don't need to - * do more, as compressed size is only used for I/O. - */ - HDassert(entry_ptr->compressed); - entry_ptr->compressed_size = new_compressed_len; - } /* end if */ } /* end if(serialize_flags != H5C__SERIALIZE_NO_FLAGS_SET) */ /* Serialize object into buffer */ - { - size_t image_len; + if(entry_ptr->type->serialize(f, entry_ptr->image_ptr, entry_ptr->size, (void *)entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to serialize entry") +#if H5C_DO_MEMORY_SANITY_CHECKS + HDassert(0 == HDmemcmp(((uint8_t *)entry_ptr->image_ptr) + entry_ptr->size, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE)); +#endif /* H5C_DO_MEMORY_SANITY_CHECKS */ + entry_ptr->image_up_to_date = TRUE; - if(entry_ptr->compressed) - image_len = entry_ptr->compressed_size; - else - image_len = entry_ptr->size; +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C__generate_image */ - /* reset cache_ptr->slist_changed so we can detect slist - * modifications in the serialize call. - */ - cache_ptr->slist_changed = FALSE; - - if(entry_ptr->type->serialize(f, entry_ptr->image_ptr, image_len, (void *)entry_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to serialize entry") + +/*------------------------------------------------------------------------- + * + * Function: H5C_remove_entry + * + * Purpose: Remove an entry from the cache. Must be not protected, pinned, + * dirty, involved in flush dependencies, etc. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * September 17, 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_remove_entry(void *_entry) +{ + H5C_cache_entry_t *entry = (H5C_cache_entry_t *)_entry; /* Entry to remove */ + H5C_t *cache; /* Cache for file */ + herr_t ret_value = SUCCEED; /* Return value */ - /* set cache_ptr->slist_change_in_serialize if the - * slist was modified. - */ - if(cache_ptr->slist_changed) - cache_ptr->slist_change_in_pre_serialize = TRUE; + FUNC_ENTER_NOAPI(FAIL) -#if H5C_DO_MEMORY_SANITY_CHECKS - HDassert(0 == HDmemcmp(((uint8_t *)entry_ptr->image_ptr) + image_len, - H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE)); -#endif /* H5C_DO_MEMORY_SANITY_CHECKS */ + /* Sanity checks */ + HDassert(entry); + HDassert(entry->ring != H5C_RING_UNDEFINED); + cache = entry->cache_ptr; + HDassert(cache); + HDassert(cache->magic == H5C__H5C_T_MAGIC); + + /* Check for error conditions */ + if(entry->is_dirty) + HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove dirty entry from cache") + if(entry->is_protected) + HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove protected entry from cache") + if(entry->is_pinned) + HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove pinned entry from cache") + if(entry->flush_dep_nparents > 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove entry with flush dependency parents from cache") + if(entry->flush_dep_nchildren > 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove entry with flush dependency children from cache") + + /* Additional internal cache consistency checks */ + HDassert(!entry->in_slist); + HDassert(!entry->flush_marker); + HDassert(!entry->flush_in_progress); + + /* Note that the algorithm below is (very) similar to the set of operations + * in H5C__flush_single_entry() and should be kept in sync with changes + * to that code. - QAK, 2016/11/30 + */ - entry_ptr->image_up_to_date = TRUE; - } /* end block */ + /* Update stats, as if we are "destroying" and taking ownership of the entry */ + H5C__UPDATE_STATS_FOR_EVICTION(cache, entry, TRUE) + + /* If the entry's type has a 'notify' callback, send a 'before eviction' + * notice while the entry is still fully integrated in the cache. + */ + if(entry->type->notify && (entry->type->notify)(H5C_NOTIFY_ACTION_BEFORE_EVICT, entry) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry to evict") + + /* Update the cache internal data structures as appropriate for a destroy. + * Specifically: + * 1) Delete it from the index + * 2) Update the replacement policy for eviction + * 3) Remove it from the tag list for this object + */ + + H5C__DELETE_FROM_INDEX(cache, entry) + + H5C__UPDATE_RP_FOR_EVICTION(cache, entry, FAIL) + + /* Remove entry from tag list */ + if(H5C__untag_entry(cache, entry) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove entry from tag list") + + /* Increment entries_removed_counter and set last_entry_removed_ptr. + * As we me be about to free the entry, recall that last_entry_removed_ptr + * must NEVER be dereferenced. + * + * Recall that these fields are maintained to allow functions that perform + * scans of lists of entries to detect the unexpected removal of entries + * (via expunge, eviction, or take ownership at present), so that they can + * re-start their scans if necessary. + * + * Also check if the entry we are watching for removal is being + * removed (usually the 'next' entry for an iteration) and reset + * it to indicate that it was removed. + */ + cache->entries_removed_counter++; + cache->last_entry_removed_ptr = entry; + if(entry == cache->entry_watched_for_removal) + cache->entry_watched_for_removal = NULL; + + /* Internal cache data structures should now be up to date, and + * consistant with the status of the entry. + * + * Now clean up internal cache fields if appropriate. + */ + + /* Free the buffer for the on disk image */ + if(entry->image_ptr != NULL) + entry->image_ptr = H5MM_xfree(entry->image_ptr); + + /* Reset the pointer to the cache the entry is within */ + entry->cache_ptr = NULL; + + /* Client is taking ownership of the entry. Set bad magic here so the + * cache will choke unless the entry is re-inserted properly + */ + entry->magic = H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC; done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5C__generate_image */ +} /* H5C__remove_entry() */ diff --git a/src/H5Cdbg.c b/src/H5Cdbg.c new file mode 100644 index 0000000..16077c8 --- /dev/null +++ b/src/H5Cdbg.c @@ -0,0 +1,981 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5Cdbg.c + * July 8 2016 + * Quincey Koziol + * + * Purpose: Debugging Routines for the generic cache structure or entries. + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5Cmodule.h" /* This source code file is part of the H5C module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Cpkg.h" /* Cache */ +#include "H5Eprivate.h" /* Error handling */ + + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + +#if 0 /* debugging routines */ +herr_t H5C_dump_cache_skip_list(H5C_t *cache_ptr, char *calling_fcn); +#endif /* debugging routines */ + + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + + + +/*------------------------------------------------------------------------- + * Function: H5C_dump_cache + * + * Purpose: Print a summary of the contents of the metadata cache for + * debugging purposes. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 10/10/10 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_dump_cache(H5C_t * cache_ptr, const char * cache_name) +{ + H5C_cache_entry_t * entry_ptr; + H5SL_t * slist_ptr = NULL; + int i; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(cache_ptr != NULL); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(cache_name != NULL ); + + /* First, create a skip list */ + if(NULL == (slist_ptr = H5SL_create(H5SL_TYPE_HADDR, NULL))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, FAIL, "can't create skip list.") + + /* Next, scan the index, and insert all entries in the skip list. + * Do this, as we want to display cache entries in increasing address + * order. + */ + for(i = 0; i < H5C__HASH_TABLE_LEN; i++) { + entry_ptr = cache_ptr->index[i]; + + while(entry_ptr != NULL) { + HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + if(H5SL_insert(slist_ptr, entry_ptr, &(entry_ptr->addr)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "can't insert entry in skip list") + + entry_ptr = entry_ptr->ht_next; + } /* end while */ + } /* end for */ + + /* If we get this far, all entries in the cache are listed in the + * skip list -- scan the skip list generating the desired output. + */ + + HDfprintf(stdout, "\n\nDump of metadata cache \"%s\"\n", cache_name); + + /* Print header */ + HDfprintf(stdout, "Entry "); + HDfprintf(stdout, "| Address "); + HDfprintf(stdout, "| Tag "); + HDfprintf(stdout, "| Size "); + HDfprintf(stdout, "| Ring "); + HDfprintf(stdout, "| Type "); + HDfprintf(stdout, "| Prot/Pin/Dirty"); + HDfprintf(stdout, "\n"); + + HDfprintf(stdout, "----------------------------------------------------------------------------------------------------------------\n"); + + i = 0; + entry_ptr = (H5C_cache_entry_t *)H5SL_remove_first(slist_ptr); + while(entry_ptr != NULL) { + HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + + /* Print entry */ + HDfprintf(stdout, "%s%5d ", cache_ptr->prefix, i); + HDfprintf(stdout, " 0x%16llx ", (long long)(entry_ptr->addr)); + if(NULL == entry_ptr->tag_info) + HDfprintf(stdout, " %16s ", "N/A"); + else + HDfprintf(stdout, " 0x%16llx ", (long long)(entry_ptr->tag_info->tag)); + HDfprintf(stdout, " %5lld ", (long long)(entry_ptr->size)); + HDfprintf(stdout, " %d ", (int)(entry_ptr->ring)); + HDfprintf(stdout, " %2d %-32s ", (int)(entry_ptr->type->id), (entry_ptr->type->name)); + HDfprintf(stdout, " %d", (int)(entry_ptr->is_protected)); + HDfprintf(stdout, " %d", (int)(entry_ptr->is_pinned)); + HDfprintf(stdout, " %d", (int)(entry_ptr->is_dirty)); + HDfprintf(stdout, "\n"); + + /* remove the next (first) item in the skip list */ + entry_ptr = (H5C_cache_entry_t *)H5SL_remove_first(slist_ptr); + + i++; + } /* end while */ + + HDfprintf(stdout, "\n\n"); + + /* Verify that all the entries were removed from the skip list */ + HDassert(H5SL_count(slist_ptr) == 0); + +done: + /* Discard the skip list */ + if(slist_ptr) + H5SL_close(slist_ptr); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_dump_cache() */ + + +/*------------------------------------------------------------------------- + * Function: H5C_dump_cache_skip_list + * + * Purpose: Debugging routine that prints a summary of the contents of + * the skip list used by the metadata cache metadata cache to + * maintain an address sorted list of dirty entries. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 11/15/14 + * + *------------------------------------------------------------------------- + */ +#if 0 /* debugging routine */ +herr_t +H5C_dump_cache_skip_list(H5C_t * cache_ptr, char * calling_fcn) +{ + herr_t ret_value = SUCCEED; /* Return value */ + int i; + H5C_cache_entry_t * entry_ptr = NULL; + H5SL_node_t * node_ptr = NULL; + + FUNC_ENTER_NOAPI(FAIL) + + HDassert(cache_ptr != NULL); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(calling_fcn != NULL); + + HDfprintf(stdout, "\n\nDumping metadata cache skip list from %s.\n", calling_fcn); + HDfprintf(stdout, " slist len = %d.\n", cache_ptr->slist_len); + HDfprintf(stdout, " slist size = %lld.\n", (long long)(cache_ptr->slist_size)); + + if(cache_ptr->slist_len > 0) { + /* If we get this far, all entries in the cache are listed in the + * skip list -- scan the skip list generating the desired output. + */ + HDfprintf(stdout, + "Num: Addr: Len: Prot/Pind: Dirty: Type:\n"); + + i = 0; + node_ptr = H5SL_first(cache_ptr->slist_ptr); + if(node_ptr != NULL) + entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); + else + entry_ptr = NULL; + + while(entry_ptr != NULL) { + HDassert( entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC ); + + HDfprintf(stdout, + "%s%d 0x%016llx %4lld %d/%d %d %s\n", + cache_ptr->prefix, i, + (long long)(entry_ptr->addr), + (long long)(entry_ptr->size), + (int)(entry_ptr->is_protected), + (int)(entry_ptr->is_pinned), + (int)(entry_ptr->is_dirty), + entry_ptr->type->name); + + HDfprintf(stdout, " node_ptr = 0x%llx, item = 0x%llx\n", + (unsigned long long)node_ptr, + (unsigned long long)H5SL_item(node_ptr)); + + /* increment node_ptr before we delete its target */ + node_ptr = H5SL_next(node_ptr); + if(node_ptr != NULL) + entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); + else + entry_ptr = NULL; + + i++; + } /* end while */ + } /* end if */ + + HDfprintf(stdout, "\n\n"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_dump_cache_skip_list() */ +#endif /* debugging routine */ + + +/*------------------------------------------------------------------------- + * Function: H5C_set_prefix + * + * Purpose: Set the values of the prefix field of H5C_t. This + * filed is used to label some debugging output. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 1/20/06 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_set_prefix(H5C_t * cache_ptr, char * prefix) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + if((cache_ptr == NULL) || (cache_ptr->magic != H5C__H5C_T_MAGIC) || + (prefix == NULL) || (HDstrlen(prefix) >= H5C__PREFIX_LEN)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Bad param(s) on entry.") + + HDstrncpy(&(cache_ptr->prefix[0]), prefix, (size_t)(H5C__PREFIX_LEN)); + + cache_ptr->prefix[H5C__PREFIX_LEN - 1] = '\0'; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_set_prefix() */ + + +/*------------------------------------------------------------------------- + * Function: H5C_set_trace_file_ptr + * + * Purpose: Set the trace_file_ptr field for the cache. + * + * This field must either be NULL (which turns of trace + * file logging), or be a pointer to an open file to which + * trace file data is to be written. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 1/20/06 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_set_trace_file_ptr(H5C_t * cache_ptr, FILE * trace_file_ptr) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* This would normally be an assert, but we need to use an HGOTO_ERROR + * call to shut up the compiler. + */ + if((NULL == cache_ptr) || (cache_ptr->magic != H5C__H5C_T_MAGIC)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Bad cache_ptr") + + cache_ptr->trace_file_ptr = trace_file_ptr; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_set_trace_file_ptr() */ + + +/*------------------------------------------------------------------------- + * Function: H5C_stats + * + * Purpose: Prints statistics about the cache. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 6/2/04 + * + * JRM -- 11/13/08 + * Added code displaying the max_clean_index_size and + * max_dirty_index_size. + * + * MAM -- 01/06/09 + * Added code displaying the calls_to_msic, + * total_entries_skipped_in_msic, total_entries_scanned_in_msic, + * and max_entries_skipped_in_msic fields. + * + * JRM -- 4/11/15 + * Added code displaying the new slist_scan_restarts, + * LRU_scan_restarts, and hash_bucket_scan_restarts fields; + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_stats(H5C_t * cache_ptr, + const char * cache_name, + hbool_t +#if !H5C_COLLECT_CACHE_STATS + H5_ATTR_UNUSED +#endif /* H5C_COLLECT_CACHE_STATS */ + display_detailed_stats) +{ +#if H5C_COLLECT_CACHE_STATS + int i; + int64_t total_hits = 0; + int64_t total_misses = 0; + int64_t total_write_protects = 0; + int64_t total_read_protects = 0; + int64_t max_read_protects = 0; + int64_t total_insertions = 0; + int64_t total_pinned_insertions = 0; + int64_t total_clears = 0; + int64_t total_flushes = 0; + int64_t total_evictions = 0; + int64_t total_take_ownerships = 0; + int64_t total_moves = 0; + int64_t total_entry_flush_moves = 0; + int64_t total_cache_flush_moves = 0; + int64_t total_size_increases = 0; + int64_t total_size_decreases = 0; + int64_t total_entry_flush_size_changes = 0; + int64_t total_cache_flush_size_changes = 0; + int64_t total_pins = 0; + int64_t total_unpins = 0; + int64_t total_dirty_pins = 0; + int64_t total_pinned_flushes = 0; + int64_t total_pinned_clears = 0; + int32_t aggregate_max_accesses = 0; + int32_t aggregate_min_accesses = 1000000; + int32_t aggregate_max_clears = 0; + int32_t aggregate_max_flushes = 0; + size_t aggregate_max_size = 0; + int32_t aggregate_max_pins = 0; + double hit_rate; + double average_successful_search_depth = 0.0f; + double average_failed_search_depth = 0.0f; + double average_entries_skipped_per_calls_to_msic = 0.0f; + double average_entries_scanned_per_calls_to_msic = 0.0f; +#endif /* H5C_COLLECT_CACHE_STATS */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); + + /* This would normally be an assert, but we need to use an HGOTO_ERROR + * call to shut up the compiler. + */ + if((NULL == cache_ptr) || (cache_ptr->magic != H5C__H5C_T_MAGIC) || + (NULL == cache_name)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Bad cache_ptr or cache_name") + +#if H5C_COLLECT_CACHE_STATS + for(i = 0; i <= cache_ptr->max_type_id; i++ ) { + total_hits += cache_ptr->hits[i]; + total_misses += cache_ptr->misses[i]; + total_write_protects += cache_ptr->write_protects[i]; + total_read_protects += cache_ptr->read_protects[i]; + if(max_read_protects < cache_ptr->max_read_protects[i]) + max_read_protects = cache_ptr->max_read_protects[i]; + total_insertions += cache_ptr->insertions[i]; + total_pinned_insertions += cache_ptr->pinned_insertions[i]; + total_clears += cache_ptr->clears[i]; + total_flushes += cache_ptr->flushes[i]; + total_evictions += cache_ptr->evictions[i]; + total_take_ownerships += cache_ptr->take_ownerships[i]; + total_moves += cache_ptr->moves[i]; + total_entry_flush_moves += cache_ptr->entry_flush_moves[i]; + total_cache_flush_moves += cache_ptr->cache_flush_moves[i]; + total_size_increases += cache_ptr->size_increases[i]; + total_size_decreases += cache_ptr->size_decreases[i]; + total_entry_flush_size_changes + += cache_ptr->entry_flush_size_changes[i]; + total_cache_flush_size_changes + += cache_ptr->cache_flush_size_changes[i]; + total_pins += cache_ptr->pins[i]; + total_unpins += cache_ptr->unpins[i]; + total_dirty_pins += cache_ptr->dirty_pins[i]; + total_pinned_flushes += cache_ptr->pinned_flushes[i]; + total_pinned_clears += cache_ptr->pinned_clears[i]; +#if H5C_COLLECT_CACHE_ENTRY_STATS + if(aggregate_max_accesses < cache_ptr->max_accesses[i]) + aggregate_max_accesses = cache_ptr->max_accesses[i]; + if(aggregate_min_accesses > aggregate_max_accesses) + aggregate_min_accesses = aggregate_max_accesses; + if(aggregate_min_accesses > cache_ptr->min_accesses[i]) + aggregate_min_accesses = cache_ptr->min_accesses[i]; + if(aggregate_max_clears < cache_ptr->max_clears[i]) + aggregate_max_clears = cache_ptr->max_clears[i]; + if(aggregate_max_flushes < cache_ptr->max_flushes[i]) + aggregate_max_flushes = cache_ptr->max_flushes[i]; + if(aggregate_max_size < cache_ptr->max_size[i]) + aggregate_max_size = cache_ptr->max_size[i]; + if(aggregate_max_pins < cache_ptr->max_pins[i]) + aggregate_max_pins = cache_ptr->max_pins[i]; +#endif /* H5C_COLLECT_CACHE_ENTRY_STATS */ + } /* end for */ + + if((total_hits > 0) || (total_misses > 0)) + hit_rate = (double)100.0f * ((double)(total_hits)) / + ((double)(total_hits + total_misses)); + else + hit_rate = 0.0f; + + if(cache_ptr->successful_ht_searches > 0) + average_successful_search_depth = + ((double)(cache_ptr->total_successful_ht_search_depth)) / + ((double)(cache_ptr->successful_ht_searches)); + + if(cache_ptr->failed_ht_searches > 0) + average_failed_search_depth = + ((double)(cache_ptr->total_failed_ht_search_depth)) / + ((double)(cache_ptr->failed_ht_searches)); + + + HDfprintf(stdout, "\n%sH5C: cache statistics for %s\n", + cache_ptr->prefix, cache_name); + + HDfprintf(stdout, "\n"); + + HDfprintf(stdout, + "%s hash table insertion / deletions = %ld / %ld\n", + cache_ptr->prefix, + (long)(cache_ptr->total_ht_insertions), + (long)(cache_ptr->total_ht_deletions)); + + HDfprintf(stdout, + "%s HT successful / failed searches = %ld / %ld\n", + cache_ptr->prefix, + (long)(cache_ptr->successful_ht_searches), + (long)(cache_ptr->failed_ht_searches)); + + HDfprintf(stdout, + "%s Av. HT suc / failed search depth = %f / %f\n", + cache_ptr->prefix, + average_successful_search_depth, + average_failed_search_depth); + + HDfprintf(stdout, + "%s current (max) index size / length = %ld (%ld) / %ld (%ld)\n", + cache_ptr->prefix, + (long)(cache_ptr->index_size), + (long)(cache_ptr->max_index_size), + (long)(cache_ptr->index_len), + (long)(cache_ptr->max_index_len)); + + HDfprintf(stdout, + "%s current (max) clean/dirty idx size = %ld (%ld) / %ld (%ld)\n", + cache_ptr->prefix, + (long)(cache_ptr->clean_index_size), + (long)(cache_ptr->max_clean_index_size), + (long)(cache_ptr->dirty_index_size), + (long)(cache_ptr->max_dirty_index_size)); + + HDfprintf(stdout, + "%s current (max) slist size / length = %ld (%ld) / %ld (%ld)\n", + cache_ptr->prefix, + (long)(cache_ptr->slist_size), + (long)(cache_ptr->max_slist_size), + (long)(cache_ptr->slist_len), + (long)(cache_ptr->max_slist_len)); + + HDfprintf(stdout, + "%s current (max) PL size / length = %ld (%ld) / %ld (%ld)\n", + cache_ptr->prefix, + (long)(cache_ptr->pl_size), + (long)(cache_ptr->max_pl_size), + (long)(cache_ptr->pl_len), + (long)(cache_ptr->max_pl_len)); + + HDfprintf(stdout, + "%s current (max) PEL size / length = %ld (%ld) / %ld (%ld)\n", + cache_ptr->prefix, + (long)(cache_ptr->pel_size), + (long)(cache_ptr->max_pel_size), + (long)(cache_ptr->pel_len), + (long)(cache_ptr->max_pel_len)); + + HDfprintf(stdout, + "%s current LRU list size / length = %ld / %ld\n", + cache_ptr->prefix, + (long)(cache_ptr->LRU_list_size), + (long)(cache_ptr->LRU_list_len)); + + HDfprintf(stdout, + "%s current clean LRU size / length = %ld / %ld\n", + cache_ptr->prefix, + (long)(cache_ptr->cLRU_list_size), + (long)(cache_ptr->cLRU_list_len)); + + HDfprintf(stdout, + "%s current dirty LRU size / length = %ld / %ld\n", + cache_ptr->prefix, + (long)(cache_ptr->dLRU_list_size), + (long)(cache_ptr->dLRU_list_len)); + + HDfprintf(stdout, + "%s Total hits / misses / hit_rate = %ld / %ld / %f\n", + cache_ptr->prefix, + (long)total_hits, + (long)total_misses, + hit_rate); + + HDfprintf(stdout, + "%s Total write / read (max) protects = %ld / %ld (%ld)\n", + cache_ptr->prefix, + (long)total_write_protects, + (long)total_read_protects, + (long)max_read_protects); + + HDfprintf(stdout, + "%s Total clears / flushes = %ld / %ld\n", + cache_ptr->prefix, + (long)total_clears, + (long)total_flushes); + + HDfprintf(stdout, + "%s Total evictions / take ownerships = %ld / %ld\n", + cache_ptr->prefix, + (long)total_evictions, + (long)total_take_ownerships); + + HDfprintf(stdout, + "%s Total insertions(pinned) / moves = %ld(%ld) / %ld\n", + cache_ptr->prefix, + (long)total_insertions, + (long)total_pinned_insertions, + (long)total_moves); + + HDfprintf(stdout, + "%s Total entry / cache flush moves = %ld / %ld\n", + cache_ptr->prefix, + (long)total_entry_flush_moves, + (long)total_cache_flush_moves); + + HDfprintf(stdout, "%s Total entry size incrs / decrs = %ld / %ld\n", + cache_ptr->prefix, + (long)total_size_increases, + (long)total_size_decreases); + + HDfprintf(stdout, "%s Ttl entry/cache flush size changes = %ld / %ld\n", + cache_ptr->prefix, + (long)total_entry_flush_size_changes, + (long)total_cache_flush_size_changes); + + HDfprintf(stdout, + "%s Total entry pins (dirty) / unpins = %ld (%ld) / %ld\n", + cache_ptr->prefix, + (long)total_pins, + (long)total_dirty_pins, + (long)total_unpins); + + HDfprintf(stdout, "%s Total pinned flushes / clears = %ld / %ld\n", + cache_ptr->prefix, + (long)total_pinned_flushes, + (long)total_pinned_clears); + + HDfprintf(stdout, "%s MSIC: (make space in cache) calls = %lld\n", + cache_ptr->prefix, + (long long)(cache_ptr->calls_to_msic)); + + if (cache_ptr->calls_to_msic > 0) { + average_entries_skipped_per_calls_to_msic = + (((double)(cache_ptr->total_entries_skipped_in_msic)) / + ((double)(cache_ptr->calls_to_msic))); + } + + HDfprintf(stdout, "%s MSIC: Average/max entries skipped = %lf / %ld\n", + cache_ptr->prefix, + (double)average_entries_skipped_per_calls_to_msic, + (long)(cache_ptr->max_entries_skipped_in_msic)); + + if(cache_ptr->calls_to_msic > 0) + average_entries_scanned_per_calls_to_msic = + (((double)(cache_ptr->total_entries_scanned_in_msic)) / + ((double)(cache_ptr->calls_to_msic))); + + HDfprintf(stdout, "%s MSIC: Average/max entries scanned = %lf / %ld\n", + cache_ptr->prefix, + (double)average_entries_scanned_per_calls_to_msic, + (long)(cache_ptr->max_entries_scanned_in_msic)); + + HDfprintf(stdout, "%s MSIC: Scanned to make space(evict) = %lld\n", + cache_ptr->prefix, + (long long)(cache_ptr->entries_scanned_to_make_space)); + + HDfprintf(stdout, "%s MSIC: Scanned to satisfy min_clean = %lld\n", + cache_ptr->prefix, + (long long)(cache_ptr->total_entries_scanned_in_msic - + cache_ptr->entries_scanned_to_make_space)); + + HDfprintf(stdout, + "%s slist/LRU/hash bkt scan restarts = %lld / %lld / %lld.\n", + cache_ptr->prefix, + (long long)(cache_ptr->slist_scan_restarts), + (long long)(cache_ptr->LRU_scan_restarts), + (long long)(cache_ptr->hash_bucket_scan_restarts)); + +#if H5C_COLLECT_CACHE_ENTRY_STATS + + HDfprintf(stdout, "%s aggregate max / min accesses = %d / %d\n", + cache_ptr->prefix, + (int)aggregate_max_accesses, + (int)aggregate_min_accesses); + + HDfprintf(stdout, "%s aggregate max_clears / max_flushes = %d / %d\n", + cache_ptr->prefix, + (int)aggregate_max_clears, + (int)aggregate_max_flushes); + + HDfprintf(stdout, "%s aggregate max_size / max_pins = %d / %d\n", + cache_ptr->prefix, + (int)aggregate_max_size, + (int)aggregate_max_pins); + +#endif /* H5C_COLLECT_CACHE_ENTRY_STATS */ + + if(display_detailed_stats) { + for(i = 0; i <= cache_ptr->max_type_id; i++) { + HDfprintf(stdout, "\n"); + + HDfprintf(stdout, "%s Stats on %s:\n", + cache_ptr->prefix, + ((cache_ptr->type_name_table_ptr))[i]); + + if((cache_ptr->hits[i] > 0) || (cache_ptr->misses[i] > 0)) + hit_rate = (double)100.0f * ((double)(cache_ptr->hits[i])) / + ((double)(cache_ptr->hits[i] + cache_ptr->misses[i])); + else + hit_rate = 0.0f; + + HDfprintf(stdout, + "%s hits / misses / hit_rate = %ld / %ld / %f\n", + cache_ptr->prefix, + (long)(cache_ptr->hits[i]), + (long)(cache_ptr->misses[i]), + hit_rate); + + HDfprintf(stdout, + "%s write / read (max) protects = %ld / %ld (%d)\n", + cache_ptr->prefix, + (long)(cache_ptr->write_protects[i]), + (long)(cache_ptr->read_protects[i]), + (int)(cache_ptr->max_read_protects[i])); + + HDfprintf(stdout, + "%s clears / flushes = %ld / %ld\n", + cache_ptr->prefix, + (long)(cache_ptr->clears[i]), + (long)(cache_ptr->flushes[i])); + + HDfprintf(stdout, + "%s evictions / take ownerships = %ld / %ld\n", + cache_ptr->prefix, + (long)(cache_ptr->evictions[i]), + (long)(cache_ptr->take_ownerships[i])); + + HDfprintf(stdout, + "%s insertions(pinned) / moves = %ld(%ld) / %ld\n", + cache_ptr->prefix, + (long)(cache_ptr->insertions[i]), + (long)(cache_ptr->pinned_insertions[i]), + (long)(cache_ptr->moves[i])); + + HDfprintf(stdout, + "%s entry / cache flush moves = %ld / %ld\n", + cache_ptr->prefix, + (long)(cache_ptr->entry_flush_moves[i]), + (long)(cache_ptr->cache_flush_moves[i])); + + HDfprintf(stdout, + "%s size increases / decreases = %ld / %ld\n", + cache_ptr->prefix, + (long)(cache_ptr->size_increases[i]), + (long)(cache_ptr->size_decreases[i])); + + HDfprintf(stdout, + "%s entry/cache flush size changes = %ld / %ld\n", + cache_ptr->prefix, + (long)(cache_ptr->entry_flush_size_changes[i]), + (long)(cache_ptr->cache_flush_size_changes[i])); + + + HDfprintf(stdout, + "%s entry pins / unpins = %ld / %ld\n", + cache_ptr->prefix, + (long)(cache_ptr->pins[i]), + (long)(cache_ptr->unpins[i])); + + HDfprintf(stdout, + "%s entry dirty pins/pin'd flushes = %ld / %ld\n", + cache_ptr->prefix, + (long)(cache_ptr->dirty_pins[i]), + (long)(cache_ptr->pinned_flushes[i])); + +#if H5C_COLLECT_CACHE_ENTRY_STATS + + HDfprintf(stdout, + "%s entry max / min accesses = %d / %d\n", + cache_ptr->prefix, + cache_ptr->max_accesses[i], + cache_ptr->min_accesses[i]); + + HDfprintf(stdout, + "%s entry max_clears / max_flushes = %d / %d\n", + cache_ptr->prefix, + cache_ptr->max_clears[i], + cache_ptr->max_flushes[i]); + + HDfprintf(stdout, + "%s entry max_size / max_pins = %d / %d\n", + cache_ptr->prefix, + (int)(cache_ptr->max_size[i]), + (int)(cache_ptr->max_pins[i])); + + +#endif /* H5C_COLLECT_CACHE_ENTRY_STATS */ + + } /* end for */ + } /* end if */ + + HDfprintf(stdout, "\n"); + +#endif /* H5C_COLLECT_CACHE_STATS */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_stats() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5C_stats__reset + * + * Purpose: Reset the stats fields to their initial values. + * + * Return: void + * + * Programmer: John Mainzer, 4/28/04 + * + * JRM 11/13/08 + * Added initialization for the new max_clean_index_size and + * max_dirty_index_size fields. + * + * MAM -- 01/06/09 + * Added code to initalize the calls_to_msic, + * total_entries_skipped_in_msic, total_entries_scanned_in_msic, + * and max_entries_skipped_in_msic fields. + * + * JRM 4/11/15 + * Added code to initialize the new slist_scan_restarts, + * LRU_scan_restarts, hash_bucket_scan_restarts, and + * take_ownerships fields. + * + *------------------------------------------------------------------------- + */ +void +#ifndef NDEBUG +H5C_stats__reset(H5C_t * cache_ptr) +#else /* NDEBUG */ +#if H5C_COLLECT_CACHE_STATS +H5C_stats__reset(H5C_t * cache_ptr) +#else /* H5C_COLLECT_CACHE_STATS */ +H5C_stats__reset(H5C_t H5_ATTR_UNUSED * cache_ptr) +#endif /* H5C_COLLECT_CACHE_STATS */ +#endif /* NDEBUG */ +{ +#if H5C_COLLECT_CACHE_STATS + int i; +#endif /* H5C_COLLECT_CACHE_STATS */ + + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + +#if H5C_COLLECT_CACHE_STATS + for(i = 0; i <= cache_ptr->max_type_id; i++) { + cache_ptr->hits[i] = 0; + cache_ptr->misses[i] = 0; + cache_ptr->write_protects[i] = 0; + cache_ptr->read_protects[i] = 0; + cache_ptr->max_read_protects[i] = 0; + cache_ptr->insertions[i] = 0; + cache_ptr->pinned_insertions[i] = 0; + cache_ptr->clears[i] = 0; + cache_ptr->flushes[i] = 0; + cache_ptr->evictions[i] = 0; + cache_ptr->take_ownerships[i] = 0; + cache_ptr->moves[i] = 0; + cache_ptr->entry_flush_moves[i] = 0; + cache_ptr->cache_flush_moves[i] = 0; + cache_ptr->pins[i] = 0; + cache_ptr->unpins[i] = 0; + cache_ptr->dirty_pins[i] = 0; + cache_ptr->pinned_flushes[i] = 0; + cache_ptr->pinned_clears[i] = 0; + cache_ptr->size_increases[i] = 0; + cache_ptr->size_decreases[i] = 0; + cache_ptr->entry_flush_size_changes[i] = 0; + cache_ptr->cache_flush_size_changes[i] = 0; + } /* end for */ + + cache_ptr->total_ht_insertions = 0; + cache_ptr->total_ht_deletions = 0; + cache_ptr->successful_ht_searches = 0; + cache_ptr->total_successful_ht_search_depth = 0; + cache_ptr->failed_ht_searches = 0; + cache_ptr->total_failed_ht_search_depth = 0; + + cache_ptr->max_index_len = 0; + cache_ptr->max_index_size = (size_t)0; + cache_ptr->max_clean_index_size = (size_t)0; + cache_ptr->max_dirty_index_size = (size_t)0; + + cache_ptr->max_slist_len = 0; + cache_ptr->max_slist_size = (size_t)0; + + cache_ptr->max_pl_len = 0; + cache_ptr->max_pl_size = (size_t)0; + + cache_ptr->max_pel_len = 0; + cache_ptr->max_pel_size = (size_t)0; + + cache_ptr->calls_to_msic = 0; + cache_ptr->total_entries_skipped_in_msic = 0; + cache_ptr->total_entries_scanned_in_msic = 0; + cache_ptr->max_entries_skipped_in_msic = 0; + cache_ptr->max_entries_scanned_in_msic = 0; + cache_ptr->entries_scanned_to_make_space = 0; + + cache_ptr->slist_scan_restarts = 0; + cache_ptr->LRU_scan_restarts = 0; + cache_ptr->hash_bucket_scan_restarts = 0; + +#if H5C_COLLECT_CACHE_ENTRY_STATS + for(i = 0; i <= cache_ptr->max_type_id; i++) { + cache_ptr->max_accesses[i] = 0; + cache_ptr->min_accesses[i] = 1000000; + cache_ptr->max_clears[i] = 0; + cache_ptr->max_flushes[i] = 0; + cache_ptr->max_size[i] = (size_t)0; + cache_ptr->max_pins[i] = 0; + } /* end for */ + +#endif /* H5C_COLLECT_CACHE_ENTRY_STATS */ +#endif /* H5C_COLLECT_CACHE_STATS */ + +} /* H5C_stats__reset() */ + +extern void +H5C__dump_entry(H5C_t *cache_ptr, const H5C_cache_entry_t *entry_ptr, + hbool_t dump_parents, const char *prefix, int indent); + +static void +H5C__dump_parents(H5C_t *cache_ptr, const H5C_cache_entry_t *entry_ptr, const char *prefix, int indent) +{ + unsigned u; + + for(u = 0; u < entry_ptr->flush_dep_nparents; u++) + H5C__dump_entry(cache_ptr, entry_ptr->flush_dep_parent[u], TRUE, prefix, indent + 2); +} + +typedef struct H5C__dump_child_ctx_t { + H5C_t *cache_ptr; + const H5C_cache_entry_t *parent; + hbool_t dump_parents; + const char *prefix; + int indent; +} H5C__dump_child_ctx_t; + +static int +H5C__dump_children_cb(H5C_cache_entry_t *entry_ptr, void *_ctx) +{ + H5C__dump_child_ctx_t *ctx = (H5C__dump_child_ctx_t *)_ctx; + + if(entry_ptr->tag_info->tag != entry_ptr->addr) { + unsigned u; + + HDassert(entry_ptr->flush_dep_nparents); + for(u = 0; u < entry_ptr->flush_dep_nparents; u++) + if(ctx->parent == entry_ptr->flush_dep_parent[u]) + H5C__dump_entry(ctx->cache_ptr, entry_ptr, ctx->dump_parents, ctx->prefix, ctx->indent + 2); + } /* end if */ + + return(H5_ITER_CONT); +} + +static void +H5C__dump_children(H5C_t *cache_ptr, const H5C_cache_entry_t *entry_ptr, + hbool_t dump_parents, const char *prefix, int indent) +{ + H5C__dump_child_ctx_t ctx; + + HDassert(entry_ptr->tag_info); + + ctx.cache_ptr = cache_ptr; + ctx.parent = entry_ptr; + ctx.dump_parents = dump_parents; + ctx.prefix = prefix; + ctx.indent = indent; + H5C__iter_tagged_entries(cache_ptr, entry_ptr->tag_info->tag, FALSE, H5C__dump_children_cb, &ctx); +} + +void +H5C__dump_entry(H5C_t *cache_ptr, const H5C_cache_entry_t *entry_ptr, + hbool_t dump_parents, const char *prefix, int indent) +{ + HDassert(cache_ptr); + HDassert(entry_ptr); + + HDfprintf(stderr, "%*s%s: entry_ptr = (%a, '%s', %a, %t, %u, %u/%u)\n", indent, "", prefix, entry_ptr->addr, entry_ptr->type->name, entry_ptr->tag_info ? entry_ptr->tag_info->tag : HADDR_UNDEF, entry_ptr->is_dirty, entry_ptr->flush_dep_nparents, entry_ptr->flush_dep_nchildren, entry_ptr->flush_dep_ndirty_children); + if(dump_parents && entry_ptr->flush_dep_nparents) + H5C__dump_parents(cache_ptr, entry_ptr, "Parent", indent); + if(entry_ptr->flush_dep_nchildren) + H5C__dump_children(cache_ptr, entry_ptr, FALSE, "Child", indent); +} diff --git a/src/H5Cepoch.c b/src/H5Cepoch.c index 7c5e441..3726aa1 100644 --- a/src/H5Cepoch.c +++ b/src/H5Cepoch.c @@ -63,22 +63,23 @@ * epochs so that they can be evicted from the cache. * ****************************************************************************/ -static herr_t H5C__epoch_marker_get_load_size(const void *udata_ptr, +static herr_t H5C__epoch_marker_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr); +static herr_t H5C__epoch_marker_get_final_load_size(const void *image_ptr, + size_t image_len_ptr, void *udata_ptr, size_t *actual_len); +static htri_t H5C__epoch_marker_verify_chksum(const void *image_ptr, + size_t len, void *udata_ptr); static void * H5C__epoch_marker_deserialize(const void * image_ptr, size_t len, void * udata, hbool_t * dirty_ptr); static herr_t H5C__epoch_marker_image_len(const void * thing, - size_t *image_len_ptr, hbool_t *compressed_ptr, size_t *compressed_len_ptr); + size_t *image_len_ptr); static herr_t H5C__epoch_marker_pre_serialize(const H5F_t *f, hid_t dxpl_id, void * thing, haddr_t addr, size_t len, - size_t compressed_len, haddr_t * new_addr_ptr, size_t * new_len_ptr, - size_t * new_compressed_len_ptr, unsigned * flags_ptr); + haddr_t * new_addr_ptr, size_t * new_len_ptr, unsigned * flags_ptr); static herr_t H5C__epoch_marker_serialize(const H5F_t *f, void * image_ptr, size_t len, void * thing); static herr_t H5C__epoch_marker_notify(H5C_notify_action_t action, void *thing); static herr_t H5C__epoch_marker_free_icr(void * thing); -static herr_t H5C__epoch_marker_clear(const H5F_t *f, void * thing, - hbool_t about_to_destroy); static herr_t H5C__epoch_marker_fsf_size(const void H5_ATTR_UNUSED * thing, size_t H5_ATTR_UNUSED * fsf_size_ptr); @@ -104,14 +105,15 @@ const H5C_class_t H5C__epoch_marker_class = /* name = */ "epoch marker", /* mem_type = */ H5FD_MEM_DEFAULT, /* value doesn't matter */ /* flags = */ H5C__CLASS_NO_FLAGS_SET, - /* get_load_size = */ H5C__epoch_marker_get_load_size, + /* get_initial_load_size = */ H5C__epoch_marker_get_initial_load_size, + /* get_final_load_size = */ H5C__epoch_marker_get_final_load_size, + /* verify_chksum = */ H5C__epoch_marker_verify_chksum, /* deserialize = */ H5C__epoch_marker_deserialize, /* image_len = */ H5C__epoch_marker_image_len, /* pre_serialize = */ H5C__epoch_marker_pre_serialize, /* serialize = */ H5C__epoch_marker_serialize, /* notify = */ H5C__epoch_marker_notify, /* free_icr = */ H5C__epoch_marker_free_icr, - /* clear = */ H5C__epoch_marker_clear, /* fsf_size = */ H5C__epoch_marker_fsf_size, }; @@ -127,7 +129,7 @@ const H5C_class_t H5C__epoch_marker_class = static herr_t -H5C__epoch_marker_get_load_size(const void H5_ATTR_UNUSED *udata_ptr, +H5C__epoch_marker_get_initial_load_size(void H5_ATTR_UNUSED *udata_ptr, size_t H5_ATTR_UNUSED *image_len_ptr) { FUNC_ENTER_STATIC_NOERR /* Yes, even though this pushes an error on the stack */ @@ -135,7 +137,32 @@ H5C__epoch_marker_get_load_size(const void H5_ATTR_UNUSED *udata_ptr, HERROR(H5E_CACHE, H5E_SYSTEM, "called unreachable fcn."); FUNC_LEAVE_NOAPI(FAIL) -} /* end H5C__epoch_marker_get_load_size() */ +} /* end H5C__epoch_marker_get_initial_load_size() */ + + +static herr_t +H5C__epoch_marker_get_final_load_size(const void H5_ATTR_UNUSED *image_ptr, + size_t H5_ATTR_UNUSED image_len, void H5_ATTR_UNUSED *udata_ptr, + size_t H5_ATTR_UNUSED *actual_len) +{ + FUNC_ENTER_STATIC_NOERR /* Yes, even though this pushes an error on the stack */ + + HERROR(H5E_CACHE, H5E_SYSTEM, "called unreachable fcn."); + + FUNC_LEAVE_NOAPI(FAIL) +} /* end H5C__epoch_marker_final_get_load_size() */ + + +static htri_t +H5C__epoch_marker_verify_chksum(const void H5_ATTR_UNUSED *image_ptr, size_t H5_ATTR_UNUSED len, + void H5_ATTR_UNUSED *udata_ptr) +{ + FUNC_ENTER_STATIC_NOERR /* Yes, even though this pushes an error on the stack */ + + HERROR(H5E_CACHE, H5E_SYSTEM, "called unreachable fcn."); + + FUNC_LEAVE_NOAPI(FALSE) +} /* end H5C__epoch_marker_verify_chksum() */ static void * @@ -152,8 +179,7 @@ H5C__epoch_marker_deserialize(const void H5_ATTR_UNUSED * image_ptr, size_t H5_A static herr_t H5C__epoch_marker_image_len(const void H5_ATTR_UNUSED *thing, - size_t H5_ATTR_UNUSED *image_len_ptr, hbool_t H5_ATTR_UNUSED *compressed_ptr, - size_t H5_ATTR_UNUSED *compressed_len_ptr) + size_t H5_ATTR_UNUSED *image_len_ptr) { FUNC_ENTER_STATIC_NOERR /* Yes, even though this pushes an error on the stack */ @@ -166,8 +192,7 @@ H5C__epoch_marker_image_len(const void H5_ATTR_UNUSED *thing, static herr_t H5C__epoch_marker_pre_serialize(const H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id, void H5_ATTR_UNUSED *thing, haddr_t H5_ATTR_UNUSED addr, size_t H5_ATTR_UNUSED len, - size_t H5_ATTR_UNUSED compressed_len, haddr_t H5_ATTR_UNUSED *new_addr_ptr, - size_t H5_ATTR_UNUSED *new_len_ptr, size_t H5_ATTR_UNUSED *new_compressed_len_ptr, + haddr_t H5_ATTR_UNUSED *new_addr_ptr, size_t H5_ATTR_UNUSED *new_len_ptr, unsigned H5_ATTR_UNUSED *flags_ptr) { FUNC_ENTER_STATIC_NOERR /* Yes, even though this pushes an error on the stack */ @@ -214,17 +239,6 @@ H5C__epoch_marker_free_icr(void H5_ATTR_UNUSED * thing) static herr_t -H5C__epoch_marker_clear(const H5F_t H5_ATTR_UNUSED *f, void H5_ATTR_UNUSED * thing, hbool_t H5_ATTR_UNUSED about_to_destroy) -{ - FUNC_ENTER_STATIC_NOERR /* Yes, even though this pushes an error on the stack */ - - HERROR(H5E_CACHE, H5E_SYSTEM, "called unreachable fcn."); - - FUNC_LEAVE_NOAPI(FAIL) -} /* end H5C__epoch_marker_clear() */ - - -static herr_t H5C__epoch_marker_fsf_size(const void H5_ATTR_UNUSED * thing, size_t H5_ATTR_UNUSED *fsf_size_ptr) { FUNC_ENTER_STATIC_NOERR /* Yes, even though this pushes an error on the stack */ diff --git a/src/H5Clog.c b/src/H5Clog.c new file mode 100644 index 0000000..e3e4388 --- /dev/null +++ b/src/H5Clog.c @@ -0,0 +1,370 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5Clog.c + * May 30 2016 + * Quincey Koziol + * + * Purpose: Functions for generic cache logging in JSON format + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ +#include "H5Cmodule.h" /* This source code file is part of the H5C module */ + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#ifdef H5_HAVE_PARALLEL +#define H5AC_FRIEND /*suppress error about including H5ACpkg */ +#include "H5ACpkg.h" /* Metadata cache */ +#endif /* H5_HAVE_PARALLEL */ +#include "H5Cpkg.h" /* Metadata cache */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5MMprivate.h" /* Memory management */ + + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + + + +/*------------------------------------------------------------------------- + * Function: H5C_set_up_logging + * + * Purpose: Setup for metadata cache logging. + * + * Metadata logging is enabled and disabled at two levels. This + * function and the associated tear_down function open and close + * the log file. the start_ and stop_logging functions are then + * used to switch logging on/off. Optionally, logging can begin + * as soon as the log file is opened (set via the start_immediately + * parameter to this function). + * + * The log functionality is split between the H5C and H5AC + * packages. Log state and direct log manipulation resides in + * H5C. Log messages are generated in H5AC and sent to + * the H5C_write_log_message function. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_set_up_logging(H5C_t *cache_ptr, const char log_location[], + hbool_t start_immediately) +{ +#ifdef H5_HAVE_PARALLEL + H5AC_aux_t *aux_ptr = NULL; +#endif /*H5_HAVE_PARALLEL*/ + char *file_name = NULL; + size_t n_chars; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + if(NULL == cache_ptr) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cache_ptr == NULL") + if(H5C__H5C_T_MAGIC != cache_ptr->magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cache magic value incorrect") + if(cache_ptr->logging_enabled) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "logging already set up") + if(NULL == log_location) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL log location not allowed") + + /* Possibly fix up the log file name. + * The extra 39 characters are for adding the rank to the file name + * under parallel HDF5. 39 characters allows > 2^127 processes which + * should be enough for anybody. + * + * allocation size = <path length> + dot + <rank # length> + \0 + */ + n_chars = HDstrlen(log_location) + 1 + 39 + 1; + if(NULL == (file_name = (char *)H5MM_calloc(n_chars * sizeof(char)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate memory for mdc log file name manipulation") + +#ifdef H5_HAVE_PARALLEL + /* Add the rank to the log file name when MPI is in use */ + aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); + + if(NULL == aux_ptr) + HDsnprintf(file_name, n_chars, "%s", log_location); + else { + if(aux_ptr->magic != H5AC__H5AC_AUX_T_MAGIC) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "bad aux_ptr->magic") + HDsnprintf(file_name, n_chars, "%s.%d", log_location, aux_ptr->mpi_rank); + } /* end else */ +#else /* H5_HAVE_PARALLEL */ + HDsnprintf(file_name, n_chars, "%s", log_location); +#endif /* H5_HAVE_PARALLEL */ + + /* Open log file */ + if(NULL == (cache_ptr->log_file_ptr = HDfopen(file_name, "w"))) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "can't create mdc log file") + + /* Set logging flags */ + cache_ptr->logging_enabled = TRUE; + cache_ptr->currently_logging = start_immediately; + + done: + if(file_name) + file_name = (char *)H5MM_xfree(file_name); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_set_up_logging() */ + + +/*------------------------------------------------------------------------- + * Function: H5C_tear_down_logging + * + * Purpose: Tear-down for metadata cache logging. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_tear_down_logging(H5C_t *cache_ptr) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + if(NULL == cache_ptr) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cache_ptr == NULL") + if(H5C__H5C_T_MAGIC != cache_ptr->magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cache magic value incorrect") + if(FALSE == cache_ptr->logging_enabled) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "logging not enabled") + + /* Unset logging flags */ + cache_ptr->logging_enabled = FALSE; + cache_ptr->currently_logging = FALSE; + + /* Close log file */ + if(EOF == HDfclose(cache_ptr->log_file_ptr)) + HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "problem closing mdc log file") + cache_ptr->log_file_ptr = NULL; + + done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_tear_down_logging() */ + + +/*------------------------------------------------------------------------- + * Function: H5C_start_logging + * + * Purpose: Start logging metadata cache operations. + * + * TODO: Add a function that dumps the current state of the + * metadata cache. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_start_logging(H5C_t *cache_ptr) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + if(NULL == cache_ptr) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cache_ptr == NULL") + if(H5C__H5C_T_MAGIC != cache_ptr->magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cache magic value incorrect") + if(FALSE == cache_ptr->logging_enabled) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "logging not enabled") + if(cache_ptr->currently_logging) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "logging already in progress") + + /* Set logging flags */ + cache_ptr->currently_logging = TRUE; + + /* TODO - Dump cache state */ + + done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_start_logging() */ + + +/*------------------------------------------------------------------------- + * Function: H5C_stop_logging + * + * Purpose: Stop logging metadata cache operations. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_stop_logging(H5C_t *cache_ptr) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + if(NULL == cache_ptr) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cache_ptr == NULL") + if(H5C__H5C_T_MAGIC != cache_ptr->magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cache magic value incorrect") + if(FALSE == cache_ptr->logging_enabled) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "logging not enabled") + if(FALSE == cache_ptr->currently_logging) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "logging not in progress") + + /* Set logging flags */ + cache_ptr->currently_logging = FALSE; + + done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_stop_logging() */ + + +/*------------------------------------------------------------------------- + * Function: H5C_get_logging_status + * + * Purpose: Determines if the cache is actively logging (via the OUT + * parameter). + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_get_logging_status(const H5C_t *cache_ptr, /*OUT*/ hbool_t *is_enabled, + /*OUT*/ hbool_t *is_currently_logging) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + if(NULL == cache_ptr) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cache_ptr == NULL") + if(H5C__H5C_T_MAGIC != cache_ptr->magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cache magic value incorrect") + if(NULL == is_enabled) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cache magic value incorrect") + if(NULL == is_currently_logging) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cache magic value incorrect") + + *is_enabled = cache_ptr->logging_enabled; + *is_currently_logging = cache_ptr->currently_logging; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_get_logging_status() */ + + +/*------------------------------------------------------------------------- + * Function: H5C_write_log_message + * + * Purpose: Write a message to the log file and flush the file. + * The message string is neither modified nor freed. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Dana Robinson + * Sunday, March 16, 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_write_log_message(const H5C_t *cache_ptr, const char message[]) +{ + size_t n_chars; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + if(NULL == cache_ptr) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cache_ptr == NULL") + if(H5C__H5C_T_MAGIC != cache_ptr->magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "cache magic value incorrect") + if(FALSE == cache_ptr->currently_logging) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "not currently logging") + if(NULL == message) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL log message not allowed") + + /* Write the log message and flush */ + n_chars = HDstrlen(message); + if((int)n_chars != HDfprintf(cache_ptr->log_file_ptr, message)) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error writing log message") + if(EOF == HDfflush(cache_ptr->log_file_ptr)) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error flushing log message") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_write_log_message() */ + diff --git a/src/H5Cmpio.c b/src/H5Cmpio.c index 4e88e44..ab94879 100644 --- a/src/H5Cmpio.c +++ b/src/H5Cmpio.c @@ -64,9 +64,7 @@ /********************/ /* Local Prototypes */ /********************/ -static herr_t H5C__collective_write(H5F_t *f, hid_t dxpl_id, - H5SL_t *collective_write_list); -static herr_t H5C__collective_write_free(void *_item, void *key, void *op_data); +static herr_t H5C__collective_write(H5F_t *f, hid_t dxpl_id); /*********************/ @@ -83,9 +81,6 @@ static herr_t H5C__collective_write_free(void *_item, void *key, void *op_data); /* Local Variables */ /*******************/ -/* Declare a free list to manage the H5C_collective_write_t struct */ -H5FL_DEFINE(H5C_collective_write_t); - /*------------------------------------------------------------------------- @@ -230,7 +225,6 @@ H5C_apply_candidate_list(H5F_t * f, H5C_cache_entry_t * entry_ptr = NULL; H5C_cache_entry_t * flush_ptr = NULL; H5C_cache_entry_t * delayed_ptr = NULL; - H5SL_t * collective_write_list = NULL; #if H5C_DO_SANITY_CHECKS haddr_t last_addr; #endif /* H5C_DO_SANITY_CHECKS */ @@ -264,8 +258,11 @@ H5C_apply_candidate_list(H5F_t * f, #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ if(f->coll_md_write) { + /* Sanity check */ + HDassert(NULL == cache_ptr->coll_write_list); + /* Create skip list of entries for collective write */ - if(NULL == (collective_write_list = H5SL_create(H5SL_TYPE_HADDR, NULL))) + if(NULL == (cache_ptr->coll_write_list = H5SL_create(H5SL_TYPE_HADDR, NULL))) HGOTO_ERROR(H5E_DATASET, H5E_CANTCREATE, FAIL, "can't create skip list for entries") } /* end if */ @@ -454,13 +451,37 @@ H5C_apply_candidate_list(H5F_t * f, (long long)clear_ptr->addr); #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - /* No need to check for the next entry in the scan being - * removed from the cache, as this call to H5C__flush_single_entry() - * will not call either the pre_serialize or serialize callbacks. + /* reset entries_removed_counter and + * last_entry_removed_ptr prior to the call to + * H5C__flush_single_entry() so that we can spot + * unexpected removals of entries from the cache, + * and set the restart_scan flag if proceeding + * would be likely to cause us to scan an entry + * that is no longer in the cache. + * + * Note that as of this writing (April 2015) this + * case cannot occur in the parallel case. However + * Quincey is making noises about changing this, hence + * the insertion of this test. + * + * Note also that there is no test code to verify + * that this code actually works (although similar code + * in the serial version exists and is tested). + * + * Implementing a test will likely require implementing + * flush op like facilities in the parallel tests. At + * a guess this will not be terribly painful, but it + * will take a bit of time. */ + cache_ptr->entries_removed_counter = 0; + cache_ptr->last_entry_removed_ptr = NULL; - if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, NULL, NULL) < 0) + if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't clear entry.") + + if((cache_ptr->entries_removed_counter > 1) || + (cache_ptr->last_entry_removed_ptr == entry_ptr)) + restart_scan = TRUE; } /* end if */ /* Else, if this process needs to flush this entry. */ @@ -505,14 +526,12 @@ H5C_apply_candidate_list(H5F_t * f, cache_ptr->last_entry_removed_ptr = NULL; /* Add this entry to the list of entries to collectively write */ - if(H5C__flush_single_entry(f, dxpl_id, flush_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, NULL, collective_write_list) < 0) + if(H5C__flush_single_entry(f, dxpl_id, flush_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush entry.") - if ( ( cache_ptr->entries_removed_counter > 1 ) || - ( cache_ptr->last_entry_removed_ptr == entry_ptr ) ) - + if((cache_ptr->entries_removed_counter > 1) || + (cache_ptr->last_entry_removed_ptr == entry_ptr)) restart_scan = TRUE; - } /* end else-if */ /* Otherwise, no action to be taken on this entry. Grab the next. */ @@ -663,7 +682,7 @@ H5C_apply_candidate_list(H5F_t * f, (long long)clear_ptr->addr); #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, NULL, NULL) < 0) + if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't clear entry.") } /* end else-if */ @@ -680,7 +699,7 @@ H5C_apply_candidate_list(H5F_t * f, #endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ /* Add this entry to the list of entries to collectively write */ - if(H5C__flush_single_entry(f, dxpl_id, flush_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, NULL, collective_write_list) < 0) + if(H5C__flush_single_entry(f, dxpl_id, flush_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't clear entry.") } /* end else-if */ } /* end if */ @@ -714,14 +733,14 @@ H5C_apply_candidate_list(H5F_t * f, if (delayed_ptr) { if (delayed_ptr->clear_on_unprotect) { - if(H5C__flush_single_entry(f, dxpl_id, delayed_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG, NULL, NULL) < 0) + if(H5C__flush_single_entry(f, dxpl_id, delayed_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__GENERATE_IMAGE_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush entry.") entry_ptr->clear_on_unprotect = FALSE; entries_cleared++; } else if (delayed_ptr->flush_immediately) { /* Add this entry to the list of entries to collectively write */ - if(H5C__flush_single_entry(f, dxpl_id, delayed_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, NULL, collective_write_list) < 0) + if(H5C__flush_single_entry(f, dxpl_id, delayed_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush entry collectively.") entry_ptr->flush_immediately = FALSE; @@ -734,10 +753,11 @@ H5C_apply_candidate_list(H5F_t * f, /* If we've deferred writing to do it collectively, take care of that now */ if(f->coll_md_write) { - HDassert(collective_write_list); + /* Sanity check */ + HDassert(cache_ptr->coll_write_list); /* Write collective list */ - if(H5C__collective_write(f, dxpl_id, collective_write_list) < 0) + if(H5C__collective_write(f, dxpl_id) < 0) HGOTO_ERROR(H5E_CACHE, H5E_WRITEERROR, FAIL, "Can't write metadata collectively") } /* end if */ @@ -760,9 +780,11 @@ done: if(candidate_assignment_table != NULL) candidate_assignment_table = (int *)H5MM_xfree((void *)candidate_assignment_table); - if(collective_write_list) - if(H5SL_destroy(collective_write_list, H5C__collective_write_free, NULL) < 0) + if(cache_ptr->coll_write_list) { + if(H5SL_close(cache_ptr->coll_write_list) < 0) HDONE_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "failed to destroy skip list") + cache_ptr->coll_write_list = NULL; + } /* end if */ FUNC_LEAVE_NOAPI(ret_value) } /* H5C_apply_candidate_list() */ @@ -1001,7 +1023,7 @@ done: * * Note that unlike H5C_apply_candidate_list(), * H5C_mark_entries_as_clean() makes all its calls to - * H6C_flush_single_entry() with the + * H5C__flush_single_entry() with the * H5C__FLUSH_CLEAR_ONLY_FLAG set. As a result, * the pre_serialize() and serialize calls are not made. * @@ -1160,7 +1182,7 @@ H5C_mark_entries_as_clean(H5F_t * f, * * Note that unlike H5C_apply_candidate_list(), * H5C_mark_entries_as_clean() makes all its calls to - * H6C_flush_single_entry() with the H5C__FLUSH_CLEAR_ONLY_FLAG + * H5C__flush_single_entry() with the H5C__FLUSH_CLEAR_ONLY_FLAG * set. As a result, the pre_serialize() and serialize calls are * not made. * @@ -1195,7 +1217,7 @@ H5C_mark_entries_as_clean(H5F_t * f, entry_ptr = entry_ptr->prev; entries_cleared++; - if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, NULL, NULL) < 0) + if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't clear entry.") } else { @@ -1223,7 +1245,7 @@ H5C_mark_entries_as_clean(H5F_t * f, entry_ptr = entry_ptr->next; entries_cleared++; - if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, NULL, NULL) < 0 ) + if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0 ) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't clear entry.") } else { @@ -1331,8 +1353,9 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5C__collective_write(H5F_t *f, hid_t dxpl_id, H5SL_t *collective_write_list) +H5C__collective_write(H5F_t *f, hid_t dxpl_id) { + H5AC_t *cache_ptr; H5P_genplist_t *plist = NULL; H5FD_mpio_xfer_t orig_xfer_mode = H5FD_MPIO_COLLECTIVE; int count; @@ -1348,6 +1371,12 @@ H5C__collective_write(H5F_t *f, hid_t dxpl_id, H5SL_t *collective_write_list) FUNC_ENTER_STATIC + /* Sanity checks */ + HDassert(f != NULL); + cache_ptr = f->shared->cache; + HDassert(cache_ptr != NULL); + HDassert(cache_ptr->coll_write_list != NULL); + /* Get original transfer mode */ if(NULL == (plist = (H5P_genplist_t *)H5I_object(dxpl_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data transfer property list") @@ -1355,12 +1384,12 @@ H5C__collective_write(H5F_t *f, hid_t dxpl_id, H5SL_t *collective_write_list) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set MPI-I/O property") /* Get number of entries in collective write list */ - count = (int)H5SL_count(collective_write_list); + count = (int)H5SL_count(cache_ptr->coll_write_list); if(count > 0) { H5FD_mpio_xfer_t xfer_mode = H5FD_MPIO_COLLECTIVE; H5SL_node_t *node; - H5C_collective_write_t *item; + H5C_cache_entry_t *entry_ptr; void *base_buf; int i; @@ -1376,27 +1405,27 @@ H5C__collective_write(H5F_t *f, hid_t dxpl_id, H5SL_t *collective_write_list) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "memory allocation failed for collective offset table length array") /* Fill arrays */ - node = H5SL_first(collective_write_list); + node = H5SL_first(cache_ptr->coll_write_list); HDassert(node); - if(NULL == (item = (H5C_collective_write_t *)H5SL_item(node))) + if(NULL == (entry_ptr = (H5C_cache_entry_t *)H5SL_item(node))) HGOTO_ERROR(H5E_CACHE, H5E_NOTFOUND, FAIL, "can't retrieve skip list item") /* Set up initial array position & buffer base address */ - length_array[0] = (int)item->length; - base_buf = item->buf; + length_array[0] = (int)entry_ptr->size; + base_buf = entry_ptr->image_ptr; buf_array[0] = (MPI_Aint)0; - offset_array[0] = (MPI_Aint)item->offset; + offset_array[0] = (MPI_Aint)entry_ptr->addr; node = H5SL_next(node); i = 1; while(node) { - if(NULL == (item = (H5C_collective_write_t *)H5SL_item(node))) + if(NULL == (entry_ptr = (H5C_cache_entry_t *)H5SL_item(node))) HGOTO_ERROR(H5E_CACHE, H5E_NOTFOUND, FAIL, "can't retrieve skip list item") /* Set up array position */ - length_array[i] = (int)item->length; - buf_array[i] = (MPI_Aint)item->buf - (MPI_Aint)base_buf; - offset_array[i] = (MPI_Aint)item->offset; + length_array[i] = (int)entry_ptr->size; + buf_array[i] = (MPI_Aint)entry_ptr->image_ptr - (MPI_Aint)base_buf; + offset_array[i] = (MPI_Aint)entry_ptr->addr; /* Advance to next node & array location */ node = H5SL_next(node); @@ -1469,36 +1498,5 @@ done: FUNC_LEAVE_NOAPI(ret_value); } /* end H5C__collective_write() */ - - -/*------------------------------------------------------------------------- - * - * Function: H5C__collective_write_free - * - * Purpose: Release node on collective write skiplist - * - * Return: FAIL if error is detected, SUCCEED otherwise. - * - * Programmer: Mohamad Chaarawi - * February, 2016 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5C__collective_write_free(void *_item, void H5_ATTR_UNUSED *key, void H5_ATTR_UNUSED *op_data) -{ - H5C_collective_write_t *item = (H5C_collective_write_t *)_item; - - FUNC_ENTER_STATIC_NOERR - - /* Sanity check */ - HDassert(item); - - if(item->free_buf) - item->buf = H5MM_xfree(item->buf); - H5FL_FREE(H5C_collective_write_t, item); - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5C__collective_write_free() */ #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h index cf2da3e..fae4d74 100644 --- a/src/H5Cpkg.h +++ b/src/H5Cpkg.h @@ -524,21 +524,17 @@ if ( ( (entry_ptr) == NULL ) || \ (cache_ptr)->max_pel_size = (cache_ptr)->pel_size; #define H5C__UPDATE_STATS_FOR_MOVE(cache_ptr, entry_ptr) \ - if ( cache_ptr->flush_in_progress ) { \ + if ( cache_ptr->flush_in_progress ) \ ((cache_ptr)->cache_flush_moves[(entry_ptr)->type->id])++; \ - } \ - if ( entry_ptr->flush_in_progress ) { \ + if ( entry_ptr->flush_in_progress ) \ ((cache_ptr)->entry_flush_moves[(entry_ptr)->type->id])++; \ - } \ (((cache_ptr)->moves)[(entry_ptr)->type->id])++; #define H5C__UPDATE_STATS_FOR_ENTRY_SIZE_CHANGE(cache_ptr, entry_ptr, new_size)\ - if ( cache_ptr->flush_in_progress ) { \ + if ( cache_ptr->flush_in_progress ) \ ((cache_ptr)->cache_flush_size_changes[(entry_ptr)->type->id])++; \ - } \ - if ( entry_ptr->flush_in_progress ) { \ + if ( entry_ptr->flush_in_progress ) \ ((cache_ptr)->entry_flush_size_changes[(entry_ptr)->type->id])++; \ - } \ if ( (entry_ptr)->size < (new_size) ) { \ ((cache_ptr)->size_increases[(entry_ptr)->type->id])++; \ H5C__UPDATE_MAX_INDEX_SIZE_STATS(cache_ptr) \ @@ -1002,7 +998,6 @@ if ( ( (cache_ptr) == NULL ) || \ ( (cache_ptr)->index_size <= 0 ) || \ ( (new_size) <= 0 ) || \ ( (old_size) > (cache_ptr)->index_size ) || \ - ( (new_size) <= 0 ) || \ ( ( (cache_ptr)->index_len == 1 ) && \ ( (cache_ptr)->index_size != (old_size) ) ) || \ ( (cache_ptr)->index_size != \ @@ -1175,19 +1170,17 @@ if ( ( (cache_ptr)->index_size != \ int k; \ H5C__PRE_HT_INSERT_SC(cache_ptr, entry_ptr, fail_val) \ k = H5C__HASH_FCN((entry_ptr)->addr); \ - if ( ((cache_ptr)->index)[k] == NULL ) \ - ((cache_ptr)->index)[k] = (entry_ptr); \ - else { \ + if(((cache_ptr)->index)[k] != NULL) { \ (entry_ptr)->ht_next = ((cache_ptr)->index)[k]; \ (entry_ptr)->ht_next->ht_prev = (entry_ptr); \ - ((cache_ptr)->index)[k] = (entry_ptr); \ } \ + ((cache_ptr)->index)[k] = (entry_ptr); \ (cache_ptr)->index_len++; \ (cache_ptr)->index_size += (entry_ptr)->size; \ ((cache_ptr)->index_ring_len[entry_ptr->ring])++; \ ((cache_ptr)->index_ring_size[entry_ptr->ring]) \ - += (entry_ptr)->size; \ - if ( (entry_ptr)->is_dirty ) { \ + += (entry_ptr)->size; \ + if((entry_ptr)->is_dirty) { \ (cache_ptr)->dirty_index_size += (entry_ptr)->size; \ ((cache_ptr)->dirty_index_ring_size[entry_ptr->ring]) \ += (entry_ptr)->size; \ @@ -1221,8 +1214,8 @@ if ( ( (cache_ptr)->index_size != \ (cache_ptr)->index_size -= (entry_ptr)->size; \ ((cache_ptr)->index_ring_len[entry_ptr->ring])--; \ ((cache_ptr)->index_ring_size[entry_ptr->ring]) \ - -= (entry_ptr)->size; \ - if ( (entry_ptr)->is_dirty ) { \ + -= (entry_ptr)->size; \ + if((entry_ptr)->is_dirty) { \ (cache_ptr)->dirty_index_size -= (entry_ptr)->size; \ ((cache_ptr)->dirty_index_ring_size[entry_ptr->ring]) \ -= (entry_ptr)->size; \ @@ -1231,7 +1224,7 @@ if ( ( (cache_ptr)->index_size != \ ((cache_ptr)->clean_index_ring_size[entry_ptr->ring]) \ -= (entry_ptr)->size; \ } \ - if ((entry_ptr)->flush_me_last) { \ + if((entry_ptr)->flush_me_last) { \ (cache_ptr)->num_last_entries--; \ HDassert((cache_ptr)->num_last_entries <= 1); \ } \ @@ -1506,40 +1499,15 @@ if ( ( (cache_ptr)->index_size != \ * * Programmer: John Mainzer, 5/10/04 * - * Modifications: - * - * JRM -- 7/21/04 - * Updated function for the addition of the hash table. - * - * JRM - 7/27/04 - * Converted from the function H5C_remove_entry_from_tree() - * to the macro H5C__REMOVE_ENTRY_FROM_TREE in the hopes of - * wringing a little more performance out of the cache. - * - * QAK -- 11/27/04 - * Switched over to using skip list routines. - * - * JRM -- 3/28/07 - * Updated sanity checks for the new is_read_only and - * ro_ref_count fields in H5C_cache_entry_t. - * - * JRM -- 12/13/14 - * Added code to set cache_ptr->slist_changed to TRUE - * when an entry is removed from the slist. - * - * JRM -- 9/1/15 - * Added code to maintain the cache_ptr->slist_ring_len - * and cache_ptr->slist_ring_size arrays. - * *------------------------------------------------------------------------- */ -#define H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr) \ +#if H5C_DO_SANITY_CHECKS +#define H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush) \ { \ HDassert( (cache_ptr) ); \ HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ HDassert( (entry_ptr) ); \ - HDassert( !((entry_ptr)->is_protected) ); \ HDassert( !((entry_ptr)->is_read_only) ); \ HDassert( ((entry_ptr)->ro_ref_count) == 0 ); \ HDassert( (entry_ptr)->size > 0 ); \ @@ -1558,7 +1526,46 @@ if ( ( (cache_ptr)->index_size != \ "Can't delete entry from skip list.") \ \ HDassert( (cache_ptr)->slist_len > 0 ); \ - (cache_ptr)->slist_changed = TRUE; \ + if(!(during_flush)) \ + (cache_ptr)->slist_changed = TRUE; \ + (cache_ptr)->slist_len--; \ + HDassert( (cache_ptr)->slist_size >= (entry_ptr)->size ); \ + (cache_ptr)->slist_size -= (entry_ptr)->size; \ + ((cache_ptr)->slist_ring_len[(entry_ptr)->ring])--; \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr->ring)] >= \ + (entry_ptr)->size ); \ + ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) -= (entry_ptr)->size; \ + (cache_ptr)->slist_len_increase--; \ + (cache_ptr)->slist_size_increase -= (int64_t)((entry_ptr)->size); \ + (entry_ptr)->in_slist = FALSE; \ +} /* H5C__REMOVE_ENTRY_FROM_SLIST */ + +#else /* H5C_DO_SANITY_CHECKS */ + +#define H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush) \ +{ \ + HDassert( (cache_ptr) ); \ + HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ + HDassert( (entry_ptr) ); \ + HDassert( !((entry_ptr)->is_read_only) ); \ + HDassert( ((entry_ptr)->ro_ref_count) == 0 ); \ + HDassert( (entry_ptr)->in_slist ); \ + HDassert( (cache_ptr)->slist_ptr ); \ + HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ + HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ + HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_len ); \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_size ); \ + \ + if ( H5SL_remove((cache_ptr)->slist_ptr, &(entry_ptr)->addr) \ + != (entry_ptr) ) \ + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, \ + "Can't delete entry from skip list.") \ + \ + HDassert( (cache_ptr)->slist_len > 0 ); \ + if(!(during_flush)) \ + (cache_ptr)->slist_changed = TRUE; \ (cache_ptr)->slist_len--; \ HDassert( (cache_ptr)->slist_size >= (entry_ptr)->size ); \ (cache_ptr)->slist_size -= (entry_ptr)->size; \ @@ -1568,6 +1575,7 @@ if ( ( (cache_ptr)->index_size != \ ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) -= (entry_ptr)->size; \ (entry_ptr)->in_slist = FALSE; \ } /* H5C__REMOVE_ENTRY_FROM_SLIST */ +#endif /* H5C_DO_SANITY_CHECKS */ /*------------------------------------------------------------------------- @@ -2388,12 +2396,11 @@ if ( ( (cache_ptr)->index_size != \ HDassert( (cache_ptr) ); \ HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ HDassert( (entry_ptr) ); \ - HDassert( !((entry_ptr)->is_protected) ); \ HDassert( !((entry_ptr)->is_read_only) ); \ HDassert( ((entry_ptr)->ro_ref_count) == 0 ); \ HDassert( (entry_ptr)->size > 0 ); \ \ - if ( ! ( (entry_ptr)->is_pinned ) ) { \ + if ( ! ( (entry_ptr)->is_pinned ) && ! ( (entry_ptr->is_protected ) ) ) { \ \ /* modified LRU specific code */ \ \ @@ -2466,12 +2473,11 @@ if ( ( (cache_ptr)->index_size != \ HDassert( (cache_ptr) ); \ HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ HDassert( (entry_ptr) ); \ - HDassert( !((entry_ptr)->is_protected) ); \ HDassert( !((entry_ptr)->is_read_only) ); \ HDassert( ((entry_ptr)->ro_ref_count) == 0 ); \ HDassert( (entry_ptr)->size > 0 ); \ \ - if ( ! ( (entry_ptr)->is_pinned ) ) { \ + if ( ! ( (entry_ptr)->is_pinned ) && ! ( (entry_ptr->is_protected ) ) ) { \ \ /* modified LRU specific code */ \ \ @@ -3146,6 +3152,40 @@ if ( ( (entry_ptr) == NULL ) || \ /**************************************************************************** * + * structure H5C_tag_info_t + * + * Structure about each set of tagged entries for an object in the file. + * + * Each H5C_tag_info_t struct corresponds to a particular object in the file. + * + * Each H5C_cache_entry struct in the linked list of entries for this tag + * also contains a pointer back to the H5C_tag_info_t struct for the + * overall object. + * + * + * The fields of this structure are discussed individually below: + * + * tag: Address (i.e. "tag") of the object header for all the entries + * corresponding to parts of that object. + * + * head: Head of doubly-linked list of all entries belonging to the tag. + * + * entry_cnt: Number of entries on linked list of entries for this tag. + * + * corked: Boolean flag indicating whether entries for this object can be + * evicted. + * + ****************************************************************************/ +typedef struct H5C_tag_info_t { + haddr_t tag; /* Tag (address) of the entries (must be first, for skiplist) */ + H5C_cache_entry_t *head; /* Head of the list of entries for this tag */ + size_t entry_cnt; /* Number of entries on list */ + hbool_t corked; /* Whether this object is corked */ +} H5C_tag_info_t; + + +/**************************************************************************** + * * structure H5C_t * * Catchall structure for all variables specific to an instance of the cache. @@ -3201,6 +3241,29 @@ if ( ( (entry_ptr) == NULL ) || \ * When we get to using H5C in other places, we may add * code to write trace file data at the H5C level as well. * + * logging_enabled: Boolean flag indicating whether cache logging + * which is used to record cache operations for use in + * debugging and performance analysis. When this flag is set + * to TRUE, it means that the log file is open and ready to + * receive log entries. It does NOT mean that cache operations + * are currently being recorded. That is controlled by the + * currently_logging flag (below). + * + * Since much of the code supporting the parallel metadata + * cache is in H5AC, we don't write the trace file from + * H5C. Instead, H5AC reads the trace_file_ptr as needed. + * + * When we get to using H5C in other places, we may add + * code to write trace file data at the H5C level as well. + * + * currently_logging: Boolean flag that indicates if cache operations are + * currently being logged. This flag is flipped by the + * H5Fstart/stop_mdc_logging functions. + * + * log_file_ptr: File pointer pointing to the log file. The I/O functions + * in stdio.h are used to write to the log file regardless of + * the VFD selected. + * * aux_ptr: Pointer to void used to allow wrapper code to associate * its data with an instance of H5C_t. The H5C cache code * sets this field to NULL, and otherwise leaves it alone. @@ -3363,7 +3426,7 @@ if ( ( (entry_ptr) == NULL ) || \ * entry is removed from the cache by any means (eviction, * expungement, or take ownership at this point in time). * Functions that perform scans on lists may set this field - * to zero prior to calling H5C_flush_single_entry(). + * to zero prior to calling H5C__flush_single_entry(). * Unexpected changes to the counter indicate that an entry * was removed from the cache as a side effect of the flush. * @@ -3371,7 +3434,7 @@ if ( ( (entry_ptr) == NULL ) || \ * which contained the last entry to be removed from the cache, * or NULL if there either is no such entry, or if a function * performing a scan of a list has set this field to NULL prior - * to calling H5C_flush_single_entry(). + * to calling H5C__flush_single_entry(). * * WARNING!!! This field must NEVER be dereferenced. It is * maintained to allow functions that perform scans of lists @@ -3380,14 +3443,13 @@ if ( ( (entry_ptr) == NULL ) || \ * pointers don't match, and if entries_removed_counter is * one. * + * entry_watched_for_removal: Pointer to an instance of H5C_cache_entry_t + * which contains the 'next' entry for an iteration. Removing + * this entry must trigger a rescan of the iteration, so each + * entry removed from the cache is compared against this pointer + * and the pointer is reset to NULL if the watched entry is removed. + * (This functions similarly to a "dead man's switch") * - * With the addition of cache entry tagging, it is possible that - * an entry may be inserted into the cache without a tag during testing - * and the tag's validity shouldn't be checked. - * - * The following field is maintained to facilitate this. - * - * ignore_tags: Boolean flag to disable tag validation during entry insertion. * * When we flush the cache, we need to write entries out in increasing * address order. An instance of a skip list is used to store dirty entries in @@ -3406,14 +3468,6 @@ if ( ( (entry_ptr) == NULL ) || \ * have modified the slist -- which obliges us to restart * the scan of the slist from the beginning. * - * slist_change_in_pre_serialize: Boolean flag used to indicate that - * a pre_serialize call has modified the slist since the - * last time this flag was reset. - * - * slist_change_in_serialize: Boolean flag used to indicate that - * a serialize call has modified the slist since the - * last time this flag was reset. - * * slist_len: Number of entries currently in the skip list * used to maintain a sorted list of dirty entries in the * cache. @@ -3473,11 +3527,22 @@ if ( ( (entry_ptr) == NULL ) || \ * to the slist since the last time this field was set to * zero. Note that this value can be negative. * - * cork_list_ptr: A skip list to track object addresses that are corked. - * When an entry is inserted or protected in the cache, - * the entry's associated object address (tag field) is - * checked against this skip list. If found, the entry - * is corked. + * Cache entries belonging to a particular object are "tagged" with that + * object's base object header address. + * + * The following fields are maintained to facilitate this. + * + * tag_list: A skip list to track entries that belong to an object. + * Each H5C_tag_info_t struct on the tag list corresponds to + * a particular object in the file. Tagged entries can be + * flushed or evicted as a group, or corked to prevent entries + * from being evicted from the cache. + * + * "Global" entries, like the superblock and the file's + * freelist, as well as shared entries like global + * heaps and shared object header messages, are not tagged. + * + * ignore_tags: Boolean flag to disable tag validation during entry insertion. * * When a cache entry is protected, it must be removed from the LRU * list(s) as it cannot be either flushed or evicted until it is unprotected. @@ -3702,13 +3767,6 @@ if ( ( (entry_ptr) == NULL ) || \ * all the ways this can happen, we simply set this flag when * we receive a new configuration. * - * cache_full: Boolean flag used to keep track of whether the cache is - * full, so we can refrain from increasing the size of a - * cache which hasn't used up the space allotted to it. - * - * The field is initialized to FALSE, and then set to TRUE - * whenever we attempt to make space in the cache. - * * resize_enabled: This is another convenience flag which is set whenever * a new set of values for resize_ctl are provided. Very * simply, @@ -3716,6 +3774,13 @@ if ( ( (entry_ptr) == NULL ) || \ * resize_enabled = size_increase_possible || * size_decrease_possible; * + * cache_full: Boolean flag used to keep track of whether the cache is + * full, so we can refrain from increasing the size of a + * cache which hasn't used up the space allotted to it. + * + * The field is initialized to FALSE, and then set to TRUE + * whenever we attempt to make space in the cache. + * * size_decreased: Boolean flag set to TRUE whenever the maximum cache * size is decreased. The flag triggers a call to * H5C_make_space_in_cache() on the next call to H5C_protect(). @@ -4000,17 +4065,17 @@ if ( ( (entry_ptr) == NULL ) || \ * obtain estimates of how frequently these restarts occur. * * slist_scan_restarts: Number of times a scan of the slist (that contains - * calls to H5C_flush_single_entry()) has been restarted to + * calls to H5C__flush_single_entry()) has been restarted to * avoid potential issues with change of status of the next * entry in the scan. * * LRU_scan_restarts: Number of times a scan of the LRU list (that contains - * calls to H5C_flush_single_entry()) has been restarted to + * calls to H5C__flush_single_entry()) has been restarted to * avoid potential issues with change of status of the next * entry in the scan. * * hash_bucket_scan_restarts: Number of times a scan of a hash bucket list - * (that contains calls to H5C_flush_single_entry()) has been + * (that contains calls to H5C__flush_single_entry()) has been * restarted to avoid potential issues with change of status * of the next entry in the scan. * @@ -4059,6 +4124,9 @@ struct H5C_t { uint32_t magic; hbool_t flush_in_progress; FILE * trace_file_ptr; + hbool_t logging_enabled; + hbool_t currently_logging; + FILE * log_file_ptr; void * aux_ptr; int32_t max_type_id; const char * (* type_name_table_ptr); @@ -4078,19 +4146,15 @@ struct H5C_t { size_t clean_index_ring_size[H5C_RING_NTYPES]; size_t dirty_index_size; size_t dirty_index_ring_size[H5C_RING_NTYPES]; - H5C_cache_entry_t * (index[H5C__HASH_TABLE_LEN]); + H5C_cache_entry_t * index[H5C__HASH_TABLE_LEN]; /* Fields to detect entries removed during scans */ int64_t entries_removed_counter; H5C_cache_entry_t * last_entry_removed_ptr; - - /* Field to disable tag validation */ - hbool_t ignore_tags; + H5C_cache_entry_t * entry_watched_for_removal; /* Fields for maintaining list of in-order entries, for flushing */ hbool_t slist_changed; - hbool_t slist_change_in_pre_serialize; - hbool_t slist_change_in_serialize; int32_t slist_len; size_t slist_size; int32_t slist_ring_len[H5C_RING_NTYPES]; @@ -4102,7 +4166,9 @@ struct H5C_t { int64_t slist_size_increase; #endif /* H5C_DO_SANITY_CHECKS */ - H5SL_t * cork_list_ptr; /* list of corked object addresses */ + /* Fields for maintaining list of tagged entries */ + H5SL_t * tag_list; + hbool_t ignore_tags; /* Fields for tracking protected entries */ int32_t pl_len; @@ -4135,10 +4201,14 @@ struct H5C_t { H5C_cache_entry_t * dLRU_tail_ptr; #ifdef H5_HAVE_PARALLEL + /* Fields for collective metadata reads */ int32_t coll_list_len; size_t coll_list_size; H5C_cache_entry_t * coll_head_ptr; H5C_cache_entry_t * coll_tail_ptr; + + /* Fields for collective metadata writes */ + H5SL_t * coll_write_list; #endif /* H5_HAVE_PARALLEL */ /* Fields for automatic cache size adjustment */ @@ -4214,7 +4284,7 @@ struct H5C_t { int32_t max_pel_len; size_t max_pel_size; - /* Fields for tacking 'make space in cache' (msic) operations */ + /* Fields for tracking 'make space in cache' (msic) operations */ int64_t calls_to_msic; int64_t total_entries_skipped_in_msic; int64_t total_entries_scanned_in_msic; @@ -4240,15 +4310,6 @@ struct H5C_t { char prefix[H5C__PREFIX_LEN]; }; -#ifdef H5_HAVE_PARALLEL -typedef struct H5C_collective_write_t { - size_t length; - hbool_t free_buf; - void *buf; - haddr_t offset; -} H5C_collective_write_t; -#endif /* H5_HAVE_PARALLEL */ - /* Define typedef for tagged cache entry iteration callbacks */ typedef int (*H5C_tag_iter_cb_t)(H5C_cache_entry_t *entry, void *ctx); @@ -4267,16 +4328,15 @@ H5_DLLVAR const H5C_class_t H5C__epoch_marker_class; /* General routines */ H5_DLL herr_t H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, - H5C_cache_entry_t *entry_ptr, unsigned flags, int64_t *entry_size_change_ptr, H5SL_t *collective_write_list); + H5C_cache_entry_t *entry_ptr, unsigned flags); H5_DLL herr_t H5C__flush_marked_entries(H5F_t * f, hid_t dxpl_id); -H5_DLL int H5C__iter_tagged_entries(H5C_t *cache, haddr_t tag, hbool_t match_global, +H5_DLL herr_t H5C__iter_tagged_entries(H5C_t *cache, haddr_t tag, hbool_t match_global, H5C_tag_iter_cb_t cb, void *cb_ctx); /* Routines for operating on entry tags */ H5_DLL herr_t H5C__tag_entry(H5C_t * cache_ptr, H5C_cache_entry_t * entry_ptr, hid_t dxpl_id); -H5_DLL herr_t H5C__mark_tagged_entries_cork(H5C_t *cache_ptr, haddr_t obj_addr, - hbool_t val); +H5_DLL herr_t H5C__untag_entry(H5C_t *cache, H5C_cache_entry_t *entry); /* Testing functions */ #ifdef H5C_TESTING diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index 975ea3e..ed75bec 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -41,7 +41,7 @@ /**************************/ /* Cache configuration settings */ -#define H5C__MAX_NUM_TYPE_IDS 28 +#define H5C__MAX_NUM_TYPE_IDS 29 #define H5C__PREFIX_LEN 32 /* This sanity checking constant was picked out of the air. Increase @@ -68,17 +68,14 @@ /* Flags for cache client class behavior */ #define H5C__CLASS_NO_FLAGS_SET ((unsigned)0x0) #define H5C__CLASS_SPECULATIVE_LOAD_FLAG ((unsigned)0x1) -#define H5C__CLASS_COMPRESSED_FLAG ((unsigned)0x2) /* The following flags may only appear in test code */ -#define H5C__CLASS_NO_IO_FLAG ((unsigned)0x4) -#define H5C__CLASS_SKIP_READS ((unsigned)0x8) -#define H5C__CLASS_SKIP_WRITES ((unsigned)0x10) +#define H5C__CLASS_SKIP_READS ((unsigned)0x2) +#define H5C__CLASS_SKIP_WRITES ((unsigned)0x4) /* Flags for pre-serialize callback */ #define H5C__SERIALIZE_NO_FLAGS_SET ((unsigned)0) #define H5C__SERIALIZE_RESIZED_FLAG ((unsigned)0x1) #define H5C__SERIALIZE_MOVED_FLAG ((unsigned)0x2) -#define H5C__SERIALIZE_COMPRESSED_FLAG ((unsigned)0x4) /* Upper and lower limits on cache size. These limits are picked * out of a hat -- you should be able to change them as necessary. @@ -172,12 +169,16 @@ * These flags apply to H5C_expunge_entry(): * H5C__FREE_FILE_SPACE_FLAG * + * These flags apply to H5C_evict(): + * H5C__EVICT_ALLOW_LAST_PINS_FLAG + * * These flags apply to H5C_flush_cache(): * H5C__FLUSH_INVALIDATE_FLAG * H5C__FLUSH_CLEAR_ONLY_FLAG * H5C__FLUSH_MARKED_ENTRIES_FLAG * H5C__FLUSH_IGNORE_PROTECTED_FLAG (can't use this flag in combination * with H5C__FLUSH_INVALIDATE_FLAG) + * H5C__DURING_FLUSH_FLAG * * These flags apply to H5C_flush_single_entry(): * H5C__FLUSH_INVALIDATE_FLAG @@ -185,29 +186,27 @@ * H5C__FLUSH_MARKED_ENTRIES_FLAG * H5C__TAKE_OWNERSHIP_FLAG * H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG + * H5C__GENERATE_IMAGE_FLAG */ -#define H5C__NO_FLAGS_SET 0x0000 -#define H5C__SET_FLUSH_MARKER_FLAG 0x0001 -#define H5C__DELETED_FLAG 0x0002 -#define H5C__DIRTIED_FLAG 0x0004 -#define H5C__PIN_ENTRY_FLAG 0x0008 -#define H5C__UNPIN_ENTRY_FLAG 0x0010 -#define H5C__FLUSH_INVALIDATE_FLAG 0x0020 -#define H5C__FLUSH_CLEAR_ONLY_FLAG 0x0040 -#define H5C__FLUSH_MARKED_ENTRIES_FLAG 0x0080 -#define H5C__FLUSH_IGNORE_PROTECTED_FLAG 0x0100 -#define H5C__READ_ONLY_FLAG 0x0200 -#define H5C__FREE_FILE_SPACE_FLAG 0x0400 -#define H5C__TAKE_OWNERSHIP_FLAG 0x0800 -#define H5C__FLUSH_LAST_FLAG 0x1000 -#define H5C__FLUSH_COLLECTIVELY_FLAG 0x2000 -#define H5C__EVICT_ALLOW_LAST_PINS_FLAG 0x4000 -#define H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG 0x8000 - -/* Definitions for cache "tag" property */ -#define H5C_TAG_NAME "H5C_tag" -#define H5C_TAG_SIZE sizeof(H5C_tag_t) -#define H5C_TAG_DEF {(haddr_t)0, H5C_GLOBALITY_NONE} +#define H5C__NO_FLAGS_SET 0x00000 +#define H5C__SET_FLUSH_MARKER_FLAG 0x00001 +#define H5C__DELETED_FLAG 0x00002 +#define H5C__DIRTIED_FLAG 0x00004 +#define H5C__PIN_ENTRY_FLAG 0x00008 +#define H5C__UNPIN_ENTRY_FLAG 0x00010 +#define H5C__FLUSH_INVALIDATE_FLAG 0x00020 +#define H5C__FLUSH_CLEAR_ONLY_FLAG 0x00040 +#define H5C__FLUSH_MARKED_ENTRIES_FLAG 0x00080 +#define H5C__FLUSH_IGNORE_PROTECTED_FLAG 0x00100 +#define H5C__READ_ONLY_FLAG 0x00200 +#define H5C__FREE_FILE_SPACE_FLAG 0x00400 +#define H5C__TAKE_OWNERSHIP_FLAG 0x00800 +#define H5C__FLUSH_LAST_FLAG 0x01000 +#define H5C__FLUSH_COLLECTIVELY_FLAG 0x02000 +#define H5C__EVICT_ALLOW_LAST_PINS_FLAG 0x04000 +#define H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG 0x08000 +#define H5C__DURING_FLUSH_FLAG 0x10000 /* Set when the entire cache is being flushed */ +#define H5C__GENERATE_IMAGE_FLAG 0x20000 /* Set during parallel I/O */ /* Debugging/sanity checking/statistics settings */ #ifndef NDEBUG @@ -271,19 +270,6 @@ /* Typedef for the main structure for the cache (defined in H5Cpkg.h) */ typedef struct H5C_t H5C_t; -/* Define enum for cache entry tag 'globality' value */ -typedef enum { - H5C_GLOBALITY_NONE=0, /* Non-global tag */ - H5C_GLOBALITY_MINOR, /* global, not flushed during single object flush */ - H5C_GLOBALITY_MAJOR /* global, needs flushed during single obect flush */ -} H5C_tag_globality_t; - -/* Cache entry tag structure */ -typedef struct H5C_tag_t { - haddr_t value; - H5C_tag_globality_t globality; -} H5C_tag_t; - /* * * Struct H5C_class_t @@ -303,19 +289,12 @@ typedef struct H5C_tag_t { * * flags: Flags indicating class-specific behavior. * - * Whoever created the flags field neglected to document the meanings - * of the flags he created. Hence the following discussion of the - * H5C__CLASS_SPECULATIVE_LOAD_FLAG and (to a lesser extent) - * H5C__CLASS_COMPRESSED_FLAG should be viewed with suspicion, - * as the meanings are divined from the source code, and thus may be - * inaccurate. Please correct any errors you find. - * * Possible flags are: * * H5C__CLASS_NO_FLAGS_SET: No special processing. * - * H5C__CLASS_SPECULATIVE_LOAD_FLAG: This flag appears to be used - * only in H5C_load_entry(). When it is set, entries are + * H5C__CLASS_SPECULATIVE_LOAD_FLAG: This flag is used only in + * H5C_load_entry(). When it is set, entries are * permitted to change their sizes on the first attempt * to load. * @@ -329,64 +308,17 @@ typedef struct H5C_tag_t { * entry in the cache. * * If the new size is smaller than the old, no new loads - * or desearializes are performed, but the new size becomes + * or deserializes are performed, but the new size becomes * the size of the entry in the cache. * * When this flag is set, an attempt to read past the - * end of file is pemitted. In this case, if the size + * end of file could occur. In this case, if the size * returned get_load_size callback would result in a - * read past the end of file, the size is trunkated to + * read past the end of file, the size is truncated to * avoid this, and processing proceeds as normal. * - * H5C__CLASS_COMPRESSED_FLAG: This flags indicates that the entry - * may be compressed -- i.e. its on disk image is run through - * filters on the way to and from disk. Thus the uncompressed - * (or unfiltered) size of the entry may be different from the - * size of the entry in file. - * - * This has the following implications: - * - * On load, uncompressed size and load size may be different. - * Presumably, load size will be smaller than uncompressed - * size, but there is no requirement for this in the code - * (but note that I have inserted an assertion to this effect, - * which has not been triggered to date). - * - * On insertion, compressed (AKA filtered, AKA on disk) size - * is unknown, as the entry has yet to be run through filters. - * Compressed size is computed whenever the entry is - * written (or the image is updated -- not relevant until - * journaling is brought back). - * - * On dirty (of a clean entry), compressed (AKA filtered, - * AKA on disk) size becomes unknown. Thus, compressed size - * must be computed by the pre-serialize callback before the - * entry may be written. - * - * Once the compressed size becomes unknown, it remains so - * until the on disk image is constructed. - * - * Observe that the cache needs to know the size of the entry - * for space allocation purposes. Since the compressed size - * can change or become unknown, it uses the uncompressed - * size which may change, but which should always be known. - * The compressed size is used only for journaling and disk I/O. - * - * While there is no logical reason why they could not be - * combined, due to absence of need and for simplicity of code, - * the cache does not permit both the the - * H5C__CLASS_COMPRESSED_FLAG and the - * H5C__CLASS_SPECULATIVE_LOAD_FLAG to be set in the same - * instance of H5C_class_t. - * * The following flags may only appear in test code. * - * H5C__CLASS_NO_IO_FLAG: This flag is intended only for use in test - * code. When it is set, any attempt to load an entry of - * the type with this flag set will trigger an assertion - * failure, and any flush of an entry with this flag set - * will not result in any write to file. - * * H5C__CLASS_SKIP_READS: This flags is intended only for use in test * code. When it is set, reads on load will be skipped, * and an uninitialize buffer will be passed to the @@ -396,54 +328,124 @@ typedef struct H5C_tag_t { * code. When it is set, writes of buffers prepared by the * serialize callback will be skipped. * - * GET_LOAD_SIZE: Pointer to the 'get load size' function. + * GET_INITIAL_LOAD_SIZE: Pointer to the 'get initial load size' function. * - * This function must be able to determine the size of the disk image of - * a metadata cache entry, given the 'udata' that will be passed to the - * 'deserialize' callback. + * This function determines the size based on the information in the + * parameter "udata" or an initial speculative guess. The size is + * returned in the parameter "image_len_ptr". * - * Note that if either the H5C__CLASS_SPECULATIVE_LOAD_FLAG or - * the H5C__CLASS_COMPRESSED_FLAG is set, the disk image size - * returned by this callback is either a first guess (if the - * H5C__CLASS_SPECULATIVE_LOAD_FLAG is set) or (if the - * H5C__CLASS_COMPRESSED_FLAG is set), the exact on disk size - * of the entry whether it has been run through filters or not. - * In all other cases, the value returned should be the correct - * uncompressed size of the entry. + * For an entry with H5C__CLASS_NO_FLAGS_SET: + * This function returns in "image_len_ptr" the on disk size of the + * entry. + * + * For an entry with H5C__CLASS_SPECULATIVE_LOAD_FLAG: + * This function returns in "image_len_ptr" an initial guess of the + * entry's on disk size. This many bytes will be loaded from + * the file and then passed to 'get_final_load_size' callback + * for the actual (final) image length to be determined. * - * The typedef for the deserialize callback is as follows: + * The typedef for the get_initial_load_size callback is as follows: * - * typedef herr_t (*H5C_get_load_size_func_t)(void *udata_ptr, - * size_t *image_len_ptr); + * typedef herr_t (*H5C_get_initial_load_size_func_t)(void *udata_ptr, + * size_t *image_len_ptr); * - * The parameters of the deserialize callback are as follows: + * The parameters of the get_initial_load_size callback are as follows: * * udata_ptr: Pointer to user data provided in the protect call, which - * will also be passed through to the deserialize callback. + * will also be passed through to the 'get_final_load_size', + * 'verify_chksum', and 'deserialize' callbacks. * - * image_len_ptr: Pointer to the location in which the length in bytes - * of the in file image to be deserialized is to be returned. + * image_len_ptr: Pointer to the length in bytes of the in-file image to + * be deserialized is to be returned. * * This value is used by the cache to determine the size of * the disk image for the metadata, in order to read the disk * image from the file. - * + * * Processing in the get_load_size function should proceed as follows: * - * If successful, the function will place the length of the on disk - * image associated with supplied user data in *image_len_ptr, and - * then return SUCCEED. + * If successful, the function will place the length in the *image_len_ptr + * associated with supplied user data and then return SUCCEED. * * On failure, the function must return FAIL and push error information * onto the error stack with the error API routines, without modifying - * the value pointed to by the image_len_ptr. + * the value pointed to by image_len_ptr. + * + * + * GET_FINAL_LOAD_SIZE: Pointer to the 'get final load size' function. + * + * This function determines the final size of a speculatively loaded + * metadata cache entry based on the parameter "image" and the "udata" + * parameters. This callback _must_ be implemented for cache clients + * which set the H5C__CLASS_SPECULATIVE_LOAD_FLAG and must return the + * actual length of on-disk image after being called once. + * + * This function might deserialize the needed metadata information to + * determine the actual size. The size is returned in the parameter + * "actual_len_ptr". + * + * The typedef for the get_load_size callback is as follows: + * + * typedef herr_t (*H5C_get_final_load_size_func_t)(const void *image_ptr, + * size_t image_len, + * void *udata_ptr, + * size_t *actual_len_ptr); + * + * The parameters of the get_load_size callback are as follows: + * + * image_ptr: Pointer to a buffer containing the (possibly partial) + * metadata read in. + * + * image_len: The length in bytes of the (possibly partial) in-file image + * to be queried for an actual length. + * + * udata_ptr: Pointer to user data provided in the protect call, which + * will also be passed through to the 'verify_chksum' and + * 'deserialize' callbacks. + * + * actual_len_ptr: Pointer to the location containing the actual length + * of the metadata entry on disk. + * + * Processing in the get_final_load_size function should proceed as follows: + * + * If successful, the function will place the length in the *actual_len_ptr + * associated with supplied image and/or user data and then return SUCCEED. + * + * On failure, the function must return FAIL and push error information + * onto the error stack with the error API routines, without modifying + * the value pointed to by actual_len_ptr. + * + * + * VERIFY_CHKSUM: Pointer to the verify_chksum function. + * + * This function verifies the checksum computed for the metadata is + * the same as the checksum stored in the metadata. + * + * It computes the checksum based on the metadata stored in the + * parameter "image_ptr" and the actual length of the metadata in the + * parameter "len" which is obtained from the "get_load_size" callback. + * + * The typedef for the verify_chksum callback is as follows: + * + * typedef htri_t (*H5C_verify_chksum_func_t)(const void *image_ptr, + * size_t len, + * void *udata_ptr); + * + * The parameters of the verify_chksum callback are as follows: + * + * image_ptr: Pointer to a buffer containing the metadata read in. + * + * len: The actual length of the metadata. + * + * udata_ptr: Pointer to user data. * * * DESERIALIZE: Pointer to the deserialize function. * - * This function must be able to read a buffer containing the on disk - * image of a metadata cache entry, allocate and load the equivalent - * in core representation, and return a pointer to that representation. + * This function must be able to deserialize a buffer containing the + * on-disk image of a metadata cache entry, allocate and initialize the + * equivalent in core representation, and return a pointer to that + * representation. * * The typedef for the deserialize callback is as follows: * @@ -476,17 +478,17 @@ typedef struct H5C_tag_t { * Processing in the deserialize function should proceed as follows: * * If the image contains valid data, and is of the correct length, - * the deserialize function must allocate space for an in core - * representation of that data, load the contents of the image into - * the space allocated for the in core representation, and return + * the deserialize function must allocate space for an in-core + * representation of that data, deserialize the contents of the image + * into the space allocated for the in-core representation, and return * a pointer to the in core representation. Observe that an * instance of H5C_cache_entry_t must be the first item in this * representation. The cache will initialize it after the callback * returns. * - * Note that the structure of the in core representation is otherwise + * Note that the structure of the in-core representation is otherwise * up to the cache client. All that is required is that the pointer - * returned be sufficient for the clients purposes when it is returned + * returned be sufficient for the client's purposes when it is returned * on a protect call. * * If the deserialize function has to clean up file corruption @@ -498,61 +500,16 @@ typedef struct H5C_tag_t { * buffer length, malloc failure, etc.) the function must return NULL and * push error information on the error stack with the error API routines. * - * Exceptions to the above: - * - * If the H5C__CLASS_SPECULATIVE_LOAD_FLAG is set, the buffer supplied - * to the function need not be currect on the first invocation of the - * callback in any single attempt to load the entry. - * - * In this case, if the buffer is larger than necessary, the function - * should load the entry as described above and not flag an error due - * to the oversized buffer. The cache will correct its mis-apprehension - * of the entry size with a subsequent call to the image_len callback. - * - * If the buffer is too small, and this is the first deserialize call - * in the entry load operation, the function should not flag an error. - * Instead, it must compute the correct size of the entry, allocate an - * in core representation and initialize it to the extent that an - * immediate call to the image len callback will return the correct - * image size. - * - * In this case, when the deserialize callback returns, the cache will - * call the image length callback, realize that the supplied buffer was - * too small, discard the returned in core representation, allocate - * and load a new buffer of the correct size from file, and then call - * the deserialize callback again. - * - * If the H5C__CLASS_COMPRESSED_FLAG is set, exceptions are as per the - * H5C__CLASS_SPECULATIVE_LOAD_FLAG, save that only oversized buffers - * are permitted. - * * * IMAGE_LEN: Pointer to the image length callback. * - * This callback exists primarily to support - * H5C__CLASS_SPECULATIVE_LOAD_FLAG and H5C__CLASS_COMPRESSED_FLAG - * discussed above, although it is also used to obtain the size of - * newly inserted entries. - * - * In the case of the H5C__CLASS_SPECULATIVE_LOAD_FLAG, it is used to - * allow the client to change the size of an entry in the deserialize - * callback. - * - * For the H5C__CLASS_COMPRESSED_FLAG, it is used to allow the client - * to indicate whether the entry is compressed (i.e. whether entries - * are run through filters) and if so, to report both the uncompressed - * and the compressed entry size (i.e. the actual on disk size after - * the entry has been run through filters) if that value is known. - * - * The callback is also used in H5C_insert_entry() to obtain the - * size of the newly inserted entry. + * The image_len callback is used to obtain the size of newly inserted + * entries and assert verification. * * The typedef for the image_len callback is as follows: * * typedef herr_t (*H5C_image_len_func_t)(void *thing, - * size_t *image_len_ptr, - * hbool_t *compressed_ptr, - * size_t *compressed_image_len_ptr); + * size_t *image_len_ptr); * * The parameters of the image_len callback are as follows: * @@ -561,84 +518,16 @@ typedef struct H5C_tag_t { * image_len_ptr: Pointer to size_t in which the callback will return * the length (in bytes) of the cache entry. * - * If the H5C__CLASS_COMPRESSED_FLAG is not set in the - * associated instance of H5C_class_t, or if the flag is - * set, and the callback sets *compressed_ptr to FALSE, - * this size is the actual size of the entry on disk. - * - * Otherwise, this size is the uncompressed size of the - * entry -- which the cache will use for all purposes OTHER - * than journal writes and disk I/O. - * - * compressed_ptr: Pointer to a boolean flag indicating whether - * the cache entry will be compressed / uncompressed on - * disk writes / reads. - * - * If the H5C__CLASS_COMPRESSED_FLAG is not set in the - * associated instance of H5C_class_t, *compressed_ptr - * must be set to FALSE. - * - * If the H5C__CLASS_COMPRESSED_FLAG is set in the - * associated instance of H5C_class_t, and filters are - * not enabled, *compressed_ptr must be set to FALSE. - * - * If the H5C__CLASS_COMPRESSED_FLAG is set in the - * associated instance of H5C_class_t, and filters are - * enabled, the callback must set *compressed_ptr to TRUE. - * - * Note that *compressed_ptr will always be set to FALSE - * by the caller prior to invocation of the callback. Thus - * callbacks for clients that don't set the - * H5C__CLASS_COMPRESSED_FLAG can ignore this parameter. - * - * compressed_image_len_ptr: Pointer to size_t in which the callback - * may return the length (in bytes) of the compressed on - * disk image of the entry, or the uncompressed size if the - * compressed size has not yet been calculated. - * - * Since computing the compressed image len is expensive, - * the callback should only report the most recently computed - * value -- which will typically be incorrect if the entry - * is dirty. - * - * If *compressed_ptr is set to FALSE, *compressed_image_len_ptr - * should be set to zero. However, as *compressed_image_len_ptr - * will be initialize to zero prior to the call, the callback - * need not modify it if the H5C__CLASS_COMPRESSED_FLAG is - * not set. - * - * If the H5C__CLASS_COMPRESSED_FLAG is not set in the associated - * instance of H5C_class_t, processing in the image_len function - * should proceed as follows: + * Processing in the image_len function should proceed as follows: * * If successful, the function will place the length of the on disk * image associated with the in core representation provided in the - * thing parameter in *image_len_ptr, and then return SUCCEED. Since - * *compressed_ptr and *compressed_image_len_ptr will be initialized to - * FALSE and zero respectively before the call, the callback need not - * modify these values, and may declare the associated parameters as - * UNUSED. - * - * If the H5C__CLASS_COMPRESSED_FLAG is set in the associated - * instance of H5C_class_t, processing in the image_len function - * should proceed as follows: - * - * If successful, the function will place the uncompressed length of - * the on disk image associated with the in core representation - * provided in the thing parameter in *image_len_ptr. If filters - * are not enabled for the entry, it will set *compressed_ptr to FALSE, - * and *compressed_image_len_ptr to zero. If filters are enabled, - * it will set *compressed_ptr to TRUE. In this case, it must set - * *compressed_image_len_ptr equal to the last computed compressed - * if the compressed size, or to the uncompressed size if that value - * is yet to be computed. In all cases, it will return SUCCEED if - * successful. - * - * In either case, if the function fails, it must return FAIL and - * push error information onto the error stack with the error API - * routines, and return without modifying the values pointed to by - * the image_len_ptr, compressed_ptr, and compressed_image_len_ptr - * parameters. + * thing parameter in *image_len_ptr, and then return SUCCEED. + * + * If the function fails, it must return FAIL and push error information + * onto the error stack with the error API routines, and return without + * modifying the values pointed to by the image_len_ptr parameter. + * * * PRE_SERIALIZE: Pointer to the pre-serialize callback. * @@ -646,11 +535,11 @@ typedef struct H5C_tag_t { * it needs a current on-disk image of the metadata entry for purposes * either constructing a journal or flushing the entry to disk. * - * If the client needs to change the address or compressed or - * uncompressed length of the entry prior to flush, the pre-serialize - * callback is responsible for these actions, so that the actual - * serialize callback (described below) is only responsible for - * serializing the data structure, not moving it on disk or resizing it. + * If the client needs to change the address or length of the entry prior + * to flush, the pre-serialize callback is responsible for these actions, + * so that the actual serialize callback (described below) is only + * responsible for serializing the data structure, not moving it on disk + * or resizing it. * * In addition, the client may use the pre-serialize callback to * ensure that the entry is ready to be flushed -- in particular, @@ -664,7 +553,7 @@ typedef struct H5C_tag_t { * However, that need not be the case as free space section info * entries will change size (and possibly location) depending on the * number of blocks of free space being manages, and fractal heap - * direct blocks can change compressed size (and possibly location) + * direct blocks can change compressed size (and possibly location) * on serialization if compression is enabled. Similarly, it may * be necessary to move entries from temporary to real file space. * @@ -678,10 +567,8 @@ typedef struct H5C_tag_t { * void * thing, * haddr_t addr, * size_t len, - * size_t compressed_len, * haddr_t * new_addr_ptr, * size_t * new_len_ptr, - * size_t * new_compressed_len_ptr, * unsigned * flags_ptr); * * The parameters of the pre-serialize callback are as follows: @@ -709,22 +596,8 @@ typedef struct H5C_tag_t { * * len: Length in bytes of the in file image of the entry to be * serialized. Also the size the image passed to the - * serialize callback (discussed below) unless either that - * value is altered by this function, or the entry will be - * compressed. In the latter case, the compressed size - * of the entry will be reported in *new_compressed_len_ptr. - * - * This parameter is supplied mainly for sanity checking. - * Sanity checks should be performed when compiled in debug - * mode, but the parameter may be unused when compiled in - * production mode. - * - * compressed_len: If the entry is to be compressed (i.e. run through - * filters) prior to flush, Length in bytes of the last know - * compressed size of the entry -- or the uncompressed size - * if no such value exists (i.e. the entry has been inserted, - * but never flushed). This parameter should be set to zero - * in all other cases. + * serialize callback (discussed below) unless that + * value is altered by this function. * * This parameter is supplied mainly for sanity checking. * Sanity checks should be performed when compiled in debug @@ -749,20 +622,12 @@ typedef struct H5C_tag_t { * *new_len_ptr is undefined on pre-serialize callback * return. * - * new_compressed_len_ptr: Pointer to size_t. If the image will be - * compressed (i.e. run through filters) prior to being - * written to disk, the compressed size (in bytes) of the - * on disk image must be stored in *new_compressed_len_ptr, - * and the appropriate flag set in *flags_ptr. - * * flags_ptr: Pointer to an unsigned integer used to return flags * indicating whether the preserialize function resized or moved - * the entry, or computed its compressed size. If the entry was - * neither resized or moved, nor will be compressed, - * the serialize function must set *flags_ptr to zero. - * H5C__SERIALIZE_RESIZED_FLAG, H5C__SERIALIZE_MOVED_FLAG - * and H5C__SERIALIZE_COMPRESSED_FLAG must be set to indicate - * a resize, a move, or compression respectively. + * the entry. If the entry was neither resized or moved, the + * serialize function must set *flags_ptr to zero. The + * H5C__SERIALIZE_RESIZED_FLAG or H5C__SERIALIZE_MOVED_FLAG must + * be set to indicate a resize or move respectively. * * If the H5C__SERIALIZE_RESIZED_FLAG is set, the new length * must be stored in *new_len_ptr. @@ -770,40 +635,21 @@ typedef struct H5C_tag_t { * If the H5C__SERIALIZE_MOVED_FLAG flag is set, the * new image base address must be stored in *new_addr_ptr. * - * If the H5C__SERIALIZE_COMPRESSED_FLAG is set, the - * compressed size of the new image must be stored in - * *new_compressed_len_ptr. - * * Processing in the pre-serialize function should proceed as follows: * * The pre-serialize function must examine the in core representation * indicated by the thing parameter, if the pre-serialize function does - * not need to change the size or location of the on-disk image, or - * compute its compress size, it must set *flags_ptr to zero. + * not need to change the size or location of the on-disk image, it must + * set *flags_ptr to zero. * - * If the (uncompressed) size of the on-disk image must be changed, - * the pre-serialize function must load the length of the new image - * into *new_len_ptr, and set the H5C__SERIALIZE_RESIZED_FLAG in - * *flags_ptr. + * If the size of the on-disk image must be changed, the pre-serialize + * function must load the length of the new image into *new_len_ptr, and + * set the H5C__SERIALIZE_RESIZED_FLAG in *flags_ptr. * * If the base address of the on disk image must be changed, the * pre-serialize function must set *new_addr_ptr to the new base address, * and set the H5C__SERIALIZE_MOVED_FLAG in *flags_ptr. * - * If the H5C__CLASS_COMPRESSED_FLAG is set in the assocated instance - * of H5C_class_t, and filters (i.e. compression) are enabled, the - * pre-serialize function must compute the compressed size of the - * on disk image, and if it has changed, load this value into - * *new_compressed_len_ptr, and set H5C__SERIALIZE_COMPRESSED_FLAG in - * *flags_ptr. - * - * Note that to do this, the preserialize function will typically have - * to serialize the entry, and run it through the filters to obtain - * the compressed size. For efficiency, the compressed image may - * be stored to be copied into the supplied buffer by the - * serialize callback. Needless to say this is awkward. We may - * want to re-work the API for cache clients to simplify this. - * * In addition, the pre-serialize callback may perform any other * processing required before the entry is written to disk * @@ -852,11 +698,7 @@ typedef struct H5C_tag_t { * the buffer. * * len: Length in bytes of the in file image of the entry to be - * serialized. Also the size of *image_ptr (below). If - * compression is not enabled, this value is simply the - * uncompressed size of the entry's image on disk. If - * compression is enabled, this value is the size of the - * compressed image. + * serialized. Also the size of *image_ptr (below). * * This parameter is supplied mainly for sanity checking. * Sanity checks should be performed when compiled in debug @@ -875,8 +717,7 @@ typedef struct H5C_tag_t { * * The serialize function must then examine the in core * representation indicated by the thing parameter, and write a - * serialized (and possibly compressed) image of its contents into - * the provided buffer. + * serialized image of its contents into the provided buffer. * * If it is successful, the function must return SUCCEED. * @@ -948,65 +789,7 @@ typedef struct H5C_tag_t { * * At least when compiled with debug, it would be useful if the * free ICR call would fail if the in core representation has been - * modified since the last serialize of clear callback. - * - * CLEAR: Pointer to the clear callback. - * - * In principle, there should be no need for the clear callback, - * as the dirty flag should be maintained by the metadata cache. - *. - * However, some clients maintain dirty bits on internal data, - * and we need some way of keeping these dirty bits in sync with - * those maintained by the metadata cache. This callback exists - * to serve this purpose. If defined, it is called whenever the - * cache marks dirty entry clean, or when the cache is about to - * discard a dirty entry without writing it to disk (This - * happens as the result of an unprotect call with the - * H5AC__DELETED_FLAG set, and the H5C__TAKE_OWNERSHIP_FLAG not - * set.) - * - * Arguably, this functionality should be in the NOTIFY callback. - * However, this callback is specific to only a few clients, and - * it will be called relatively frequently. Hence it is made its - * own callback to minimize overhead. - * - * The typedef for the clear callback is as follows: - * - * typedef herr_t (*H5C_clear_func_t)(const H5F_t *f, - * void * thing, - * hbool_t about_to_destroy); - * - * The parameters of the clear callback are as follows: - * - * f: File pointer. - * - * thing: Pointer to void containing the address of the in core - * representation of the target metadata cache entry. This - * is the same pointer that would be returned by a protect() - * call of the associated addr and len. - * - * about_to_destroy: Boolean flag used to indicate whether the - * metadata cache is about to destroy the target metadata - * cache entry. The callback may use this flag to omit - * operations that are irrelevant it the entry is about - * to be destroyed. - * - * Processing in the clear function should proceed as follows: - * - * Reset all internal dirty bits in the target metadata cache entry. - * - * If the about_to_destroy flag is TRUE, the clear function may - * ommit any dirty bit that will not trigger a sanity check failure - * or otherwise cause problems in the subsequent free icr call. - * In particular, the call must ensure that the free icr call will - * not fail due to changes prior to this call, and after the - * last serialize or clear call. - * - * If the function is successful, it must return SUCCEED. - * - * If it fails for any reason, the function must return FAIL and - * push error information on the error stack with the error API - * routines. + * modified since the last serialize callback. * * GET_FSF_SIZE: Pointer to the get file space free size callback. * @@ -1042,12 +825,12 @@ typedef struct H5C_tag_t { * At present this callback is used only by the H5FA and H5EA dblock * and dblock page client classes. * - * The typedef for the clear callback is as follows: + * The typedef for the get_fsf_size callback is as follows: * * typedef herr_t (*H5C_get_fsf_size_t)(const void * thing, * size_t *fsf_size_ptr); * - * The parameters of the clear callback are as follows: + * The parameters of the get_fsf_size callback are as follows: * * thing: Pointer to void containing the address of the in core * representation of the target metadata cache entry. This @@ -1081,28 +864,30 @@ typedef enum H5C_notify_action_t { H5C_NOTIFY_ACTION_AFTER_FLUSH, /* Entry has just been flushed to * file. */ - H5C_NOTIFY_ACTION_BEFORE_EVICT /* Entry is about to be evicted + H5C_NOTIFY_ACTION_BEFORE_EVICT, /* Entry is about to be evicted * from cache. */ + H5C_NOTIFY_ACTION_ENTRY_DIRTIED, /* Entry has been marked dirty. */ + H5C_NOTIFY_ACTION_ENTRY_CLEANED, /* Entry has been marked clean. */ + H5C_NOTIFY_ACTION_CHILD_DIRTIED, /* Dependent child has been marked dirty. */ + H5C_NOTIFY_ACTION_CHILD_CLEANED /* Dependent child has been marked clean. */ } H5C_notify_action_t; /* Cache client callback function pointers */ -typedef herr_t (*H5C_get_load_size_func_t)(const void *udata_ptr, - size_t *image_len_ptr); +typedef herr_t (*H5C_get_initial_load_size_func_t)(void *udata_ptr, size_t *image_len_ptr); +typedef herr_t (*H5C_get_final_load_size_func_t)(const void *image_ptr, + size_t image_len, void *udata_ptr, size_t *actual_len_ptr); +typedef htri_t (*H5C_verify_chksum_func_t)(const void *image_ptr, size_t len, void *udata_ptr); typedef void *(*H5C_deserialize_func_t)(const void *image_ptr, size_t len, void *udata_ptr, hbool_t *dirty_ptr); -typedef herr_t (*H5C_image_len_func_t)(const void *thing, - size_t *image_len_ptr, hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +typedef herr_t (*H5C_image_len_func_t)(const void *thing, size_t *image_len_ptr); typedef herr_t (*H5C_pre_serialize_func_t)(const H5F_t *f, hid_t dxpl_id, - void *thing, haddr_t addr, size_t len, size_t compressed_len, - haddr_t *new_addr_ptr, size_t *new_len_ptr, size_t *new_compressed_len_ptr, - unsigned *flags_ptr); + void *thing, haddr_t addr, size_t len, haddr_t *new_addr_ptr, + size_t *new_len_ptr, unsigned *flags_ptr); typedef herr_t (*H5C_serialize_func_t)(const H5F_t *f, void *image_ptr, size_t len, void *thing); typedef herr_t (*H5C_notify_func_t)(H5C_notify_action_t action, void *thing); typedef herr_t (*H5C_free_icr_func_t)(void *thing); -typedef herr_t (*H5C_clear_func_t)(const H5F_t *f, void * thing, - hbool_t about_to_destroy); typedef herr_t (*H5C_get_fsf_size_t)(const void * thing, size_t *fsf_size_ptr); /* Metadata cache client class definition */ @@ -1111,14 +896,15 @@ typedef struct H5C_class_t { const char * name; H5FD_mem_t mem_type; unsigned flags; - H5C_get_load_size_func_t get_load_size; + H5C_get_initial_load_size_func_t get_initial_load_size; + H5C_get_final_load_size_func_t get_final_load_size; + H5C_verify_chksum_func_t verify_chksum; H5C_deserialize_func_t deserialize; H5C_image_len_func_t image_len; H5C_pre_serialize_func_t pre_serialize; H5C_serialize_func_t serialize; H5C_notify_func_t notify; H5C_free_icr_func_t free_icr; - H5C_clear_func_t clear; H5C_get_fsf_size_t fsf_size; } H5C_class_t; @@ -1223,51 +1009,11 @@ typedef int H5C_ring_t; * * addr: Base address of the cache entry on disk. * - * size: Length of the cache entry on disk in bytes(exception: if - * the entry is compressed on disk, this field contains the - * uncompressed size of the entry -- see discussion of - * compressed entries below). Note that unlike normal - * caches, the entries in this cache are of arbitrary size. - * - * With the exception of compressed entries, the file space - * allocations for cache entries implied by the addr and size - * fields must be disjoint. For compressed entries, - * the size field contains the uncompressed size -- thus in - * in this case, substitution of compressed size for size - * must result in disjoint file space allocations. However, - * as discussed below, the compressed size may not be know. - * - * Any entry whose associated instance of H5C_class_t has the - * H5C__CLASS_COMPRESSED_FLAG set may be compressed. When - * an entry is compressed (that is, when filters are enabled - * on it), the compressed flag (see below) must be set, and - * the compressed size (if known), must be stored in - * the compressed_size field. - * - * Since the compressed size will be unknown unless the - * entry is clean, or has an up to date image (see the - * image_ptr and image_up_to_date fields below), we use the - * uncompressed size for all purposes other than disk I/O. - * - * compressed: Boolean flag that is set iff the instance of H5C_class_t - * associated with the entry has the H5C__CLASS_COMPRESSED_FLAG - * set, and filters are enabled on the entry. - * - * compressed_size: If compressed is TRUE, this field contains the actual - * compressed size of the entry in bytes, which is also its - * true size on disk -- or the uncompressed size if the - * compressed size is unknown (i.e. the entry has been - * inserted in the cache, but it has not been compressed yet). - * Note that this value will usually be incorrect if the - * entry is dirty. - * - * Since this value is frequently out of date and expensive to - * compute, it is used only for disk I/O. The uncompressed - * size of the entry (stored in the size field above) is used - * for all other purposes (i.e. computing the sum of the sizes - * of all entries in the cache, etc.). - * - * If compressed is FALSE, this field should contain 0. + * size: Length of the cache entry on disk in bytes Note that unlike + * normal caches, the entries in this cache are of arbitrary size. + * + * The file space allocations for cache entries implied by the + * addr and size fields must be disjoint. * * image_ptr: Pointer to void. When not NULL, this field points to a * dynamically allocated block of size bytes in which the @@ -1288,32 +1034,10 @@ typedef int H5C_ring_t; * The name is not particularly descriptive, but is retained * to avoid changes in existing code. * - * is_corked: Boolean flag indicating whether the cache entry associated - * with an object is corked or not corked. - * * is_dirty: Boolean flag indicating whether the contents of the cache * entry has been modified since the last time it was written * to disk. * - * NOTE: For historical reasons, this field is not maintained - * by the cache. Instead, the module using the cache - * sets this flag when it modifies the entry, and the - * flush and clear functions supplied by that module - * reset the dirty when appropriate. - * - * This is a bit quirky, so we may want to change this - * someday. However it will require a change in the - * cache interface. - * - * Update: Management of the is_dirty field has been largely - * moved into the cache. The only remaining exceptions - * are the flush and clear functions supplied by the - * modules using the cache. These still clear the - * is_dirty field as before. -- JRM 7/5/05 - * - * Update: Management of the is_dirty field is now entirely - * in the cache. -- JRM 7/5/07 - * * dirtied: Boolean flag used to indicate that the entry has been * dirtied while protected. * @@ -1493,7 +1217,7 @@ typedef int H5C_ring_t; * * Fields supporting the hash table: * - * Fields in the cache are indexed by a more or less conventional hash table. + * Entries in the cache are indexed by a more or less conventional hash table. * If there are multiple entries in any hash bin, they are stored in a doubly * linked list. * @@ -1580,6 +1304,28 @@ typedef int H5C_ring_t; * In either case, when there is no previous item, it should * be NULL. * + * + * Fields supporting tagged entries: + * + * Entries in the cache that belong to a single object in the file are + * joined into a doubly-linked list, and are "tagged" with the object header + * address for that object's base header "chunk" (which is used as the + * canonical address for the object). Global and shared entries are + * not tagged. Tagged entries have a pointer to the tag info for the object, + * which is shared state for all the entries for that object. + * + * tl_next: Pointer to the next entry in the tag list for an object. + * NULL for the tail entry in the list, as well as untagged + * entries. + * + * tl_prev: Pointer to the previous entry in the tag list for an object. + * NULL for the head entry in the list, as well as untagged + * entries. + * + * tag_info: Pointer to the common tag state for all entries belonging to + * an object. NULL for untagged entries. + * + * * Cache entry stats collection fields: * * These fields should only be compiled in when both H5C_COLLECT_CACHE_STATS @@ -1604,14 +1350,9 @@ typedef struct H5C_cache_entry_t { H5C_t * cache_ptr; haddr_t addr; size_t size; - hbool_t compressed; - size_t compressed_size; void * image_ptr; hbool_t image_up_to_date; const H5C_class_t * type; - haddr_t tag; - H5C_tag_globality_t globality; - hbool_t is_corked; hbool_t is_dirty; hbool_t dirtied; hbool_t is_protected; @@ -1655,6 +1396,11 @@ typedef struct H5C_cache_entry_t { struct H5C_cache_entry_t * coll_prev; #endif /* H5_HAVE_PARALLEL */ + /* fields supporting tag lists */ + struct H5C_cache_entry_t * tl_next; + struct H5C_cache_entry_t * tl_prev; + struct H5C_tag_info_t * tag_info; + #if H5C_COLLECT_CACHE_ENTRY_STATS /* cache entry stats fields */ int32_t accesses; @@ -1961,19 +1707,28 @@ H5_DLL H5C_t *H5C_create(size_t max_cache_size, size_t min_clean_size, int max_type_id, const char *(*type_name_table_ptr), H5C_write_permitted_func_t check_write_permitted, hbool_t write_permitted, H5C_log_flush_func_t log_flush, void *aux_ptr); +H5_DLL herr_t H5C_set_up_logging(H5C_t *cache_ptr, const char log_location[], hbool_t start_immediately); +H5_DLL herr_t H5C_tear_down_logging(H5C_t *cache_ptr); +H5_DLL herr_t H5C_start_logging(H5C_t *cache_ptr); +H5_DLL herr_t H5C_stop_logging(H5C_t *cache_ptr); +H5_DLL herr_t H5C_get_logging_status(const H5C_t *cache_ptr, /*OUT*/ hbool_t *is_enabled, + /*OUT*/ hbool_t *is_currently_logging); +H5_DLL herr_t H5C_write_log_message(const H5C_t *cache_ptr, const char message[]); H5_DLL void H5C_def_auto_resize_rpt_fcn(H5C_t *cache_ptr, int32_t version, double hit_rate, enum H5C_resize_status status, size_t old_max_cache_size, size_t new_max_cache_size, size_t old_min_clean_size, size_t new_min_clean_size); H5_DLL herr_t H5C_dest(H5F_t *f, hid_t dxpl_id); +H5_DLL herr_t H5C_evict(H5F_t *f, hid_t dxpl_id); H5_DLL herr_t H5C_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, haddr_t addr, unsigned flags); H5_DLL herr_t H5C_flush_cache(H5F_t *f, hid_t dxpl_id, unsigned flags); H5_DLL herr_t H5C_flush_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag); -H5_DLL herr_t H5C_evict_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag); +H5_DLL herr_t H5C_evict_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag, hbool_t match_global); H5_DLL herr_t H5C_expunge_tag_type_metadata(H5F_t *f, hid_t dxpl_id, haddr_t tag, int type_id, unsigned flags); +H5_DLL herr_t H5C_get_tag(const void *thing, /*OUT*/ haddr_t *tag); #if H5C_DO_TAGGING_SANITY_CHECKS -herr_t H5C_verify_tag(int id, haddr_t tag, H5C_tag_globality_t globality); +herr_t H5C_verify_tag(int id, haddr_t tag); #endif H5_DLL herr_t H5C_flush_to_min_clean(H5F_t *f, hid_t dxpl_id); H5_DLL herr_t H5C_get_cache_auto_resize_config(const H5C_t *cache_ptr, @@ -1993,6 +1748,7 @@ H5_DLL FILE *H5C_get_trace_file_ptr_from_entry(const H5C_cache_entry_t *entry_pt H5_DLL herr_t H5C_insert_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, haddr_t addr, void *thing, unsigned int flags); H5_DLL herr_t H5C_mark_entry_dirty(void *thing); +H5_DLL herr_t H5C_mark_entry_clean(void *thing); H5_DLL herr_t H5C_move_entry(H5C_t *cache_ptr, const H5C_class_t *type, haddr_t old_addr, haddr_t new_addr); H5_DLL herr_t H5C_pin_protected_entry(void *thing); @@ -2020,6 +1776,7 @@ H5_DLL hbool_t H5C_get_ignore_tags(const H5C_t *cache_ptr); H5_DLL herr_t H5C_retag_entries(H5C_t * cache_ptr, haddr_t src_tag, haddr_t dest_tag); H5_DLL herr_t H5C_cork(H5C_t *cache_ptr, haddr_t obj_addr, unsigned action, hbool_t *corked); H5_DLL herr_t H5C_get_entry_ring(const H5F_t *f, haddr_t addr, H5C_ring_t *ring); +H5_DLL herr_t H5C_remove_entry(void * thing); #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5C_apply_candidate_list(H5F_t *f, hid_t dxpl_id, diff --git a/src/H5Cquery.c b/src/H5Cquery.c index 874b12f..be0d4fa 100644 --- a/src/H5Cquery.c +++ b/src/H5Cquery.c @@ -275,7 +275,7 @@ H5C_get_entry_status(const H5F_t *f, if(is_pinned_ptr != NULL) *is_pinned_ptr = entry_ptr->is_pinned; if(is_corked_ptr != NULL) - *is_corked_ptr = entry_ptr->is_corked; + *is_corked_ptr = entry_ptr->tag_info ? entry_ptr->tag_info->corked : FALSE; if(is_flush_dep_parent_ptr != NULL) *is_flush_dep_parent_ptr = (entry_ptr->flush_dep_nchildren > 0); if(is_flush_dep_child_ptr != NULL) diff --git a/src/H5Ctag.c b/src/H5Ctag.c index d560e25..157a838 100644 --- a/src/H5Ctag.c +++ b/src/H5Ctag.c @@ -62,11 +62,6 @@ typedef struct { hbool_t pinned_entries_need_evicted; /* Flag to indicate that a pinned entry was attempted to be evicted */ } H5C_tag_iter_evict_ctx_t; -/* Typedef for tagged entry iterator callback context - retag tagged entries */ -typedef struct { - haddr_t dest_tag; /* New tag value for matching entries */ -} H5C_tag_iter_retag_ctx_t; - /* Typedef for tagged entry iterator callback context - expunge tag type metadata */ typedef struct { H5F_t * f; /* File pointer for evicting entry */ @@ -91,6 +86,9 @@ static herr_t H5C__mark_tagged_entries(H5C_t *cache_ptr, haddr_t tag); /* Package Variables */ /*********************/ +/* Declare extern free list to manage the tag info struct */ +H5FL_EXTERN(H5C_tag_info_t); + /*****************************/ /* Library Private Variables */ @@ -182,39 +180,32 @@ H5C_get_ignore_tags(const H5C_t *cache_ptr) * Programmer: Mike McGreevy * January 14, 2010 * - * Modifications: - * *------------------------------------------------------------------------- */ herr_t -H5C__tag_entry(H5C_t * cache_ptr, H5C_cache_entry_t * entry_ptr, hid_t dxpl_id) +H5C__tag_entry(H5C_t *cache, H5C_cache_entry_t *entry, hid_t dxpl_id) { H5P_genplist_t *dxpl; /* dataset transfer property list */ - H5C_tag_t tag; /* Tag structure */ + H5C_tag_info_t *tag_info; /* Points to a tag info struct */ + haddr_t tag; /* Tag value */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE /* Assertions */ - HDassert(cache_ptr != NULL); - HDassert(entry_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(cache != NULL); + HDassert(entry != NULL); + HDassert(cache->magic == H5C__H5C_T_MAGIC); /* Get the dataset transfer property list */ if(NULL == (dxpl = (H5P_genplist_t *)H5I_object_verify(dxpl_id, H5I_GENPROP_LST))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") /* Get the tag from the DXPL */ - if((H5P_get(dxpl, "H5C_tag", &tag)) < 0) + if((H5P_get(dxpl, H5AC_TAG_NAME, &tag)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to query property value") - if(cache_ptr->ignore_tags != TRUE) { -#if H5C_DO_TAGGING_SANITY_CHECKS - /* Perform some sanity checks to ensure that a correct tag is being applied */ - if(H5C_verify_tag(entry_ptr->type->id, tag.value, tag.globality) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "tag verification failed") -#endif - } else { + if(cache->ignore_tags) { /* if we're ignoring tags, it's because we're running tests on internal functions and may not have inserted a tag value into a given dxpl_id before creating some metadata. Thus, @@ -222,17 +213,48 @@ H5C__tag_entry(H5C_t * cache_ptr, H5C_cache_entry_t * entry_ptr, hid_t dxpl_id) arbitrarily set it to something for the sake of passing the tests. If the tag value is set, then we'll just let it get assigned without additional checking for correctness. */ - if(!tag.value) { - tag.value = H5AC__IGNORE_TAG; - tag.globality = H5C_GLOBALITY_NONE; - } /* end if */ + if(!H5F_addr_defined(tag)) + tag = H5AC__IGNORE_TAG; } /* end if */ +#if H5C_DO_TAGGING_SANITY_CHECKS + else { + /* Perform some sanity checks to ensure that a correct tag is being applied */ + if(H5C_verify_tag(entry->type->id, tag) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "tag verification failed") + } /* end else */ +#endif - /* Apply the tag to the entry */ - entry_ptr->tag = tag.value; + /* Search the list of tagged object addresses in the cache */ + tag_info = (H5C_tag_info_t *)H5SL_search(cache->tag_list, &tag); - /* Apply the tag globality to the entry */ - entry_ptr->globality = tag.globality; + /* Check if this is the first entry for this tagged object */ + if(NULL == tag_info) { + /* Allocate new tag info struct */ + if(NULL == (tag_info = H5FL_CALLOC(H5C_tag_info_t))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "can't allocate tag info for cache entry") + + /* Set the tag for all entries */ + tag_info->tag = tag; + + /* Insert tag info into skip list */ + if(H5SL_insert(cache->tag_list, tag_info, &(tag_info->tag)) < 0 ) + HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "can't insert tag info in skip list") + } /* end if */ + else + HDassert(tag_info->corked || (tag_info->entry_cnt > 0 && tag_info->head)); + + /* Sanity check entry, to avoid double insertions, etc */ + HDassert(entry->tl_next == NULL); + HDassert(entry->tl_prev == NULL); + HDassert(entry->tag_info == NULL); + + /* Add the entry to the list for the tagged object */ + entry->tl_next = tag_info->head; + entry->tag_info = tag_info; + if(tag_info->head) + tag_info->head->tl_prev = entry; + tag_info->head = entry; + tag_info->entry_cnt++; done: FUNC_LEAVE_NOAPI(ret_value) @@ -241,7 +263,70 @@ done: /*------------------------------------------------------------------------- * - * Function: H5C_iter_tagged_entries + * Function: H5C__untag_entry + * + * Purpose: Removes an entry from a tag list, possibly removing the tag + * info from the list of tagged objects with entries. + * + * Return: FAIL if error is detected, SUCCEED otherwise. + * + * Programmer: Quincey Koziol + * July 8, 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C__untag_entry(H5C_t *cache, H5C_cache_entry_t *entry) +{ + H5C_tag_info_t *tag_info; /* Points to a tag info struct */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Assertions */ + HDassert(cache != NULL); + HDassert(entry != NULL); + HDassert(cache->magic == H5C__H5C_T_MAGIC); + + /* Get the entry's tag info struct */ + if(NULL != (tag_info = entry->tag_info)) { + /* Remove the entry from the list */ + if(entry->tl_next) + entry->tl_next->tl_prev = entry->tl_prev; + if(entry->tl_prev) + entry->tl_prev->tl_next = entry->tl_next; + if(tag_info->head == entry) + tag_info->head = entry->tl_next; + tag_info->entry_cnt--; + + /* Reset pointers, to avoid confusion */ + entry->tl_next = NULL; + entry->tl_prev = NULL; + entry->tag_info = NULL; + + /* Remove the tag info from the tag list, if there's no more entries with this tag */ + if(!tag_info->corked && 0 == tag_info->entry_cnt) { + /* Sanity check */ + HDassert(NULL == tag_info->head); + + if(H5SL_remove(cache->tag_list, &(tag_info->tag)) != tag_info) + HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove tag info from list") + + /* Release the tag info */ + tag_info = H5FL_FREE(H5C_tag_info_t, tag_info); + } /* end if */ + else + HDassert(tag_info->corked || NULL != tag_info->head); + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C__untag_entry */ + + +/*------------------------------------------------------------------------- + * + * Function: H5C__iter_tagged_entries_real * * Purpose: Iterate over tagged entries, making a callback for matches * @@ -252,39 +337,92 @@ done: * *------------------------------------------------------------------------- */ -int -H5C__iter_tagged_entries(H5C_t *cache, haddr_t tag, hbool_t match_global, - H5C_tag_iter_cb_t cb, void *cb_ctx) +static herr_t +H5C__iter_tagged_entries_real(H5C_t *cache, haddr_t tag, H5C_tag_iter_cb_t cb, + void *cb_ctx) { - unsigned u; /* Local index variable */ - int ret_value = H5_ITER_CONT; /* Return value */ + H5C_tag_info_t *tag_info; /* Points to a tag info struct */ + herr_t ret_value = SUCCEED; /* Return value */ /* Function enter macro */ - FUNC_ENTER_PACKAGE + FUNC_ENTER_STATIC /* Sanity checks */ HDassert(cache != NULL); HDassert(cache->magic == H5C__H5C_T_MAGIC); - /* Iterate through entries in the index. */ - for(u = 0; u < H5C__HASH_TABLE_LEN; u++) { + /* Search the list of tagged object addresses in the cache */ + tag_info = (H5C_tag_info_t *)H5SL_search(cache->tag_list, &tag); + + /* If there's any entries for this tag, iterate over them */ + if(tag_info) { H5C_cache_entry_t *entry; /* Pointer to current entry */ H5C_cache_entry_t *next_entry; /* Pointer to next entry in hash bucket chain */ - next_entry = cache->index[u]; - while(next_entry != NULL) { - /* Acquire pointer to current entry and to next entry */ + /* Sanity check */ + HDassert(tag_info->head); + HDassert(tag_info->entry_cnt > 0); + + /* Iterate over the entries for this tag */ + entry = tag_info->head; + while(entry) { + /* Acquire pointer to next entry */ + next_entry = entry->tl_next; + + /* Make callback for entry */ + if((cb)(entry, cb_ctx) != H5_ITER_CONT) + HGOTO_ERROR(H5E_CACHE, H5E_BADITER, FAIL, "tagged entry iteration callback failed") + + /* Advance to next entry */ entry = next_entry; - next_entry = entry->ht_next; - - /* Check for entry matching tag and/or globality */ - if((entry->tag == tag) || (match_global && entry->globality == H5C_GLOBALITY_MAJOR)) { - /* Make callback for entry */ - if((ret_value = (cb)(entry, cb_ctx)) != H5_ITER_CONT) - HGOTO_ERROR(H5E_CACHE, H5E_BADITER, H5_ITER_ERROR, "Iteration of tagged entries failed") - } /* end if */ } /* end while */ - } /* end for */ + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C__iter_tagged_entries_real() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5C__iter_tagged_entries + * + * Purpose: Iterate over tagged entries, making a callback for matches + * + * Return: FAIL if error is detected, SUCCEED otherwise. + * + * Programmer: Quincey Koziol + * June 7, 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C__iter_tagged_entries(H5C_t *cache, haddr_t tag, hbool_t match_global, + H5C_tag_iter_cb_t cb, void *cb_ctx) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + /* Function enter macro */ + FUNC_ENTER_PACKAGE + + /* Sanity checks */ + HDassert(cache != NULL); + HDassert(cache->magic == H5C__H5C_T_MAGIC); + + /* Iterate over the entries for this tag */ + if(H5C__iter_tagged_entries_real(cache, tag, cb, cb_ctx) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_BADITER, FAIL, "iteration of tagged entries failed") + + /* Check for iterating over global metadata */ + if(match_global) { + /* Iterate over the entries for SOHM entries */ + if(H5C__iter_tagged_entries_real(cache, H5AC__SOHM_TAG, cb, cb_ctx) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_BADITER, FAIL, "iteration of tagged entries failed") + + /* Iterate over the entries for global heap entries */ + if(H5C__iter_tagged_entries_real(cache, H5AC__GLOBALHEAP_TAG, cb, cb_ctx) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_BADITER, FAIL, "iteration of tagged entries failed") + } /* end if */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -327,11 +465,12 @@ H5C__evict_tagged_entries_cb(H5C_cache_entry_t *entry, void *_ctx) entry and we'll loop back around again (as evicting other entries will hopefully unpin this entry) */ ctx->pinned_entries_need_evicted = TRUE; - else + else { /* Evict the Entry */ - if(H5C__flush_single_entry(ctx->f, ctx->dxpl_id, entry, H5C__FLUSH_INVALIDATE_FLAG | H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, NULL, NULL) < 0) + if(H5C__flush_single_entry(ctx->f, ctx->dxpl_id, entry, H5C__FLUSH_INVALIDATE_FLAG | H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, H5_ITER_ERROR, "Entry eviction failed.") - ctx->evicted_entries_last_pass = TRUE; + ctx->evicted_entries_last_pass = TRUE; + } /* end else */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -352,7 +491,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5C_evict_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag) +H5C_evict_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag, hbool_t match_global) { H5C_t *cache; /* Pointer to cache structure */ H5C_tag_iter_evict_ctx_t ctx; /* Context for iterator callback */ @@ -374,15 +513,15 @@ H5C_evict_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag) /* Start evicting entries */ do { - /* Reset pinned/evicted tracking flags */ - ctx.pinned_entries_need_evicted = FALSE; - ctx.evicted_entries_last_pass = FALSE; + /* Reset pinned/evicted tracking flags */ + ctx.pinned_entries_need_evicted = FALSE; + ctx.evicted_entries_last_pass = FALSE; - /* Iterate through entries in the cache */ - if(H5C__iter_tagged_entries(cache, tag, TRUE, H5C__evict_tagged_entries_cb, &ctx) < 0) + /* Iterate through entries in the cache */ + if(H5C__iter_tagged_entries(cache, tag, match_global, H5C__evict_tagged_entries_cb, &ctx) < 0) HGOTO_ERROR(H5E_CACHE, H5E_BADITER, FAIL, "Iteration of tagged entries failed") - /* Keep doing this until we have stopped evicted entries */ + /* Keep doing this until we have stopped evicted entries */ } while(TRUE == ctx.evicted_entries_last_pass); /* Fail if we have finished evicting entries and pinned entries still need evicted */ @@ -477,7 +616,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5C_verify_tag(int id, haddr_t tag, H5C_tag_globality_t globality) +H5C_verify_tag(int id, haddr_t tag) { herr_t ret_value = SUCCEED; @@ -488,8 +627,10 @@ H5C_verify_tag(int id, haddr_t tag, H5C_tag_globality_t globality) * constraints are met. */ if(tag == H5AC__IGNORE_TAG) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "cannot ignore a tag while doing verification.") - else if(tag == H5AC__INVALID_TAG) - HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "no metadata tag provided") + else if(tag == H5AC__INVALID_TAG) { + if(id != H5AC_PROXY_ENTRY_ID) + HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "no metadata tag provided") + } /* end else-if */ else { /* Perform some sanity checks on tag value. Certain entry * types require certain tag values, so check that these @@ -499,8 +640,6 @@ H5C_verify_tag(int id, haddr_t tag, H5C_tag_globality_t globality) if((id == H5AC_SUPERBLOCK_ID) || (id == H5AC_DRVRINFO_ID)) { if(tag != H5AC__SUPERBLOCK_TAG) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "superblock not tagged with H5AC__SUPERBLOCK_TAG") - if(globality != H5C_GLOBALITY_MAJOR) - HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "superblock/driver-info globality not marked with H5C_GLOBALITY_MAJOR") } /* end if */ else { if(tag == H5AC__SUPERBLOCK_TAG) @@ -508,31 +647,18 @@ H5C_verify_tag(int id, haddr_t tag, H5C_tag_globality_t globality) } /* end else */ /* Free Space Manager */ - if((id == H5AC_FSPACE_HDR_ID) || (id == H5AC_FSPACE_SINFO_ID)) { - if(tag != H5AC__FREESPACE_TAG) - HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "freespace entry not tagged with H5AC__FREESPACE_TAG") - if(globality != H5C_GLOBALITY_MINOR) - HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "freespace entry globality not marked with H5C_GLOBALITY_MINOR") - } /* end if */ - else { - if(tag == H5AC__FREESPACE_TAG) + if(tag == H5AC__FREESPACE_TAG && ((id != H5AC_FSPACE_HDR_ID) && (id != H5AC_FSPACE_SINFO_ID))) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "H5AC__FREESPACE_TAG applied to non-freespace entry") - } /* end else */ /* SOHM */ - if((id == H5AC_SOHM_TABLE_ID) || (id == H5AC_SOHM_LIST_ID)) { + if((id == H5AC_SOHM_TABLE_ID) || (id == H5AC_SOHM_LIST_ID)) if(tag != H5AC__SOHM_TAG) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "sohm entry not tagged with H5AC__SOHM_TAG") - if(globality != H5C_GLOBALITY_MAJOR) - HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "sohm entry globality not marked with H5C_GLOBALITY_MAJOR") - } /* end if */ /* Global Heap */ if(id == H5AC_GHEAP_ID) { if(tag != H5AC__GLOBALHEAP_TAG) HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "global heap not tagged with H5AC__GLOBALHEAP_TAG") - if(globality != H5C_GLOBALITY_MAJOR) - HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "global heap entry globality not marked with H5C_GLOBALITY_MAJOR") } /* end if */ else { if(tag == H5AC__GLOBALHEAP_TAG) @@ -590,37 +716,6 @@ done: /*------------------------------------------------------------------------- * - * Function: H5C__retag_entries_cb - * - * Purpose: Change tag for entries that match current tag - * - * Return: H5_ITER_CONT (can't fail) - * - * Programmer: Mike McGreevy - * March 17, 2010 - * - *------------------------------------------------------------------------- - */ -static int -H5C__retag_entries_cb(H5C_cache_entry_t *entry, void *_ctx) -{ - H5C_tag_iter_retag_ctx_t *ctx = (H5C_tag_iter_retag_ctx_t *)_ctx; /* Get pointer to iterator context */ - - /* Function enter macro */ - FUNC_ENTER_STATIC_NOERR - - /* Santify checks */ - HDassert(entry); - HDassert(ctx); - - entry->tag = ctx->dest_tag; - - FUNC_LEAVE_NOAPI(H5_ITER_CONT) -} /* H5C_retag_entries_cb() */ - - -/*------------------------------------------------------------------------- - * * Function: H5C_retag_entries * * Purpose: Searches through cache index for all entries with the @@ -637,7 +732,7 @@ H5C__retag_entries_cb(H5C_cache_entry_t *entry, void *_ctx) herr_t H5C_retag_entries(H5C_t *cache, haddr_t src_tag, haddr_t dest_tag) { - H5C_tag_iter_retag_ctx_t ctx; /* Iterator callback context */ + H5C_tag_info_t *tag_info; /* Points to a tag info struct */ herr_t ret_value = SUCCEED; /* Return value */ /* Function enter macro */ @@ -646,12 +741,15 @@ H5C_retag_entries(H5C_t *cache, haddr_t src_tag, haddr_t dest_tag) /* Sanity check */ HDassert(cache); - /* Construct context for iterator callbacks */ - ctx.dest_tag = dest_tag; + /* Remove tag info from tag list */ + if(NULL != (tag_info = (H5C_tag_info_t *)H5SL_remove(cache->tag_list, &src_tag))) { + /* Change to new tag */ + tag_info->tag = dest_tag; - /* Iterate through entries, retagging those with the src_tag tag */ - if(H5C__iter_tagged_entries(cache, src_tag, FALSE, H5C__retag_entries_cb, &ctx) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_BADITER, FAIL, "Iteration of tagged entries failed") + /* Re-insert tag info into skip list */ + if(H5SL_insert(cache->tag_list, tag_info, &(tag_info->tag)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "can't insert tag info in skip list") + } /* end if */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -743,71 +841,32 @@ done: /*------------------------------------------------------------------------- - * Function: H5C__mark_tagged_entries_cork_cb - * - * Purpose: The "is_corked" field to "val" for en entry - * - * Return: H5_ITER_ERROR if error is detected, H5_ITER_CONT otherwise. * - * Programmer: Vailin Choi - * January 2014 + * Function: H5C_get_tag() * - *------------------------------------------------------------------------- - */ -static int -H5C__mark_tagged_entries_cork_cb(H5C_cache_entry_t *entry, void *_ctx) -{ - H5C_tag_iter_cork_ctx_t *ctx = (H5C_tag_iter_cork_ctx_t *)_ctx; /* Get pointer to iterator context */ - - /* Function enter macro */ - FUNC_ENTER_STATIC_NOERR - - /* Santify checks */ - HDassert(entry); - HDassert(ctx); - - /* Set the entry's "corked" field to "val" */ - entry->is_corked = ctx->cork_val; - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5C__mark_tagged_entries_cork_cb() */ - - -/*------------------------------------------------------------------------- - * Function: H5C__mark_tagged_entries_cork - * - * Purpose: To set the "is_corked" field to "val" for entries in cache - * with the entry's tag equals to "obj_addr". + * Purpose: Get the tag for a metadata cache entry. * - * Return: FAIL if error is detected, SUCCEED otherwise. + * Return: SUCCEED (can't fail) * - * Programmer: Vailin Choi - * January 2014 + * Programmer: Dana Robinson + * Fall 2016 * *------------------------------------------------------------------------- */ herr_t -H5C__mark_tagged_entries_cork(H5C_t *cache, haddr_t obj_addr, hbool_t val) +H5C_get_tag(const void *thing, haddr_t *tag /*OUT*/) { - H5C_tag_iter_cork_ctx_t ctx; /* Context for iterator callback */ - herr_t ret_value = SUCCEED; /* Return value */ - - /* Function enter macro */ - FUNC_ENTER_PACKAGE + const H5C_cache_entry_t *entry = (const H5C_cache_entry_t *)thing; /* Pointer to cache entry */ - /* Sanity checks */ - HDassert(cache); - HDassert(cache->magic == H5C__H5C_T_MAGIC); + FUNC_ENTER_NOAPI_NOERR - /* Construct context for iterator callbacks */ - ctx.cork_val = val; + HDassert(entry); + HDassert(entry->tag_info); + HDassert(tag); - /* Iterate through entries, find each entry with the specified tag */ - /* and set the entry's "corked" field to "val" */ - if(H5C__iter_tagged_entries(cache, obj_addr, FALSE, H5C__mark_tagged_entries_cork_cb, &ctx) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_BADITER, FAIL, "Iteration of tagged entries failed") + /* Return the tag */ + *tag = entry->tag_info->tag; -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5C__mark_tagged_entries_cork() */ + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5C_get_tag() */ diff --git a/src/H5Ctest.c b/src/H5Ctest.c index b049a75..876b63a 100644 --- a/src/H5Ctest.c +++ b/src/H5Ctest.c @@ -97,6 +97,7 @@ static int H5C__verify_cork_tag_test_cb(H5C_cache_entry_t *entry, void *_ctx) { H5C_tag_iter_vct_ctx_t *ctx = (H5C_tag_iter_vct_ctx_t *)_ctx; /* Get pointer to iterator context */ + hbool_t is_corked; /* Corked status for entry */ int ret_value = H5_ITER_CONT; /* Return value */ /* Function enter macro */ @@ -106,8 +107,11 @@ H5C__verify_cork_tag_test_cb(H5C_cache_entry_t *entry, void *_ctx) HDassert(entry); HDassert(ctx); + /* Retrieve corked status for entry */ + is_corked = entry->tag_info ? entry->tag_info->corked : FALSE; + /* Verify corked status for entry */ - if(entry->is_corked != ctx->status) + if(is_corked != ctx->status) HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, H5_ITER_ERROR, "bad cork status") done: @@ -308,37 +308,37 @@ done: /*------------------------------------------------------------------------- - * Function: H5Dclose + * Function: H5Dclose * - * Purpose: Closes access to a dataset (DATASET_ID) and releases - * resources used by it. It is illegal to subsequently use that - * same dataset ID in calls to other dataset functions. + * Purpose: Closes access to a dataset (DATASET_ID) and releases + * resources used by it. It is illegal to subsequently use that + * same dataset ID in calls to other dataset functions. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * - * Programmer: Robb Matzke - * Thursday, December 4, 1997 + * Programmer: Robb Matzke + * Thursday, December 4, 1997 * *------------------------------------------------------------------------- */ herr_t H5Dclose(hid_t dset_id) { - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE1("e", "i", dset_id); /* Check args */ if(NULL == H5I_object_verify(dset_id, H5I_DATASET)) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset") + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset") /* * Decrement the counter on the dataset. It will be freed if the count * reaches zero. */ if(H5I_dec_app_ref_always_close(dset_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTDEC, FAIL, "can't decrement count on dataset ID") + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEC, FAIL, "can't decrement count on dataset ID") done: FUNC_LEAVE_API(ret_value) diff --git a/src/H5Dbtree.c b/src/H5Dbtree.c index 85238da..8ef14b9 100644 --- a/src/H5Dbtree.c +++ b/src/H5Dbtree.c @@ -153,6 +153,7 @@ static herr_t H5D__btree_idx_dest(const H5D_chk_idx_info_t *idx_info); /* v1 B-tree indexed chunk I/O ops */ const H5D_chunk_ops_t H5D_COPS_BTREE[1] = {{ + FALSE, /* v1 B-tree indices does not support SWMR access */ H5D__btree_idx_init, /* insert */ H5D__btree_idx_create, /* create */ H5D__btree_idx_is_space_alloc, /* is_space_alloc */ diff --git a/src/H5Dbtree2.c b/src/H5Dbtree2.c index 3b9b803..f687a5d 100644 --- a/src/H5Dbtree2.c +++ b/src/H5Dbtree2.c @@ -105,6 +105,7 @@ static herr_t H5D__bt2_filt_debug(FILE *stream, int indent, int fwidth, /* Helper routine */ static herr_t H5D__bt2_idx_open(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__btree2_idx_depend(const H5D_chk_idx_info_t *idx_info); /* Callback for H5B2_iterate() which is called in H5D__bt2_idx_iterate() */ static int H5D__bt2_idx_iterate_cb(const void *_record, void *_udata); @@ -152,6 +153,7 @@ static herr_t H5D__bt2_idx_dest(const H5D_chk_idx_info_t *idx_info); /* Chunked dataset I/O ops for v2 B-tree indexing */ const H5D_chunk_ops_t H5D_COPS_BT2[1] = {{ + TRUE, /* Fixed array indices support SWMR access */ H5D__bt2_idx_init, /* init */ H5D__bt2_idx_create, /* create */ H5D__bt2_idx_is_space_alloc, /* is_space_alloc */ @@ -623,6 +625,68 @@ H5D__bt2_idx_init(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info, /*------------------------------------------------------------------------- + * Function: H5D__btree2_idx_depend + * + * Purpose: Create flush dependency between v2 B-tree and dataset's + * object header. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Friday, December 18, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__btree2_idx_depend(const H5D_chk_idx_info_t *idx_info) +{ + H5O_t *oh = NULL; /* Object header */ + H5O_loc_t oloc; /* Temporary object header location for dataset */ + H5AC_proxy_entry_t *oh_proxy; /* Dataset's object header proxy */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check args */ + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE); + HDassert(idx_info->pline); + HDassert(idx_info->layout); + HDassert(H5D_CHUNK_IDX_BT2 == idx_info->layout->idx_type); + HDassert(idx_info->storage); + HDassert(H5D_CHUNK_IDX_BT2 == idx_info->storage->idx_type); + HDassert(H5F_addr_defined(idx_info->storage->idx_addr)); + HDassert(idx_info->storage->u.btree2.bt2); + + /* Set up object header location for dataset */ + H5O_loc_reset(&oloc); + oloc.file = idx_info->f; + oloc.addr = idx_info->storage->u.btree.dset_ohdr_addr; + + /* Get header */ + if(NULL == (oh = H5O_protect(&oloc, idx_info->dxpl_id, H5AC__READ_ONLY_FLAG, TRUE))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTPROTECT, FAIL, "unable to protect object header") + + /* Retrieve the dataset's object header proxy */ + if(NULL == (oh_proxy = H5O_get_proxy(oh))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get dataset object header proxy") + + /* Make the v2 B-tree a child flush dependency of the dataset's object header proxy */ + if(H5B2_depend(idx_info->storage->u.btree2.bt2, idx_info->dxpl_id, oh_proxy) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL, "unable to create flush dependency on object header proxy") + +done: + /* Release the object header from the cache */ + if(oh && H5O_unprotect(&oloc, idx_info->dxpl_id, oh, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTUNPROTECT, FAIL, "unable to release object header") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__btree2_idx_depend() */ + + +/*------------------------------------------------------------------------- * Function: H5D__bt2_idx_open() * * Purpose: Opens an existing v2 B-tree. @@ -667,6 +731,11 @@ H5D__bt2_idx_open(const H5D_chk_idx_info_t *idx_info) if(NULL == (idx_info->storage->u.btree2.bt2 = H5B2_open(idx_info->f, idx_info->dxpl_id, idx_info->storage->idx_addr, &u_ctx))) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't open v2 B-tree for tracking chunked dataset") + /* Check for SWMR writes to the file */ + if(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE) + if(H5D__btree2_idx_depend(idx_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL, "unable to create flush dependency on object header") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__bt2_idx_open() */ @@ -738,6 +807,11 @@ H5D__bt2_idx_create(const H5D_chk_idx_info_t *idx_info) if(H5B2_get_addr(idx_info->storage->u.btree2.bt2, &(idx_info->storage->idx_addr)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get v2 B-tree address for tracking chunked dataset") + /* Check for SWMR writes to the file */ + if(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE) + if(H5D__btree2_idx_depend(idx_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL, "unable to create flush dependency on object header") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__bt2_idx_create() */ @@ -1186,7 +1260,7 @@ H5D__bt2_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t *u /* Remove the record for the "dataset chunk" object from the v2 B-tree */ /* (space in the file for the object is freed in the 'remove' callback) */ - if(H5B2_remove(bt2, idx_info->dxpl_id, &bt2_udata, H5D__bt2_remove_cb, &remove_udata) < 0) + if(H5B2_remove(bt2, idx_info->dxpl_id, &bt2_udata, (H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE) ? NULL : H5D__bt2_remove_cb, &remove_udata) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTREMOVE, FAIL, "can't remove object from B-tree") done: @@ -1243,8 +1317,11 @@ H5D__bt2_idx_delete(const H5D_chk_idx_info_t *idx_info) remove_udata.f = idx_info->f; remove_udata.dxpl_id = idx_info->dxpl_id; - /* Set remove operation. */ - remove_op = H5D__bt2_remove_cb; + /* Set remove operation. Do not remove chunks in SWMR_WRITE mode */ + if(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE) + remove_op = NULL; + else + remove_op = H5D__bt2_remove_cb; /* Delete the v2 B-tree */ /*(space in the file for each object is freed in the 'remove' callback) */ diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index dde83fe..7a646af 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -4851,7 +4851,6 @@ H5D__chunk_prune_by_extent(H5D_t *dset, hid_t dxpl_id, const hsize_t *old_dim) * will never change. */ chk_store.chunk.scaled = scaled; H5D_BUILD_IO_INFO_RD(&chk_io_info, dset, dxpl_cache, dxpl_id, H5AC_rawdata_dxpl_id, &chk_store, NULL); - chk_io_info.raw_dxpl_id = H5AC_rawdata_dxpl_id; /* Compose chunked index info struct */ idx_info.f = dset->oloc.file; @@ -6293,8 +6292,13 @@ H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info, const H5F_block_t *old /* Check for chunk being same size */ if(new_chunk->length != old_chunk->length) { /* Release previous chunk */ - if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, old_chunk->offset, old_chunk->length) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk") + /* Only free the old location if not doing SWMR writes - otherwise + * we must keep the old chunk around in case a reader has an + * outdated version of the B-tree node + */ + if(!(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)) + if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, old_chunk->offset, old_chunk->length) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk") alloc_chunk = TRUE; } /* end if */ else { diff --git a/src/H5Dearray.c b/src/H5Dearray.c index 9f95b30..e9dbd0d 100644 --- a/src/H5Dearray.c +++ b/src/H5Dearray.c @@ -148,6 +148,7 @@ static herr_t H5D__earray_idx_dest(const H5D_chk_idx_info_t *idx_info); /* Generic extensible array routines */ static herr_t H5D__earray_idx_open(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__earray_idx_depend(const H5D_chk_idx_info_t *idx_info); /*********************/ @@ -156,6 +157,7 @@ static herr_t H5D__earray_idx_open(const H5D_chk_idx_info_t *idx_info); /* Extensible array indexed chunk I/O ops */ const H5D_chunk_ops_t H5D_COPS_EARRAY[1] = {{ + TRUE, /* Extensible array indices support SWMR access */ H5D__earray_idx_init, /* init */ H5D__earray_idx_create, /* create */ H5D__earray_idx_is_space_alloc, /* is_space_alloc */ @@ -657,7 +659,7 @@ H5D__earray_crt_dbg_context(H5F_t *f, hid_t dxpl_id, haddr_t obj_addr) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, NULL, "can't get layout info") /* close the object header */ - if(H5O_close(&obj_loc) < 0) + if(H5O_close(&obj_loc, NULL) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, NULL, "can't close object header") /* Create user data */ @@ -676,7 +678,7 @@ done: /* Close object header */ if(obj_opened) { - if(H5O_close(&obj_loc) < 0) + if(H5O_close(&obj_loc, NULL) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, NULL, "can't close object header") } /* end if */ } /* end if */ @@ -716,6 +718,68 @@ H5D__earray_dst_dbg_context(void *_dbg_ctx) /*------------------------------------------------------------------------- + * Function: H5D__earray_idx_depend + * + * Purpose: Create flush dependency between extensible array and dataset's + * object header. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Tuesday, June 2, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__earray_idx_depend(const H5D_chk_idx_info_t *idx_info) +{ + H5O_t *oh = NULL; /* Object header */ + H5O_loc_t oloc; /* Temporary object header location for dataset */ + H5AC_proxy_entry_t *oh_proxy; /* Dataset's object header proxy */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check args */ + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE); + HDassert(idx_info->pline); + HDassert(idx_info->layout); + HDassert(H5D_CHUNK_IDX_EARRAY == idx_info->layout->idx_type); + HDassert(idx_info->storage); + HDassert(H5D_CHUNK_IDX_EARRAY == idx_info->storage->idx_type); + HDassert(H5F_addr_defined(idx_info->storage->idx_addr)); + HDassert(idx_info->storage->u.earray.ea); + + /* Set up object header location for dataset */ + H5O_loc_reset(&oloc); + oloc.file = idx_info->f; + oloc.addr = idx_info->storage->u.earray.dset_ohdr_addr; + + /* Get header */ + if(NULL == (oh = H5O_protect(&oloc, idx_info->dxpl_id, H5AC__READ_ONLY_FLAG, TRUE))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTPROTECT, FAIL, "unable to protect object header") + + /* Retrieve the dataset's object header proxy */ + if(NULL == (oh_proxy = H5O_get_proxy(oh))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get dataset object header proxy") + + /* Make the extensible array a child flush dependency of the dataset's object header */ + if(H5EA_depend(idx_info->storage->u.earray.ea, idx_info->dxpl_id, oh_proxy) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL, "unable to create flush dependency on object header proxy") + +done: + /* Release the object header from the cache */ + if(oh && H5O_unprotect(&oloc, idx_info->dxpl_id, oh, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTUNPROTECT, FAIL, "unable to release object header") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__earray_idx_depend() */ + + +/*------------------------------------------------------------------------- * Function: H5D__earray_idx_open * * Purpose: Opens an existing extensible array. @@ -760,6 +824,11 @@ H5D__earray_idx_open(const H5D_chk_idx_info_t *idx_info) if(NULL == (idx_info->storage->u.earray.ea = H5EA_open(idx_info->f, idx_info->dxpl_id, idx_info->storage->idx_addr, &udata))) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't open extensible array") + /* Check for SWMR writes to the file */ + if(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE) + if(H5D__earray_idx_depend(idx_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL, "unable to create flush dependency on object header") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__earray_idx_open() */ @@ -910,6 +979,11 @@ H5D__earray_idx_create(const H5D_chk_idx_info_t *idx_info) if(H5EA_get_addr(idx_info->storage->u.earray.ea, &(idx_info->storage->idx_addr)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't query extensible array address") + /* Check for SWMR writes to the file */ + if(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE) + if(H5D__earray_idx_depend(idx_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL, "unable to create flush dependency on object header") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__earray_idx_create() */ @@ -1355,11 +1429,13 @@ H5D__earray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t if(H5EA_get(ea, idx_info->dxpl_id, idx, &elmt) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info") - /* Remove raw data chunk from file */ + /* Remove raw data chunk from file if not doing SWMR writes */ HDassert(H5F_addr_defined(elmt.addr)); - H5_CHECK_OVERFLOW(elmt.nbytes, /*From: */uint32_t, /*To: */hsize_t); - if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, elmt.addr, (hsize_t)elmt.nbytes) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk") + if(!(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)) { + H5_CHECK_OVERFLOW(elmt.nbytes, /*From: */uint32_t, /*To: */hsize_t); + if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, elmt.addr, (hsize_t)elmt.nbytes) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk") + } /* end if */ /* Reset the info about the chunk for the index */ elmt.addr = HADDR_UNDEF; @@ -1375,11 +1451,13 @@ H5D__earray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t if(H5EA_get(ea, idx_info->dxpl_id, idx, &addr) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") - /* Remove raw data chunk from file */ + /* Remove raw data chunk from file if not doing SWMR writes */ HDassert(H5F_addr_defined(addr)); - H5_CHECK_OVERFLOW(idx_info->layout->size, /*From: */uint32_t, /*To: */hsize_t); - if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, addr, (hsize_t)idx_info->layout->size) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk") + if(!(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)) { + H5_CHECK_OVERFLOW(idx_info->layout->size, /*From: */uint32_t, /*To: */hsize_t); + if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, addr, (hsize_t)idx_info->layout->size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk") + } /* end if */ /* Reset the address of the chunk for the index */ addr = HADDR_UNDEF; diff --git a/src/H5Dfarray.c b/src/H5Dfarray.c index 76ea229..6b95e12 100644 --- a/src/H5Dfarray.c +++ b/src/H5Dfarray.c @@ -145,8 +145,9 @@ static herr_t H5D__farray_idx_reset(H5O_storage_chunk_t *storage, hbool_t reset_ static herr_t H5D__farray_idx_dump(const H5O_storage_chunk_t *storage, FILE *stream); static herr_t H5D__farray_idx_dest(const H5D_chk_idx_info_t *idx_info); -/* Generic extensible array routines */ +/* Generic fixed array routines */ static herr_t H5D__farray_idx_open(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D__farray_idx_depend(const H5D_chk_idx_info_t *idx_info); /*********************/ /* Package Variables */ @@ -154,6 +155,7 @@ static herr_t H5D__farray_idx_open(const H5D_chk_idx_info_t *idx_info); /* Fixed array indexed chunk I/O ops */ const H5D_chunk_ops_t H5D_COPS_FARRAY[1] = {{ + TRUE, /* Fixed array indices support SWMR access */ H5D__farray_idx_init, /* init */ H5D__farray_idx_create, /* create */ H5D__farray_idx_is_space_alloc, /* is_space_alloc */ @@ -497,7 +499,7 @@ H5D__farray_crt_dbg_context(H5F_t *f, hid_t dxpl_id, haddr_t obj_addr) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, NULL, "can't get layout info") /* close the object header */ - if(H5O_close(&obj_loc) < 0) + if(H5O_close(&obj_loc, NULL) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, NULL, "can't close object header") /* Create user data */ @@ -516,7 +518,7 @@ done: /* Close object header */ if(obj_opened) { - if(H5O_close(&obj_loc) < 0) + if(H5O_close(&obj_loc, NULL) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, NULL, "can't close object header") } /* end if */ } /* end if */ @@ -716,6 +718,68 @@ H5D__farray_filt_debug(FILE *stream, int indent, int fwidth, hsize_t idx, /*------------------------------------------------------------------------- + * Function: H5D__farray_idx_depend + * + * Purpose: Create flush dependency between fixed array and dataset's + * object header. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi + * Thursday, April 30, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__farray_idx_depend(const H5D_chk_idx_info_t *idx_info) +{ + H5O_t *oh = NULL; /* Object header */ + H5O_loc_t oloc; /* Temporary object header location for dataset */ + H5AC_proxy_entry_t *oh_proxy; /* Dataset's object header proxy */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check args */ + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE); + HDassert(idx_info->pline); + HDassert(idx_info->layout); + HDassert(H5D_CHUNK_IDX_FARRAY == idx_info->layout->idx_type); + HDassert(idx_info->storage); + HDassert(H5D_CHUNK_IDX_FARRAY == idx_info->storage->idx_type); + HDassert(H5F_addr_defined(idx_info->storage->idx_addr)); + HDassert(idx_info->storage->u.farray.fa); + + /* Set up object header location for dataset */ + H5O_loc_reset(&oloc); + oloc.file = idx_info->f; + oloc.addr = idx_info->storage->u.farray.dset_ohdr_addr; + + /* Get header */ + if(NULL == (oh = H5O_protect(&oloc, idx_info->dxpl_id, H5AC__READ_ONLY_FLAG, TRUE))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTPROTECT, FAIL, "unable to protect object header") + + /* Retrieve the dataset's object header proxy */ + if(NULL == (oh_proxy = H5O_get_proxy(oh))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get dataset object header proxy") + + /* Make the fixed array a child flush dependency of the dataset's object header proxy */ + if(H5FA_depend(idx_info->storage->u.farray.fa, idx_info->dxpl_id, oh_proxy) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL, "unable to create flush dependency on object header proxy") + +done: + /* Release the object header from the cache */ + if(oh && H5O_unprotect(&oloc, idx_info->dxpl_id, oh, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTUNPROTECT, FAIL, "unable to release object header") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__farray_idx_depend() */ + + +/*------------------------------------------------------------------------- * Function: H5D__farray_idx_init * * Purpose: Initialize the indexing information for a dataset. @@ -784,6 +848,11 @@ H5D__farray_idx_open(const H5D_chk_idx_info_t *idx_info) if(NULL == (idx_info->storage->u.farray.fa = H5FA_open(idx_info->f, idx_info->dxpl_id, idx_info->storage->idx_addr, &udata))) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't open fixed array") + /* Check for SWMR writes to the file */ + if(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE) + if(H5D__farray_idx_depend(idx_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL, "unable to create flush dependency on object header") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__farray_idx_open() */ @@ -860,6 +929,11 @@ H5D__farray_idx_create(const H5D_chk_idx_info_t *idx_info) if(H5FA_get_addr(idx_info->storage->u.farray.fa, &(idx_info->storage->idx_addr)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't query fixed array address") + /* Check for SWMR writes to the file */ + if(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE) + if(H5D__farray_idx_depend(idx_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL, "unable to create flush dependency on object header") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__farray_idx_create() */ @@ -1224,11 +1298,13 @@ H5D__farray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t if(H5FA_get(fa, idx_info->dxpl_id, idx, &elmt) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info") - /* Remove raw data chunk from file */ + /* Remove raw data chunk from file if not doing SWMR writes */ HDassert(H5F_addr_defined(elmt.addr)); - H5_CHECK_OVERFLOW(elmt.nbytes, /*From: */uint32_t, /*To: */hsize_t); - if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, elmt.addr, (hsize_t)elmt.nbytes) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk") + if(!(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)) { + H5_CHECK_OVERFLOW(elmt.nbytes, /*From: */uint32_t, /*To: */hsize_t); + if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, elmt.addr, (hsize_t)elmt.nbytes) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk") + } /* end if */ /* Reset the info about the chunk for the index */ elmt.addr = HADDR_UNDEF; @@ -1244,11 +1320,13 @@ H5D__farray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t if(H5FA_get(fa, idx_info->dxpl_id, idx, &addr) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") - /* Remove raw data chunk from file */ + /* Remove raw data chunk from file if not doing SWMR writes */ HDassert(H5F_addr_defined(addr)); - H5_CHECK_OVERFLOW(idx_info->layout->size, /*From: */uint32_t, /*To: */hsize_t); - if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, addr, (hsize_t)idx_info->layout->size) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk") + if(!(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)) { + H5_CHECK_OVERFLOW(idx_info->layout->size, /*From: */uint32_t, /*To: */hsize_t); + if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, addr, (hsize_t)idx_info->layout->size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk") + } /* end if */ /* Reset the address of the chunk for the index */ addr = HADDR_UNDEF; diff --git a/src/H5Dint.c b/src/H5Dint.c index 8a00be2..5a11581 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -1303,7 +1303,7 @@ done: if(H5F_addr_defined(new_dset->oloc.addr)) { if(H5O_dec_rc_by_loc(&(new_dset->oloc), dxpl_id) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTDEC, NULL, "unable to decrement refcount on newly created object") - if(H5O_close(&(new_dset->oloc)) < 0) + if(H5O_close(&(new_dset->oloc), NULL) < 0) HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, NULL, "unable to release object header") if(file) { if(H5O_delete(file, dxpl_id, new_dset->oloc.addr) < 0) @@ -1738,7 +1738,7 @@ H5D__open_oid(H5D_t *dataset, hid_t dapl_id, hid_t dxpl_id) done: if(ret_value < 0) { - if(H5F_addr_defined(dataset->oloc.addr) && H5O_close(&(dataset->oloc)) < 0) + if(H5F_addr_defined(dataset->oloc.addr) && H5O_close(&(dataset->oloc), NULL) < 0) HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release object header") if(dataset->shared) { if(layout_init) @@ -1780,9 +1780,10 @@ done: herr_t H5D_close(H5D_t *dataset) { - hbool_t free_failed = FALSE; - hbool_t corked; /* Whether the dataset is corked or not */ - herr_t ret_value = SUCCEED; /* Return value */ + hbool_t free_failed = FALSE; /* Set if freeing sub-components failed */ + hbool_t corked; /* Whether the dataset is corked or not */ + hbool_t file_closed = TRUE; /* H5O_close also closed the file? */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -1797,6 +1798,7 @@ H5D_close(H5D_t *dataset) dataset->shared->fo_count--; if(dataset->shared->fo_count == 0) { + /* Flush the dataset's information. Continue to close even if it fails. */ if(H5D__flush_real(dataset, H5AC_ind_read_dxpl_id) < 0) HDONE_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to flush cached dataset info") @@ -1891,12 +1893,12 @@ H5D_close(H5D_t *dataset) (H5O_msg_reset(H5O_FILL_ID, &dataset->shared->dcpl_cache.fill) < 0) || (H5O_msg_reset(H5O_EFL_ID, &dataset->shared->dcpl_cache.efl) < 0); - /* Uncork cache entries with object address tag */ - if(H5AC_cork(dataset->oloc.file, dataset->oloc.addr, H5AC__GET_CORKED, &corked) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve an object's cork status") - if(corked) - if(H5AC_cork(dataset->oloc.file, dataset->oloc.addr, H5AC__UNCORK, NULL) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTUNCORK, FAIL, "unable to uncork an object") + /* Uncork cache entries with object address tag */ + if(H5AC_cork(dataset->oloc.file, dataset->oloc.addr, H5AC__GET_CORKED, &corked) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve an object's cork status") + if(corked) + if(H5AC_cork(dataset->oloc.file, dataset->oloc.addr, H5AC__UNCORK, NULL) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTUNCORK, FAIL, "unable to uncork an object") /* * Release datatype, dataspace and creation property list -- there isn't @@ -1914,9 +1916,17 @@ H5D_close(H5D_t *dataset) /* Close the dataset object */ /* (This closes the file, if this is the last object open) */ - if(H5O_close(&(dataset->oloc)) < 0) + if(H5O_close(&(dataset->oloc), &file_closed) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release object header") + /* Evict dataset metadata if evicting on close */ + if(!file_closed && H5F_SHARED(dataset->oloc.file) && H5F_EVICT_ON_CLOSE(dataset->oloc.file)) { + if(H5AC_flush_tagged_metadata(dataset->oloc.file, dataset->oloc.addr, H5AC_ind_read_dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush tagged metadata") + if(H5AC_evict_tagged_metadata(dataset->oloc.file, dataset->oloc.addr, FALSE, H5AC_ind_read_dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to evict tagged metadata") + } /* end if */ + /* * Free memory. Before freeing the memory set the file pointer to NULL. * We always check for a null file pointer in other H5D functions to be @@ -1924,8 +1934,8 @@ H5D_close(H5D_t *dataset) * above). */ dataset->oloc.file = NULL; - dataset->shared = H5FL_FREE(H5D_shared_t, dataset->shared); + } /* end if */ else { /* Decrement the ref. count for this object in the top file */ @@ -1934,7 +1944,7 @@ H5D_close(H5D_t *dataset) /* Check reference count for this object in the top file */ if(H5FO_top_count(dataset->oloc.file, dataset->oloc.addr) == 0) { - if(H5O_close(&(dataset->oloc)) < 0) + if(H5O_close(&(dataset->oloc), NULL) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to close") } /* end if */ else @@ -1943,16 +1953,16 @@ H5D_close(H5D_t *dataset) HGOTO_ERROR(H5E_DATASET, H5E_CANTRELEASE, FAIL, "problem attempting to free location") } /* end else */ - /* Release the dataset's path info */ - if(H5G_name_free(&(dataset->path)) < 0) - free_failed = TRUE; + /* Release the dataset's path info */ + if(H5G_name_free(&(dataset->path)) < 0) + free_failed = TRUE; /* Free the dataset's memory structure */ dataset = H5FL_FREE(H5D_t, dataset); /* Check if anything failed in the middle... */ if(free_failed) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "couldn't free a component of the dataset, but the dataset was freed anyway.") + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "couldn't free a component of the dataset, but the dataset was freed anyway.") done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Dnone.c b/src/H5Dnone.c index 677ca67..0cadac2 100644 --- a/src/H5Dnone.c +++ b/src/H5Dnone.c @@ -82,6 +82,7 @@ static herr_t H5D__none_idx_dump(const H5O_storage_chunk_t *storage, FILE *strea /* Non Index chunk I/O ops */ const H5D_chunk_ops_t H5D_COPS_NONE[1] = {{ + FALSE, /* Non-indexed chunking don't current support SWMR access */ NULL, /* init */ H5D__none_idx_create, /* create */ H5D__none_idx_is_space_alloc, /* is_space_alloc */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 05f49d1..f54a9f2 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -325,6 +325,7 @@ typedef herr_t (*H5D_chunk_dest_func_t)(const H5D_chk_idx_info_t *idx_info); /* Typedef for grouping chunk I/O routines */ typedef struct H5D_chunk_ops_t { + hbool_t can_swim; /* Flag to indicate that the index supports SWMR access */ H5D_chunk_init_func_t init; /* Routine to initialize indexing information in memory */ H5D_chunk_create_func_t create; /* Routine to create chunk index */ H5D_chunk_is_space_alloc_func_t is_space_alloc; /* Query routine to determine if storage/index is allocated */ @@ -394,6 +395,10 @@ typedef struct H5D_chunk_cached_t { unsigned filter_mask; /*excluded filters */ } H5D_chunk_cached_t; +/****************************/ +/* Virtual dataset typedefs */ +/****************************/ + /* List of files held open during refresh operations */ typedef struct H5D_virtual_held_file_t { H5F_t *file; /* Pointer to file held open */ @@ -685,6 +690,8 @@ H5_DLL herr_t H5D__chunk_direct_write(const H5D_t *dset, hid_t dxpl_id, uint32_t #ifdef H5D_CHUNK_DEBUG H5_DLL herr_t H5D__chunk_stats(const H5D_t *dset, hbool_t headers); #endif /* H5D_CHUNK_DEBUG */ + +/* format convert */ H5_DLL herr_t H5D__chunk_format_convert(H5D_t *dset, H5D_chk_idx_info_t *idx_info, H5D_chk_idx_info_t *new_idx_info); /* Functions that operate on compact dataset storage */ @@ -770,6 +777,7 @@ H5_DLL htri_t H5D__mpio_opt_possible(const H5D_io_info_t *io_info, H5_DLL herr_t H5D__layout_version_test(hid_t did, unsigned *version); H5_DLL herr_t H5D__layout_contig_size_test(hid_t did, hsize_t *size); H5_DLL herr_t H5D__layout_idx_type_test(hid_t did, H5D_chunk_index_t *idx_type); +H5_DLL herr_t H5D__layout_type_test(hid_t did, H5D_layout_t *layout_type); H5_DLL herr_t H5D__current_cache_size_test(hid_t did, size_t *nbytes_used, int *nused); #endif /* H5D_TESTING */ diff --git a/src/H5Dsingle.c b/src/H5Dsingle.c index e09a5cf..04b8971 100644 --- a/src/H5Dsingle.c +++ b/src/H5Dsingle.c @@ -84,6 +84,7 @@ static herr_t H5D__single_idx_dump(const H5O_storage_chunk_t *storage, FILE *str /* Non Index chunk I/O ops */ const H5D_chunk_ops_t H5D_COPS_SINGLE[1] = {{ + FALSE, /* Single Chunk indexing doesn't current support SWMR access */ H5D__single_idx_init, /* init */ H5D__single_idx_create, /* create */ H5D__single_idx_is_space_alloc, /* is_space_alloc */ diff --git a/src/H5Dtest.c b/src/H5Dtest.c index c3b0b19..56f14f7 100644 --- a/src/H5Dtest.c +++ b/src/H5Dtest.c @@ -144,6 +144,47 @@ done: /*-------------------------------------------------------------------------- NAME + H5D__layout_type_test + PURPOSE + Determine the storage layout type for a dataset + USAGE + herr_t H5D__layout_type_test(did, layout_type) + hid_t did; IN: Dataset to query + H5D_layout_t *layout_type; OUT: Pointer to location to place layout info + RETURNS + Non-negative on success, negative on failure + DESCRIPTION + Checks the layout type for a dataset. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + DO NOT USE THIS FUNCTION FOR ANYTHING EXCEPT TESTING + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5D__layout_type_test(hid_t did, H5D_layout_t *layout_type) +{ + H5D_t *dset; /* Pointer to dataset to query */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_PACKAGE + + HDassert(layout_type); + + /* Check args */ + if(NULL == (dset = (H5D_t *)H5I_object_verify(did, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset") + + if(layout_type) + *layout_type = dset->shared->layout.type; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D__layout_type_test() */ + + +/*-------------------------------------------------------------------------- + NAME H5D__layout_idx_type_test PURPOSE Determine the storage layout index type for a dataset's layout information diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 9a90fa7..c85b1e9 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -768,7 +768,7 @@ H5D__virtual_open_source_dset(const H5D_t *vdset, /* Check if we need to open the source file */ if(HDstrcmp(source_dset->file_name, ".")) { /* Open the source file */ - if(NULL == (src_file = H5F_open(source_dset->file_name, H5F_INTENT(vdset->oloc.file) & H5F_ACC_RDWR, H5P_FILE_CREATE_DEFAULT, vdset->shared->layout.storage.u.virt.source_fapl, dxpl_id))) + if(NULL == (src_file = H5F_open(source_dset->file_name, H5F_INTENT(vdset->oloc.file) & (H5F_ACC_RDWR | H5F_ACC_SWMR_WRITE | H5F_ACC_SWMR_READ), H5P_FILE_CREATE_DEFAULT, vdset->shared->layout.storage.u.virt.source_fapl, dxpl_id))) H5E_clear_stack(NULL); /* Quick hack until proper support for H5Fopen with missing file is implemented */ else src_file_open = TRUE; @@ -807,7 +807,7 @@ H5D__virtual_open_source_dset(const H5D_t *vdset, done: /* Close source file */ if(src_file_open) - if(H5F_try_close(src_file) < 0) + if(H5F_try_close(src_file, NULL) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTCLOSEFILE, FAIL, "can't close source file") FUNC_LEAVE_NOAPI(ret_value) @@ -2931,7 +2931,7 @@ H5D__virtual_release_source_dset_files(H5D_virtual_held_file_t *head) * essentially "private" to the virtual dataset, since it wasn't * opened through an API routine -QAK) */ - if(H5F_try_close(head->file) < 0) + if(H5F_try_close(head->file, NULL) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEFILE, FAIL, "problem attempting file close") /* Delete node */ @@ -78,6 +78,8 @@ static herr_t H5EA__lookup_elmt(const H5EA_t *ea, hid_t dxpl_id, hsize_t idx, hbool_t will_extend, unsigned thing_acc, void **thing, uint8_t **thing_elmt_buf, hsize_t *thing_elmt_idx, H5EA__unprotect_func_t *thing_unprot_func); +static H5EA_t *H5EA__new(H5F_t *f, hid_t dxpl_id, haddr_t ea_addr, + hbool_t from_open, void *ctx_udata); /*********************/ @@ -116,49 +118,45 @@ H5FL_BLK_DEFINE(ea_native_elmt); /*------------------------------------------------------------------------- - * Function: H5EA_create + * Function: H5EA__new * - * Purpose: Creates a new empty extensible array in the file. + * Purpose: Allocate and initialize a new extensible array wrapper in memory * - * Return: Pointer to earray wrapper on success + * Return: Pointer to earray wrapper success * NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org - * Jun 17 2008 + * koziol@lbl.gov + * Oct 10 2016 * *------------------------------------------------------------------------- */ -BEGIN_FUNC(PRIV, ERR, +BEGIN_FUNC(STATIC, ERR, H5EA_t *, NULL, NULL, -H5EA_create(H5F_t *f, hid_t dxpl_id, const H5EA_create_t *cparam, void *ctx_udata)) +H5EA__new(H5F_t *f, hid_t dxpl_id, haddr_t ea_addr, hbool_t from_open, void *ctx_udata)) /* Local variables */ H5EA_t *ea = NULL; /* Pointer to new extensible array */ H5EA_hdr_t *hdr = NULL; /* The extensible array header information */ - haddr_t ea_addr; /* Array header address */ /* * Check arguments. */ HDassert(f); - HDassert(cparam); - - /* H5EA interface sanity check */ - HDcompile_assert(H5EA_NUM_CLS_ID == NELMTS(H5EA_client_class_g)); - - /* Create extensible array header */ - if(HADDR_UNDEF == (ea_addr = H5EA__hdr_create(f, dxpl_id, cparam, ctx_udata))) - H5E_THROW(H5E_CANTINIT, "can't create extensible array header") + HDassert(H5F_addr_defined(ea_addr)); /* Allocate extensible array wrapper */ - if(NULL == (ea = H5FL_MALLOC(H5EA_t))) + if(NULL == (ea = H5FL_CALLOC(H5EA_t))) H5E_THROW(H5E_CANTALLOC, "memory allocation failed for extensible array info") /* Lock the array header into memory */ - if(NULL == (hdr = H5EA__hdr_protect(f, dxpl_id, ea_addr, ctx_udata, H5AC__NO_FLAGS_SET))) + if(NULL == (hdr = H5EA__hdr_protect(f, dxpl_id, ea_addr, ctx_udata, H5AC__READ_ONLY_FLAG))) H5E_THROW(H5E_CANTPROTECT, "unable to load extensible array header") + /* Check for pending array deletion */ + if(from_open && hdr->pending_delete) + H5E_THROW(H5E_CANTOPENOBJ, "can't open extensible array pending deletion") + /* Point extensible array wrapper at header and bump it's ref count */ ea->hdr = hdr; if(H5EA__hdr_incr(ea->hdr) < 0) @@ -182,6 +180,57 @@ CATCH if(ea && H5EA_close(ea, dxpl_id) < 0) H5E_THROW(H5E_CLOSEERROR, "unable to close extensible array") +END_FUNC(STATIC) /* end H5EA__new() */ + + +/*------------------------------------------------------------------------- + * Function: H5EA_create + * + * Purpose: Creates a new empty extensible array in the file. + * + * Return: Pointer to earray wrapper on success + * NULL on failure + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Jun 17 2008 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(PRIV, ERR, +H5EA_t *, NULL, NULL, +H5EA_create(H5F_t *f, hid_t dxpl_id, const H5EA_create_t *cparam, void *ctx_udata)) + + /* Local variables */ + H5EA_t *ea = NULL; /* Pointer to new extensible array */ + haddr_t ea_addr; /* Array header address */ + + /* + * Check arguments. + */ + HDassert(f); + HDassert(cparam); + + /* H5EA interface sanity check */ + HDcompile_assert(H5EA_NUM_CLS_ID == NELMTS(H5EA_client_class_g)); + + /* Create extensible array header */ + if(HADDR_UNDEF == (ea_addr = H5EA__hdr_create(f, dxpl_id, cparam, ctx_udata))) + H5E_THROW(H5E_CANTINIT, "can't create extensible array header") + + /* Allocate and initialize new extensible array wrapper */ + if(NULL == (ea = H5EA__new(f, dxpl_id, ea_addr, FALSE, ctx_udata))) + H5E_THROW(H5E_CANTINIT, "allocation and/or initialization failed for extensible array wrapper") + + /* Set the return value */ + ret_value = ea; + +CATCH + + if(!ret_value) + if(ea && H5EA_close(ea, dxpl_id) < 0) + H5E_THROW(H5E_CLOSEERROR, "unable to close extensible array") + END_FUNC(PRIV) /* end H5EA_create() */ @@ -205,7 +254,6 @@ H5EA_open(H5F_t *f, hid_t dxpl_id, haddr_t ea_addr, void *ctx_udata)) /* Local variables */ H5EA_t *ea = NULL; /* Pointer to new extensible array wrapper */ - H5EA_hdr_t *hdr = NULL; /* The extensible array header information */ /* * Check arguments. @@ -213,37 +261,15 @@ H5EA_open(H5F_t *f, hid_t dxpl_id, haddr_t ea_addr, void *ctx_udata)) HDassert(f); HDassert(H5F_addr_defined(ea_addr)); - /* Load the array header into memory */ - if(NULL == (hdr = H5EA__hdr_protect(f, dxpl_id, ea_addr, ctx_udata, H5AC__READ_ONLY_FLAG))) - H5E_THROW(H5E_CANTPROTECT, "unable to load extensible array header, address = %llu", (unsigned long long)ea_addr) - - /* Check for pending array deletion */ - if(hdr->pending_delete) - H5E_THROW(H5E_CANTOPENOBJ, "can't open extensible array pending deletion") - - /* Create fractal heap info */ - if(NULL == (ea = H5FL_MALLOC(H5EA_t))) - H5E_THROW(H5E_CANTALLOC, "memory allocation failed for extensible array info") - - /* Point extensible array wrapper at header */ - ea->hdr = hdr; - if(H5EA__hdr_incr(ea->hdr) < 0) - H5E_THROW(H5E_CANTINC, "can't increment reference count on shared array header") - - /* Increment # of files using this array header */ - if(H5EA__hdr_fuse_incr(ea->hdr) < 0) - H5E_THROW(H5E_CANTINC, "can't increment file reference count on shared array header") - - /* Set file pointer for this array open context */ - ea->f = f; + /* Allocate and initialize new extensible array wrapper */ + if(NULL == (ea = H5EA__new(f, dxpl_id, ea_addr, TRUE, ctx_udata))) + H5E_THROW(H5E_CANTINIT, "allocation and/or initialization failed for extensible array wrapper") /* Set the return value */ ret_value = ea; CATCH - if(hdr && H5EA__hdr_unprotect(hdr, dxpl_id, H5AC__NO_FLAGS_SET) < 0) - H5E_THROW(H5E_CANTUNPROTECT, "unable to release extensible array header") if(!ret_value) if(ea && H5EA_close(ea, dxpl_id) < 0) H5E_THROW(H5E_CLOSEERROR, "unable to close extensible array") @@ -756,8 +782,8 @@ END_FUNC(PRIV) /* end H5EA_get() */ /*------------------------------------------------------------------------- * Function: H5EA_depend * - * Purpose: Make a child flush dependency between the extensible array's - * header and another piece of metadata in the file. + * Purpose: Make a child flush dependency between the extensible array + * and another piece of metadata in the file. * * Return: SUCCEED/FAIL * @@ -769,7 +795,7 @@ END_FUNC(PRIV) /* end H5EA_get() */ */ BEGIN_FUNC(PRIV, ERR, herr_t, SUCCEED, FAIL, -H5EA_depend(H5AC_info_t *parent_entry, H5EA_t *ea)) +H5EA_depend(H5EA_t *ea, hid_t dxpl_id, H5AC_proxy_entry_t *parent)) /* Local variables */ H5EA_hdr_t *hdr = ea->hdr; /* Header for EA */ @@ -779,13 +805,25 @@ H5EA_depend(H5AC_info_t *parent_entry, H5EA_t *ea)) */ HDassert(ea); HDassert(hdr); + HDassert(parent); - /* Set the shared array header's file context for this operation */ - hdr->f = ea->f; + /* + * Check to see if a flush dependency between the extensible array + * and another data structure in the file has already been set up. + * If it hasn't, do so now. + */ + if(NULL == hdr->parent) { + /* Sanity check */ + HDassert(hdr->top_proxy); + + /* Set the shared array header's file context for this operation */ + hdr->f = ea->f; - /* Set up flush dependency between parent entry and extensible array header */ - if(H5EA__create_flush_depend(parent_entry, (H5AC_info_t *)hdr) < 0) - H5E_THROW(H5E_CANTDEPEND, "unable to create flush dependency on file metadata") + /* Add the extensible array as a child of the parent (proxy) */ + if(H5AC_proxy_entry_add_child(parent, hdr->f, dxpl_id, hdr->top_proxy) < 0) + H5E_THROW(H5E_CANTSET, "unable to add extensible array as child of proxy") + hdr->parent = parent; + } /* end if */ CATCH @@ -818,26 +856,28 @@ H5EA_close(H5EA_t *ea, hid_t dxpl_id)) */ HDassert(ea); - /* Decrement file reference & check if this is the last open extensible array using the shared array header */ - if(0 == H5EA__hdr_fuse_decr(ea->hdr)) { - /* Set the shared array header's file context for this operation */ - ea->hdr->f = ea->f; - - /* Shut down anything that can't be put in the header's 'flush' callback */ - - /* Check for pending array deletion */ - if(ea->hdr->pending_delete) { - /* Set local info, so array deletion can occur after decrementing the - * header's ref count - */ - pending_delete = TRUE; - ea_addr = ea->hdr->addr; + /* Close the header, if it was set */ + if(ea->hdr) { + /* Decrement file reference & check if this is the last open extensible array using the shared array header */ + if(0 == H5EA__hdr_fuse_decr(ea->hdr)) { + /* Set the shared array header's file context for this operation */ + ea->hdr->f = ea->f; + + /* Shut down anything that can't be put in the header's 'flush' callback */ + + /* Check for pending array deletion */ + if(ea->hdr->pending_delete) { + /* Set local info, so array deletion can occur after decrementing the + * header's ref count + */ + pending_delete = TRUE; + ea_addr = ea->hdr->addr; + } /* end if */ } /* end if */ - } /* end if */ - /* Check for pending array deletion */ - if(pending_delete) { - H5EA_hdr_t *hdr; /* Another pointer to extensible array header */ + /* Check for pending array deletion */ + if(pending_delete) { + H5EA_hdr_t *hdr; /* Another pointer to extensible array header */ #ifndef NDEBUG { @@ -854,33 +894,34 @@ H5EA_close(H5EA_t *ea, hid_t dxpl_id)) } #endif /* NDEBUG */ - /* Lock the array header into memory */ - /* (OK to pass in NULL for callback context, since we know the header must be in the cache) */ - if(NULL == (hdr = H5EA__hdr_protect(ea->f, dxpl_id, ea_addr, NULL, H5AC__NO_FLAGS_SET))) - H5E_THROW(H5E_CANTLOAD, "unable to load extensible array header") + /* Lock the array header into memory */ + /* (OK to pass in NULL for callback context, since we know the header must be in the cache) */ + if(NULL == (hdr = H5EA__hdr_protect(ea->f, dxpl_id, ea_addr, NULL, H5AC__NO_FLAGS_SET))) + H5E_THROW(H5E_CANTLOAD, "unable to load extensible array header") - /* Set the shared array header's file context for this operation */ - hdr->f = ea->f; + /* Set the shared array header's file context for this operation */ + hdr->f = ea->f; - /* Decrement the reference count on the array header */ - /* (don't put in H5EA_hdr_fuse_decr() as the array header may be evicted - * immediately -QAK) - */ - if(H5EA__hdr_decr(ea->hdr) < 0) - H5E_THROW(H5E_CANTDEC, "can't decrement reference count on shared array header") + /* Decrement the reference count on the array header */ + /* (don't put in H5EA_hdr_fuse_decr() as the array header may be evicted + * immediately -QAK) + */ + if(H5EA__hdr_decr(ea->hdr) < 0) + H5E_THROW(H5E_CANTDEC, "can't decrement reference count on shared array header") - /* Delete array, starting with header (unprotects header) */ - if(H5EA__hdr_delete(hdr, dxpl_id) < 0) - H5E_THROW(H5E_CANTDELETE, "unable to delete extensible array") + /* Delete array, starting with header (unprotects header) */ + if(H5EA__hdr_delete(hdr, dxpl_id) < 0) + H5E_THROW(H5E_CANTDELETE, "unable to delete extensible array") + } /* end if */ + else { + /* Decrement the reference count on the array header */ + /* (don't put in H5EA_hdr_fuse_decr() as the array header may be evicted + * immediately -QAK) + */ + if(H5EA__hdr_decr(ea->hdr) < 0) + H5E_THROW(H5E_CANTDEC, "can't decrement reference count on shared array header") + } /* end else */ } /* end if */ - else { - /* Decrement the reference count on the array header */ - /* (don't put in H5EA_hdr_fuse_decr() as the array header may be evicted - * immediately -QAK) - */ - if(H5EA__hdr_decr(ea->hdr) < 0) - H5E_THROW(H5E_CANTDEC, "can't decrement reference count on shared array header") - } /* end else */ /* Release the extensible array wrapper */ ea = (H5EA_t *)H5FL_FREE(H5EA_t, ea); diff --git a/src/H5EAcache.c b/src/H5EAcache.c index 9224916..1119e39 100644 --- a/src/H5EAcache.c +++ b/src/H5EAcache.c @@ -73,55 +73,53 @@ /********************/ /* Metadata cache (H5AC) callbacks */ -static herr_t H5EA__cache_hdr_get_load_size(const void *udata, size_t *image_len); +static herr_t H5EA__cache_hdr_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5EA__cache_hdr_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5EA__cache_hdr_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5EA__cache_hdr_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5EA__cache_hdr_image_len(const void *thing, size_t *image_len); static herr_t H5EA__cache_hdr_serialize(const H5F_t *f, void *image, size_t len, void *thing); +static herr_t H5EA__cache_hdr_notify(H5AC_notify_action_t action, void *thing); static herr_t H5EA__cache_hdr_free_icr(void *thing); -static herr_t H5EA__cache_iblock_get_load_size(const void *udata, size_t *image_len); +static herr_t H5EA__cache_iblock_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5EA__cache_iblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5EA__cache_iblock_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5EA__cache_iblock_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5EA__cache_iblock_image_len(const void *thing, size_t *image_len); static herr_t H5EA__cache_iblock_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5EA__cache_iblock_notify(H5AC_notify_action_t action, void *thing); static herr_t H5EA__cache_iblock_free_icr(void *thing); -static herr_t H5EA__cache_sblock_get_load_size(const void *udata, size_t *image_len); +static herr_t H5EA__cache_sblock_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5EA__cache_sblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5EA__cache_sblock_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5EA__cache_sblock_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5EA__cache_sblock_image_len(const void *thing, size_t *image_len); static herr_t H5EA__cache_sblock_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5EA__cache_sblock_notify(H5AC_notify_action_t action, void *thing); static herr_t H5EA__cache_sblock_free_icr(void *thing); -static herr_t H5EA__cache_dblock_get_load_size(const void *udata, size_t *image_len); +static herr_t H5EA__cache_dblock_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5EA__cache_dblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5EA__cache_dblock_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5EA__cache_dblock_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5EA__cache_dblock_image_len(const void *thing, size_t *image_len); static herr_t H5EA__cache_dblock_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5EA__cache_dblock_notify(H5AC_notify_action_t action, void *thing); static herr_t H5EA__cache_dblock_free_icr(void *thing); static herr_t H5EA__cache_dblock_fsf_size(const void *thing, size_t *fsf_size); -static herr_t H5EA__cache_dblk_page_get_load_size(const void *udata, size_t *image_len); +static herr_t H5EA__cache_dblk_page_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5EA__cache_dblk_page_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5EA__cache_dblk_page_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); static herr_t H5EA__cache_dblk_page_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); + size_t *image_len); static herr_t H5EA__cache_dblk_page_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5EA__cache_dblk_page_notify(H5AC_notify_action_t action, void *thing); @@ -138,14 +136,15 @@ const H5AC_class_t H5AC_EARRAY_HDR[1] = {{ "Extensible Array Header", /* Metadata client name (for debugging) */ H5FD_MEM_EARRAY_HDR, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5EA__cache_hdr_get_load_size, /* 'get_load_size' callback */ + H5EA__cache_hdr_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5EA__cache_hdr_verify_chksum, /* 'verify_chksum' callback */ H5EA__cache_hdr_deserialize, /* 'deserialize' callback */ H5EA__cache_hdr_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5EA__cache_hdr_serialize, /* 'serialize' callback */ - NULL, /* 'notify' callback */ + H5EA__cache_hdr_notify, /* 'notify' callback */ H5EA__cache_hdr_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -155,14 +154,15 @@ const H5AC_class_t H5AC_EARRAY_IBLOCK[1] = {{ "Extensible Array Index Block", /* Metadata client name (for debugging) */ H5FD_MEM_EARRAY_IBLOCK, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5EA__cache_iblock_get_load_size, /* 'get_load_size' callback */ + H5EA__cache_iblock_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5EA__cache_iblock_verify_chksum, /* 'verify_chksum' callback */ H5EA__cache_iblock_deserialize, /* 'deserialize' callback */ H5EA__cache_iblock_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5EA__cache_iblock_serialize, /* 'serialize' callback */ H5EA__cache_iblock_notify, /* 'notify' callback */ H5EA__cache_iblock_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -172,14 +172,15 @@ const H5AC_class_t H5AC_EARRAY_SBLOCK[1] = {{ "Extensible Array Super Block", /* Metadata client name (for debugging) */ H5FD_MEM_EARRAY_SBLOCK, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5EA__cache_sblock_get_load_size, /* 'get_load_size' callback */ + H5EA__cache_sblock_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5EA__cache_sblock_verify_chksum, /* 'verify_chksum' callback */ H5EA__cache_sblock_deserialize, /* 'deserialize' callback */ H5EA__cache_sblock_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5EA__cache_sblock_serialize, /* 'serialize' callback */ H5EA__cache_sblock_notify, /* 'notify' callback */ H5EA__cache_sblock_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -189,14 +190,15 @@ const H5AC_class_t H5AC_EARRAY_DBLOCK[1] = {{ "Extensible Array Data Block", /* Metadata client name (for debugging) */ H5FD_MEM_EARRAY_DBLOCK, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5EA__cache_dblock_get_load_size, /* 'get_load_size' callback */ + H5EA__cache_dblock_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5EA__cache_dblock_verify_chksum, /* 'verify_chksum' callback */ H5EA__cache_dblock_deserialize, /* 'deserialize' callback */ H5EA__cache_dblock_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5EA__cache_dblock_serialize, /* 'serialize' callback */ H5EA__cache_dblock_notify, /* 'notify' callback */ H5EA__cache_dblock_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ H5EA__cache_dblock_fsf_size, /* 'fsf_size' callback */ }}; @@ -206,14 +208,15 @@ const H5AC_class_t H5AC_EARRAY_DBLK_PAGE[1] = {{ "Extensible Array Data Block Page", /* Metadata client name (for debugging) */ H5FD_MEM_EARRAY_DBLK_PAGE, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5EA__cache_dblk_page_get_load_size, /* 'get_load_size' callback */ + H5EA__cache_dblk_page_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5EA__cache_dblk_page_verify_chksum, /* 'verify_chksum' callback */ H5EA__cache_dblk_page_deserialize, /* 'deserialize' callback */ H5EA__cache_dblk_page_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5EA__cache_dblk_page_serialize, /* 'serialize' callback */ H5EA__cache_dblk_page_notify, /* 'notify' callback */ H5EA__cache_dblk_page_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -230,7 +233,7 @@ const H5AC_class_t H5AC_EARRAY_DBLK_PAGE[1] = {{ /*------------------------------------------------------------------------- - * Function: H5EA__cache_hdr_get_load_size + * Function: H5EA__cache_hdr_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -244,21 +247,54 @@ const H5AC_class_t H5AC_EARRAY_DBLK_PAGE[1] = {{ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_hdr_get_load_size(const void *_udata, size_t *image_len)) +H5EA__cache_hdr_get_initial_load_size(void *_udata, size_t *image_len)) /* Local variables */ - const H5EA_hdr_cache_ud_t *udata = (const H5EA_hdr_cache_ud_t *)_udata; /* User data for callback */ + H5EA_hdr_cache_ud_t *udata = (H5EA_hdr_cache_ud_t *)_udata; /* User data for callback */ /* Check arguments */ HDassert(udata); HDassert(udata->f); - HDassert(H5F_addr_defined(udata->addr)); HDassert(image_len); /* Set the image length size */ *image_len = (size_t)H5EA_HEADER_SIZE_FILE(udata->f); -END_FUNC(STATIC) /* end H5EA__cache_hdr_get_load_size() */ +END_FUNC(STATIC) /* end H5EA__cache_hdr_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5EA__cache_hdr_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, NOERR, +htri_t, TRUE, -, +H5EA__cache_hdr_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)) + + /* Local variables */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + +END_FUNC(STATIC) /* end H5EA__cache_hdr_verify_chksum() */ /*------------------------------------------------------------------------- @@ -286,7 +322,6 @@ H5EA__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, H5EA_hdr_cache_ud_t *udata = (H5EA_hdr_cache_ud_t *)_udata; const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ /* Check arguments */ HDassert(image); @@ -361,17 +396,11 @@ H5EA__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, /* (allow for checksum not decoded yet) */ HDassert((size_t)(image - (const uint8_t *)_image) == (len - H5EA_SIZEOF_CHKSUM)); - /* Compute checksum on entire header */ - /* (including the filter information, if present) */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array header") - /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == len); @@ -408,8 +437,7 @@ END_FUNC(STATIC) /* end H5EA__cache_hdr_deserialize() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_hdr_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr)) +H5EA__cache_hdr_image_len(const void *_thing, size_t *image_len)) /* Local variables */ const H5EA_hdr_t *hdr = (const H5EA_hdr_t *)_thing; /* Pointer to the object */ @@ -494,6 +522,80 @@ END_FUNC(STATIC) /* end H5EA__cache_hdr_serialize() */ /*------------------------------------------------------------------------- + * Function: H5EA__cache_hdr_notify + * + * Purpose: Handle cache action notifications + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 11/30/15 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, ERR, +herr_t, SUCCEED, FAIL, +H5EA__cache_hdr_notify(H5AC_notify_action_t action, void *_thing)) + + /* Local variables */ + H5EA_hdr_t *hdr = (H5EA_hdr_t *)_thing; /* Pointer to the object */ + + /* Sanity check */ + HDassert(hdr); + + /* Check if the file was opened with SWMR-write access */ + if(hdr->swmr_write) { + /* Determine which action to take */ + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + case H5AC_NOTIFY_ACTION_AFTER_LOAD: + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + /* If hdr->parent != NULL, hdr->parent is used to destroy + * the flush dependency before the header is evicted. + */ + if(hdr->parent) { + /* Sanity check */ + HDassert(hdr->top_proxy); + + /* Destroy flush dependency on object header proxy */ + if(H5AC_proxy_entry_remove_child((H5AC_proxy_entry_t *)hdr->parent, (void *)hdr->top_proxy) < 0) + H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between extensible array and proxy") + hdr->parent = NULL; + } /* end if */ + + /* Detach from 'top' proxy for extensible array */ + if(hdr->top_proxy) { + if(H5AC_proxy_entry_remove_child(hdr->top_proxy, hdr) < 0) + H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between header and extensible array 'top' proxy") + /* Don't reset hdr->top_proxy here, it's destroyed when the header is freed -QAK */ + } /* end if */ + break; + + default: +#ifdef NDEBUG + H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Unknown action?!?"); +#endif /* NDEBUG */ + } /* end switch */ + } /* end if */ + else + HDassert(NULL == hdr->parent); + +CATCH + +END_FUNC(STATIC) /* end H5EA__cache_hdr_notify() */ + + +/*------------------------------------------------------------------------- * Function: H5EA__cache_hdr_free_icr * * Purpose: Destroy/release an "in core representation" of a data @@ -524,7 +626,7 @@ END_FUNC(STATIC) /* end H5EA__cache_hdr_free_icr() */ /*------------------------------------------------------------------------- - * Function: H5EA__cache_iblock_get_load_size + * Function: H5EA__cache_iblock_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -538,11 +640,11 @@ END_FUNC(STATIC) /* end H5EA__cache_hdr_free_icr() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_iblock_get_load_size(const void *_udata, size_t *image_len)) +H5EA__cache_iblock_get_initial_load_size(void *_udata, size_t *image_len)) /* Local variables */ - const H5EA_hdr_t *hdr = (const H5EA_hdr_t *)_udata; /* User data for callback */ - H5EA_iblock_t iblock; /* Fake index block for computing size */ + H5EA_hdr_t *hdr = (H5EA_hdr_t *)_udata; /* User data for callback */ + H5EA_iblock_t iblock; /* Fake index block for computing size */ /* Check arguments */ HDassert(hdr); @@ -558,7 +660,41 @@ H5EA__cache_iblock_get_load_size(const void *_udata, size_t *image_len)) /* Set the image length size */ *image_len = (size_t)H5EA_IBLOCK_SIZE(&iblock); -END_FUNC(STATIC) /* end H5EA__cache_iblock_get_load_size() */ +END_FUNC(STATIC) /* end H5EA__cache_iblock_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5EA__cache_iblock_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, NOERR, +htri_t, TRUE, -, +H5EA__cache_iblock_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)) + + /* Local variables */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + +END_FUNC(STATIC) /* end H5EA__cache_iblock_verify_chksum() */ /*------------------------------------------------------------------------- @@ -585,7 +721,6 @@ H5EA__cache_iblock_deserialize(const void *_image, size_t len, H5EA_hdr_t *hdr = (H5EA_hdr_t *)_udata; /* User data for callback */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ haddr_t arr_addr; /* Address of array header in the file */ size_t u; /* Local index variable */ @@ -649,8 +784,7 @@ H5EA__cache_iblock_deserialize(const void *_image, size_t len, /* Save the index block's size */ iblock->size = len; - /* Compute checksum on index block */ - computed_chksum = H5_checksum_metadata((const uint8_t *)_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -658,10 +792,6 @@ H5EA__cache_iblock_deserialize(const void *_image, size_t len, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == iblock->size); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array index block") - /* Set return value */ ret_value = iblock; @@ -690,8 +820,7 @@ END_FUNC(STATIC) /* end H5EA__cache_iblock_deserialize() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_iblock_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr)) +H5EA__cache_iblock_image_len(const void *_thing, size_t *image_len)) /* Local variables */ const H5EA_iblock_t *iblock = (const H5EA_iblock_t *)_thing; /* Pointer to the object */ @@ -824,18 +953,33 @@ H5EA__cache_iblock_notify(H5AC_notify_action_t action, void *_thing)) H5E_THROW(H5E_CANTDEPEND, "unable to create flush dependency between index block and header, address = %llu", (unsigned long long)iblock->addr) break; - case H5AC_NOTIFY_ACTION_AFTER_FLUSH: - /* do nothing */ - break; + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + /* do nothing */ + break; case H5AC_NOTIFY_ACTION_BEFORE_EVICT: /* Destroy flush dependency on extensible array header */ if(H5EA__destroy_flush_depend((H5AC_info_t *)iblock->hdr, (H5AC_info_t *)iblock) < 0) H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between index block and header, address = %llu", (unsigned long long)iblock->addr) + + /* Detach from 'top' proxy for extensible array */ + if(iblock->top_proxy) { + if(H5AC_proxy_entry_remove_child(iblock->top_proxy, iblock) < 0) + H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between index block and extensible array 'top' proxy") + iblock->top_proxy = NULL; + } /* end if */ break; default: +#ifdef NDEBUG H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Unknown action?!?"); +#endif /* NDEBUG */ } /* end switch */ CATCH @@ -874,7 +1018,7 @@ END_FUNC(STATIC) /* end H5EA__cache_iblock_free_icr() */ /*------------------------------------------------------------------------- - * Function: H5EA__cache_sblock_get_load_size + * Function: H5EA__cache_sblock_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -888,16 +1032,15 @@ END_FUNC(STATIC) /* end H5EA__cache_iblock_free_icr() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_sblock_get_load_size(const void *_udata, size_t *image_len)) +H5EA__cache_sblock_get_initial_load_size(void *_udata, size_t *image_len)) /* Local variables */ - const H5EA_sblock_cache_ud_t *udata = (const H5EA_sblock_cache_ud_t *)_udata; /* User data */ - H5EA_sblock_t sblock; /* Fake super block for computing size */ + H5EA_sblock_cache_ud_t *udata = (H5EA_sblock_cache_ud_t *)_udata; /* User data */ + H5EA_sblock_t sblock; /* Fake super block for computing size */ /* Check arguments */ HDassert(udata); HDassert(udata->hdr); - HDassert(udata->parent); HDassert(udata->sblk_idx > 0); HDassert(H5F_addr_defined(udata->sblk_addr)); HDassert(image_len); @@ -928,7 +1071,41 @@ H5EA__cache_sblock_get_load_size(const void *_udata, size_t *image_len)) /* Set the image length size */ *image_len = (size_t)H5EA_SBLOCK_SIZE(&sblock); -END_FUNC(STATIC) /* end H5EA__cache_sblock_get_load_size() */ +END_FUNC(STATIC) /* end H5EA__cache_sblock_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5EA__cache_sblock_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, NOERR, +htri_t, TRUE, -, +H5EA__cache_sblock_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)) + + /* Local variables */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + +END_FUNC(STATIC) /* end H5EA__cache_sblock_verify_chksum() */ /*------------------------------------------------------------------------- @@ -955,7 +1132,6 @@ H5EA__cache_sblock_deserialize(const void *_image, size_t len, H5EA_sblock_cache_ud_t *udata = (H5EA_sblock_cache_ud_t *)_udata; /* User data */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ haddr_t arr_addr; /* Address of array header in the file */ size_t u; /* Local index variable */ @@ -1016,8 +1192,7 @@ H5EA__cache_sblock_deserialize(const void *_image, size_t len, /* Save the super block's size */ sblock->size = len; - /* Compute checksum on super block */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -1025,10 +1200,6 @@ H5EA__cache_sblock_deserialize(const void *_image, size_t len, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == sblock->size); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array super block") - /* Set return value */ ret_value = sblock; @@ -1057,8 +1228,7 @@ END_FUNC(STATIC) /* end H5EA__cache_sblock_deserialize() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_sblock_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr)) +H5EA__cache_sblock_image_len(const void *_thing, size_t *image_len)) /* Local variables */ const H5EA_sblock_t *sblock = (const H5EA_sblock_t *)_thing; /* Pointer to the object */ @@ -1178,14 +1348,14 @@ H5EA__cache_sblock_notify(H5AC_notify_action_t action, void *_thing)) H5E_THROW(H5E_CANTDEPEND, "unable to create flush dependency between super block and index block, address = %llu", (unsigned long long)sblock->addr) break; - case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: /* Destroy flush dependency on extensible array header, if set */ if(sblock->has_hdr_depend) { if(H5EA__destroy_flush_depend((H5AC_info_t *)sblock->hdr, (H5AC_info_t *)sblock) < 0) H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between super block and header, address = %llu", (unsigned long long)sblock->addr) sblock->has_hdr_depend = FALSE; } /* end if */ - break; + break; case H5AC_NOTIFY_ACTION_BEFORE_EVICT: /* Destroy flush dependency on index block */ @@ -1198,10 +1368,28 @@ H5EA__cache_sblock_notify(H5AC_notify_action_t action, void *_thing)) H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between super block and header, address = %llu", (unsigned long long)sblock->addr) sblock->has_hdr_depend = FALSE; } /* end if */ + + /* Detach from 'top' proxy for extensible array */ + if(sblock->top_proxy) { + if(H5AC_proxy_entry_remove_child(sblock->top_proxy, sblock) < 0) + H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between super block and extensible array 'top' proxy") + sblock->top_proxy = NULL; + } /* end if */ + break; + + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + /* do nothing */ break; default: +#ifdef NDEBUG H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Unknown action?!?"); +#endif /* NDEBUG */ } /* end switch */ CATCH @@ -1240,7 +1428,7 @@ END_FUNC(STATIC) /* end H5EA__cache_sblock_free_icr() */ /*------------------------------------------------------------------------- - * Function: H5EA__cache_dblock_get_load_size + * Function: H5EA__cache_dblock_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -1254,16 +1442,15 @@ END_FUNC(STATIC) /* end H5EA__cache_sblock_free_icr() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_dblock_get_load_size(const void *_udata, size_t *image_len)) +H5EA__cache_dblock_get_initial_load_size(void *_udata, size_t *image_len)) /* Local variables */ - const H5EA_dblock_cache_ud_t *udata = (const H5EA_dblock_cache_ud_t *)_udata; /* User data */ - H5EA_dblock_t dblock; /* Fake data block for computing size */ + H5EA_dblock_cache_ud_t *udata = (H5EA_dblock_cache_ud_t *)_udata; /* User data */ + H5EA_dblock_t dblock; /* Fake data block for computing size */ /* Check arguments */ HDassert(udata); HDassert(udata->hdr); - HDassert(udata->parent); HDassert(udata->nelmts > 0); HDassert(image_len); @@ -1295,7 +1482,41 @@ H5EA__cache_dblock_get_load_size(const void *_udata, size_t *image_len)) else *image_len = H5EA_DBLOCK_PREFIX_SIZE(&dblock); -END_FUNC(STATIC) /* end H5EA__cache_dblock_get_load_size() */ +END_FUNC(STATIC) /* end H5EA__cache_dblock_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5EA__cache_dblock_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, NOERR, +htri_t, TRUE, -, +H5EA__cache_dblock_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)) + + /* Local variables */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + +END_FUNC(STATIC) /* end H5EA__cache_sblock_verify_chksum() */ /*------------------------------------------------------------------------- @@ -1322,7 +1543,6 @@ H5EA__cache_dblock_deserialize(const void *_image, size_t len, H5EA_dblock_cache_ud_t *udata = (H5EA_dblock_cache_ud_t *)_udata; /* User data */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ haddr_t arr_addr; /* Address of array header in the file */ /* Check arguments */ @@ -1382,8 +1602,7 @@ H5EA__cache_dblock_deserialize(const void *_image, size_t len, /* (Note: This is not the same as the image length, for paged data blocks) */ dblock->size = H5EA_DBLOCK_SIZE(dblock); - /* Compute checksum on data block */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -1391,10 +1610,6 @@ H5EA__cache_dblock_deserialize(const void *_image, size_t len, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == len); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array data block") - /* Set return value */ ret_value = dblock; @@ -1423,8 +1638,7 @@ END_FUNC(STATIC) /* end H5EA__cache_dblock_deserialize() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_dblock_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr)) +H5EA__cache_dblock_image_len(const void *_thing, size_t *image_len)) /* Local variables */ const H5EA_dblock_t *dblock = (const H5EA_dblock_t *)_thing; /* Pointer to the object */ @@ -1545,14 +1759,14 @@ H5EA__cache_dblock_notify(H5AC_notify_action_t action, void *_thing)) H5E_THROW(H5E_CANTDEPEND, "unable to create flush dependency between data block and parent, address = %llu", (unsigned long long)dblock->addr) break; - case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: /* Destroy flush dependency on extensible array header, if set */ if(dblock->has_hdr_depend) { if(H5EA__destroy_flush_depend((H5AC_info_t *)dblock->hdr, (H5AC_info_t *)dblock) < 0) H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between direct block and header, address = %llu", (unsigned long long)dblock->addr) dblock->has_hdr_depend = FALSE; } /* end if */ - break; + break; case H5AC_NOTIFY_ACTION_BEFORE_EVICT: /* Destroy flush dependency on parent */ @@ -1562,13 +1776,31 @@ H5EA__cache_dblock_notify(H5AC_notify_action_t action, void *_thing)) /* Destroy flush dependency on extensible array header, if set */ if(dblock->has_hdr_depend) { if(H5EA__destroy_flush_depend((H5AC_info_t *)dblock->hdr, (H5AC_info_t *)dblock) < 0) - H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between direct block and header, address = %llu", (unsigned long long)dblock->addr) + H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between data block and header, address = %llu", (unsigned long long)dblock->addr) dblock->has_hdr_depend = FALSE; } /* end if */ + + /* Detach from 'top' proxy for extensible array */ + if(dblock->top_proxy) { + if(H5AC_proxy_entry_remove_child(dblock->top_proxy, dblock) < 0) + H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between data block and extensible array 'top' proxy") + dblock->top_proxy = NULL; + } /* end if */ + break; + + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + /* do nothing */ break; default: +#ifdef NDEBUG H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Unknown action?!?"); +#endif /* NDEBUG */ } /* end switch */ CATCH @@ -1614,7 +1846,7 @@ END_FUNC(STATIC) /* end H5EA__cache_dblock_free_icr() */ * file space block set. * * This function is needed when the data block is paged, as - * the datablock header and all its pages are allocted as a + * the datablock header and all its pages are allocated as a * single contiguous chunk of file space, and must be * deallocated the same way. * @@ -1654,7 +1886,7 @@ END_FUNC(STATIC) /* end H5EA__cache_dblock_fsf_size() */ /*------------------------------------------------------------------------- - * Function: H5EA__cache_dblk_page_get_load_size + * Function: H5EA__cache_dblk_page_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -1668,20 +1900,54 @@ END_FUNC(STATIC) /* end H5EA__cache_dblock_fsf_size() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_dblk_page_get_load_size(const void *_udata, size_t *image_len)) +H5EA__cache_dblk_page_get_initial_load_size(void *_udata, size_t *image_len)) /* Local variables */ - const H5EA_dblk_page_cache_ud_t *udata = (const H5EA_dblk_page_cache_ud_t *)_udata; /* User data */ + H5EA_dblk_page_cache_ud_t *udata = (H5EA_dblk_page_cache_ud_t *)_udata; /* User data */ /* Check arguments */ HDassert(udata); HDassert(udata->hdr); - HDassert(udata->parent); HDassert(image_len); + /* Set the image length size */ *image_len = (size_t)H5EA_DBLK_PAGE_SIZE(udata->hdr); -END_FUNC(STATIC) /* end H5EA__cache_dblk_page_get_load_size() */ +END_FUNC(STATIC) /* end H5EA__cache_dblk_page_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5EA__cache_dblk_page_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, NOERR, +htri_t, TRUE, -, +H5EA__cache_dblk_page_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)) + + /* Local variables */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + +END_FUNC(STATIC) /* end H5EA__cache_dblk_page_verify_chksum() */ /*------------------------------------------------------------------------- @@ -1708,7 +1974,6 @@ H5EA__cache_dblk_page_deserialize(const void *_image, size_t len, H5EA_dblk_page_cache_ud_t *udata = (H5EA_dblk_page_cache_ud_t *)_udata; /* User data for loading data block page */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ /* Sanity check */ HDassert(udata); @@ -1738,8 +2003,7 @@ H5EA__cache_dblk_page_deserialize(const void *_image, size_t len, /* Set the data block page's size */ dblk_page->size = len; - /* Compute checksum on data block page */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -1747,10 +2011,6 @@ H5EA__cache_dblk_page_deserialize(const void *_image, size_t len, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == dblk_page->size); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array data block page") - /* Set return value */ ret_value = dblk_page; @@ -1779,8 +2039,7 @@ END_FUNC(STATIC) /* end H5EA__cache_dblk_page_deserialize() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_dblk_page_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr)) +H5EA__cache_dblk_page_image_len(const void *_thing, size_t *image_len)) /* Local variables */ const H5EA_dblk_page_t *dblk_page = (const H5EA_dblk_page_t *)_thing; /* Pointer to the object */ @@ -1879,14 +2138,14 @@ H5EA__cache_dblk_page_notify(H5AC_notify_action_t action, void *_thing)) H5E_THROW(H5E_CANTDEPEND, "unable to create flush dependency between data block page and parent, address = %llu", (unsigned long long)dblk_page->addr) break; - case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: /* Destroy flush dependency on extensible array header, if set */ if(dblk_page->has_hdr_depend) { if(H5EA__destroy_flush_depend((H5AC_info_t *)dblk_page->hdr, (H5AC_info_t *)dblk_page) < 0) H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between data block page and header, address = %llu", (unsigned long long)dblk_page->addr) dblk_page->has_hdr_depend = FALSE; } /* end if */ - break; + break; case H5AC_NOTIFY_ACTION_BEFORE_EVICT: /* Destroy flush dependency on parent */ @@ -1899,10 +2158,28 @@ H5EA__cache_dblk_page_notify(H5AC_notify_action_t action, void *_thing)) H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between data block page and header, address = %llu", (unsigned long long)dblk_page->addr) dblk_page->has_hdr_depend = FALSE; } /* end if */ + + /* Detach from 'top' proxy for extensible array */ + if(dblk_page->top_proxy) { + if(H5AC_proxy_entry_remove_child(dblk_page->top_proxy, dblk_page) < 0) + H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between data block page and extensible array 'top' proxy") + dblk_page->top_proxy = NULL; + } /* end if */ + break; + + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + /* do nothing */ break; default: +#ifdef NDEBUG H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Unknown action?!?"); +#endif /* NDEBUG */ } /* end switch */ CATCH diff --git a/src/H5EAdblkpage.c b/src/H5EAdblkpage.c index 327feb0..927dcb6 100644 --- a/src/H5EAdblkpage.c +++ b/src/H5EAdblkpage.c @@ -154,7 +154,8 @@ H5EA__dblk_page_create(H5EA_hdr_t *hdr, hid_t dxpl_id, H5EA_sblock_t *parent, haddr_t addr)) /* Local variables */ - H5EA_dblk_page_t *dblk_page = NULL; /* Extensible array data block page */ + H5EA_dblk_page_t *dblk_page = NULL; /* Extensible array data block page */ + hbool_t inserted = FALSE; /* Whether the header was inserted into cache */ /* Sanity check */ HDassert(hdr); @@ -174,10 +175,23 @@ H5EA__dblk_page_create(H5EA_hdr_t *hdr, hid_t dxpl_id, H5EA_sblock_t *parent, /* Cache the new extensible array data block page */ if(H5AC_insert_entry(hdr->f, dxpl_id, H5AC_EARRAY_DBLK_PAGE, dblk_page->addr, dblk_page, H5AC__NO_FLAGS_SET) < 0) H5E_THROW(H5E_CANTINSERT, "can't add extensible array data block page to cache") + inserted = TRUE; + + /* Add data block page as child of 'top' proxy */ + if(hdr->top_proxy) { + if(H5AC_proxy_entry_add_child(hdr->top_proxy, hdr->f, dxpl_id, dblk_page) < 0) + H5E_THROW(H5E_CANTSET, "unable to add extensible array entry as child of array proxy") + dblk_page->top_proxy = hdr->top_proxy; + } /* end if */ CATCH if(ret_value < 0) if(dblk_page) { + /* Remove from cache, if inserted */ + if(inserted) + if(H5AC_remove_entry(dblk_page) < 0) + H5E_THROW(H5E_CANTREMOVE, "unable to remove extensible array data block page from cache") + /* Destroy data block page */ if(H5EA__dblk_page_dest(dblk_page) < 0) H5E_THROW(H5E_CANTFREE, "unable to destroy extensible array data block page") @@ -206,7 +220,8 @@ H5EA__dblk_page_protect(H5EA_hdr_t *hdr, hid_t dxpl_id, H5EA_sblock_t *parent, haddr_t dblk_page_addr, unsigned flags)) /* Local variables */ - H5EA_dblk_page_cache_ud_t udata; /* Information needed for loading data block page */ + H5EA_dblk_page_t *dblk_page = NULL; /* Extensible array data block page */ + H5EA_dblk_page_cache_ud_t udata; /* Information needed for loading data block page */ /* Sanity check */ HDassert(hdr); @@ -221,10 +236,27 @@ H5EA__dblk_page_protect(H5EA_hdr_t *hdr, hid_t dxpl_id, H5EA_sblock_t *parent, udata.dblk_page_addr = dblk_page_addr; /* Protect the data block page */ - if(NULL == (ret_value = (H5EA_dblk_page_t *)H5AC_protect(hdr->f, dxpl_id, H5AC_EARRAY_DBLK_PAGE, dblk_page_addr, &udata, flags))) + if(NULL == (dblk_page = (H5EA_dblk_page_t *)H5AC_protect(hdr->f, dxpl_id, H5AC_EARRAY_DBLK_PAGE, dblk_page_addr, &udata, flags))) H5E_THROW(H5E_CANTPROTECT, "unable to protect extensible array data block page, address = %llu", (unsigned long long)dblk_page_addr) + /* Create top proxy, if it doesn't exist */ + if(hdr->top_proxy && NULL == dblk_page->top_proxy) { + /* Add data block page as child of 'top' proxy */ + if(H5AC_proxy_entry_add_child(hdr->top_proxy, hdr->f, dxpl_id, dblk_page) < 0) + H5E_THROW(H5E_CANTSET, "unable to add extensible array entry as child of array proxy") + dblk_page->top_proxy = hdr->top_proxy; + } /* end if */ + + /* Set return value */ + ret_value = dblk_page; + CATCH + /* Clean up on error */ + if(!ret_value) { + /* Release the data block page, if it was protected */ + if(dblk_page && H5AC_unprotect(hdr->f, dxpl_id, H5AC_EARRAY_DBLK_PAGE, dblk_page->addr, dblk_page, H5AC__NO_FLAGS_SET) < 0) + H5E_THROW(H5E_CANTUNPROTECT, "unable to unprotect extensible array data block page, address = %llu", (unsigned long long)dblk_page->addr) + } /* end if */ END_FUNC(PKG) /* end H5EA__dblk_page_protect() */ @@ -299,6 +331,9 @@ H5EA__dblk_page_dest(H5EA_dblk_page_t *dblk_page)) dblk_page->hdr = NULL; } /* end if */ + /* Sanity check */ + HDassert(NULL == dblk_page->top_proxy); + /* Free the data block page itself */ dblk_page = H5FL_FREE(H5EA_dblk_page_t, dblk_page); diff --git a/src/H5EAdblock.c b/src/H5EAdblock.c index 9511a9d..c288c69 100644 --- a/src/H5EAdblock.c +++ b/src/H5EAdblock.c @@ -168,6 +168,7 @@ H5EA__dblock_create(H5EA_hdr_t *hdr, hid_t dxpl_id, void *parent, /* Local variables */ H5EA_dblock_t *dblock = NULL; /* Extensible array data block */ haddr_t dblock_addr; /* Extensible array data block address */ + hbool_t inserted = FALSE; /* Whether the header was inserted into cache */ /* Sanity check */ HDassert(hdr); @@ -198,6 +199,14 @@ H5EA__dblock_create(H5EA_hdr_t *hdr, hid_t dxpl_id, void *parent, /* Cache the new extensible array data block */ if(H5AC_insert_entry(hdr->f, dxpl_id, H5AC_EARRAY_DBLOCK, dblock_addr, dblock, H5AC__NO_FLAGS_SET) < 0) H5E_THROW(H5E_CANTINSERT, "can't add extensible array data block to cache") + inserted = TRUE; + + /* Add data block as child of 'top' proxy */ + if(hdr->top_proxy) { + if(H5AC_proxy_entry_add_child(hdr->top_proxy, hdr->f, dxpl_id, dblock) < 0) + H5E_THROW(H5E_CANTSET, "unable to add extensible array entry as child of array proxy") + dblock->top_proxy = hdr->top_proxy; + } /* end if */ /* Update extensible array data block statistics */ hdr->stats.stored.ndata_blks++; @@ -215,6 +224,11 @@ H5EA__dblock_create(H5EA_hdr_t *hdr, hid_t dxpl_id, void *parent, CATCH if(!H5F_addr_defined(ret_value)) if(dblock) { + /* Remove from cache, if inserted */ + if(inserted) + if(H5AC_remove_entry(dblock) < 0) + H5E_THROW(H5E_CANTREMOVE, "unable to remove extensible array data block from cache") + /* Release data block's disk space */ if(H5F_addr_defined(dblock->addr) && H5MF_xfree(hdr->f, H5FD_MEM_EARRAY_DBLOCK, dxpl_id, dblock->addr, (hsize_t)dblock->size) < 0) H5E_THROW(H5E_CANTFREE, "unable to release extensible array data block") @@ -284,7 +298,8 @@ H5EA__dblock_protect(H5EA_hdr_t *hdr, hid_t dxpl_id, void *parent, haddr_t dblk_addr, size_t dblk_nelmts, unsigned flags)) /* Local variables */ - H5EA_dblock_cache_ud_t udata; /* Information needed for loading data block */ + H5EA_dblock_t *dblock; /* Extensible array data block */ + H5EA_dblock_cache_ud_t udata; /* Information needed for loading data block */ /* Sanity check */ HDassert(hdr); @@ -301,11 +316,29 @@ H5EA__dblock_protect(H5EA_hdr_t *hdr, hid_t dxpl_id, void *parent, udata.dblk_addr = dblk_addr; /* Protect the data block */ - if(NULL == (ret_value = (H5EA_dblock_t *)H5AC_protect(hdr->f, dxpl_id, H5AC_EARRAY_DBLOCK, dblk_addr, &udata, flags))) + if(NULL == (dblock = (H5EA_dblock_t *)H5AC_protect(hdr->f, dxpl_id, H5AC_EARRAY_DBLOCK, dblk_addr, &udata, flags))) H5E_THROW(H5E_CANTPROTECT, "unable to protect extensible array data block, address = %llu", (unsigned long long)dblk_addr) + /* Create top proxy, if it doesn't exist */ + if(hdr->top_proxy && NULL == dblock->top_proxy) { + /* Add data block as child of 'top' proxy */ + if(H5AC_proxy_entry_add_child(hdr->top_proxy, hdr->f, dxpl_id, dblock) < 0) + H5E_THROW(H5E_CANTSET, "unable to add extensible array entry as child of array proxy") + dblock->top_proxy = hdr->top_proxy; + } /* end if */ + + /* Set return value */ + ret_value = dblock; + CATCH + /* Clean up on error */ + if(!ret_value) { + /* Release the data block, if it was protected */ + if(dblock && H5AC_unprotect(hdr->f, dxpl_id, H5AC_EARRAY_DBLOCK, dblock->addr, dblock, H5AC__NO_FLAGS_SET) < 0) + H5E_THROW(H5E_CANTUNPROTECT, "unable to unprotect extensible array data block, address = %llu", (unsigned long long)dblock->addr) + } /* end if */ + END_FUNC(PKG) /* end H5EA__dblock_protect() */ @@ -442,6 +475,9 @@ H5EA__dblock_dest(H5EA_dblock_t *dblock)) dblock->hdr = NULL; } /* end if */ + /* Sanity check */ + HDassert(NULL == dblock->top_proxy); + /* Free the data block itself */ dblock = H5FL_FREE(H5EA_dblock_t, dblock); diff --git a/src/H5EAhdr.c b/src/H5EAhdr.c index da15087..76d8733 100644 --- a/src/H5EAhdr.c +++ b/src/H5EAhdr.c @@ -135,6 +135,7 @@ H5EA__hdr_alloc(H5F_t *f)) /* Set the internal parameters for the array */ hdr->f = f; + hdr->swmr_write = (H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) > 0; hdr->sizeof_addr = H5F_SIZEOF_ADDR(f); hdr->sizeof_size = H5F_SIZEOF_SIZE(f); @@ -204,9 +205,6 @@ H5EA__hdr_init(H5EA_hdr_t *hdr, void *ctx_udata)) hdr->nsblks = 1 + (hdr->cparam.max_nelmts_bits - H5VM_log2_of2(hdr->cparam.data_blk_min_elmts)); hdr->dblk_page_nelmts = (size_t)1 << hdr->cparam.max_dblk_page_nelmts_bits; hdr->arr_off_size = (unsigned char)H5EA_SIZEOF_OFFSET_BITS(hdr->cparam.max_nelmts_bits); -#ifdef QAK -HDfprintf(stderr, "%s: hdr->nsblks = %Zu\n", FUNC, hdr->nsblks); -#endif /* QAK */ /* Allocate information for each super block */ if(NULL == (hdr->sblk_info = H5FL_SEQ_MALLOC(H5EA_sblk_info_t, hdr->nsblks))) @@ -220,9 +218,6 @@ HDfprintf(stderr, "%s: hdr->nsblks = %Zu\n", FUNC, hdr->nsblks); hdr->sblk_info[u].dblk_nelmts = H5EA_SBLK_DBLK_NELMTS(u, hdr->cparam.data_blk_min_elmts); hdr->sblk_info[u].start_idx = start_idx; hdr->sblk_info[u].start_dblk = start_dblk; -#ifdef QAK -HDfprintf(stderr, "%s: hdr->sblk_info[%Zu] = {%Zu, %Zu, %Hu, %Hu}\n", FUNC, u, hdr->sblk_info[u].ndblks, hdr->sblk_info[u].dblk_nelmts, hdr->sblk_info[u].start_idx, hdr->sblk_info[u].start_dblk); -#endif /* QAK */ /* Advance starting indices for next super block */ start_idx += (hsize_t)hdr->sblk_info[u].ndblks * (hsize_t)hdr->sblk_info[u].dblk_nelmts; @@ -271,9 +266,6 @@ H5EA__hdr_alloc_elmts(H5EA_hdr_t *hdr, size_t nelmts)) /* Compute the index of the element buffer factory */ H5_CHECK_OVERFLOW(nelmts, /*From:*/size_t, /*To:*/uint32_t); idx = H5VM_log2_of2((uint32_t)nelmts) - H5VM_log2_of2((uint32_t)hdr->cparam.data_blk_min_elmts); -#ifdef QAK -HDfprintf(stderr, "%s: nelmts = %Zu, hdr->data_blk_min_elmts = %u, idx = %u\n", FUNC, nelmts, (unsigned)hdr->data_blk_min_elmts, idx); -#endif /* QAK */ /* Check for needing to increase size of array of factories */ if(idx >= hdr->elmt_fac.nalloc) { @@ -341,9 +333,6 @@ H5EA__hdr_free_elmts(H5EA_hdr_t *hdr, size_t nelmts, void *elmts)) /* Compute the index of the element buffer factory */ H5_CHECK_OVERFLOW(nelmts, /*From:*/size_t, /*To:*/uint32_t); idx = H5VM_log2_of2((uint32_t)nelmts) - H5VM_log2_of2((uint32_t)hdr->cparam.data_blk_min_elmts); -#ifdef QAK -HDfprintf(stderr, "%s: nelmts = %Zu, hdr->data_blk_min_elmts = %u, idx = %u\n", FUNC, nelmts, (unsigned)hdr->data_blk_min_elmts, idx); -#endif /* QAK */ /* Free buffer for elements in index block */ HDassert(idx < hdr->elmt_fac.nalloc); @@ -373,10 +362,7 @@ H5EA__hdr_create(H5F_t *f, hid_t dxpl_id, const H5EA_create_t *cparam, /* Local variables */ H5EA_hdr_t *hdr = NULL; /* Extensible array header */ - -#ifdef QAK -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* QAK */ + hbool_t inserted = FALSE; /* Whether the header was inserted into cache */ /* Check arguments */ HDassert(f); @@ -434,9 +420,20 @@ HDfprintf(stderr, "%s: Called\n", FUNC); if(HADDR_UNDEF == (hdr->addr = H5MF_alloc(f, H5FD_MEM_EARRAY_HDR, dxpl_id, (hsize_t)hdr->size))) H5E_THROW(H5E_CANTALLOC, "file allocation failed for extensible array header") + /* Create 'top' proxy for extensible array entries */ + if(hdr->swmr_write) + if(NULL == (hdr->top_proxy = H5AC_proxy_entry_create())) + H5E_THROW(H5E_CANTCREATE, "can't create extensible array entry proxy") + /* Cache the new extensible array header */ if(H5AC_insert_entry(f, dxpl_id, H5AC_EARRAY_HDR, hdr->addr, hdr, H5AC__NO_FLAGS_SET) < 0) H5E_THROW(H5E_CANTINSERT, "can't add extensible array header to cache") + inserted = TRUE; + + /* Add header as child of 'top' proxy */ + if(hdr->top_proxy) + if(H5AC_proxy_entry_add_child(hdr->top_proxy, f, dxpl_id, hdr) < 0) + H5E_THROW(H5E_CANTSET, "unable to add extensible array entry as child of array proxy") /* Set address of array header to return */ ret_value = hdr->addr; @@ -444,6 +441,11 @@ HDfprintf(stderr, "%s: Called\n", FUNC); CATCH if(!H5F_addr_defined(ret_value)) if(hdr) { + /* Remove from cache, if inserted */ + if(inserted) + if(H5AC_remove_entry(hdr) < 0) + H5E_THROW(H5E_CANTREMOVE, "unable to remove extensible array header from cache") + /* Release header's disk space */ if(H5F_addr_defined(hdr->addr) && H5MF_xfree(f, H5FD_MEM_EARRAY_HDR, dxpl_id, hdr->addr, (hsize_t)hdr->size) < 0) H5E_THROW(H5E_CANTFREE, "unable to free extensible array header") @@ -630,6 +632,7 @@ H5EA__hdr_protect(H5F_t *f, hid_t dxpl_id, haddr_t ea_addr, void *ctx_udata, unsigned flags)) /* Local variables */ + H5EA_hdr_t *hdr; /* Extensible array header */ H5EA_hdr_cache_ud_t udata; /* User data for cache callbacks */ /* Sanity check */ @@ -645,9 +648,23 @@ H5EA__hdr_protect(H5F_t *f, hid_t dxpl_id, haddr_t ea_addr, void *ctx_udata, udata.ctx_udata = ctx_udata; /* Protect the header */ - if(NULL == (ret_value = (H5EA_hdr_t *)H5AC_protect(f, dxpl_id, H5AC_EARRAY_HDR, ea_addr, &udata, flags))) + if(NULL == (hdr = (H5EA_hdr_t *)H5AC_protect(f, dxpl_id, H5AC_EARRAY_HDR, ea_addr, &udata, flags))) H5E_THROW(H5E_CANTPROTECT, "unable to protect extensible array header, address = %llu", (unsigned long long)ea_addr) - ret_value->f = f; /* (Must be set again here, in case the header was already in the cache -QAK) */ + hdr->f = f; /* (Must be set again here, in case the header was already in the cache -QAK) */ + + /* Create top proxy, if it doesn't exist */ + if(hdr->swmr_write && NULL == hdr->top_proxy) { + /* Create 'top' proxy for extensible array entries */ + if(NULL == (hdr->top_proxy = H5AC_proxy_entry_create())) + H5E_THROW(H5E_CANTCREATE, "can't create extensible array entry proxy") + + /* Add header as child of 'top' proxy */ + if(H5AC_proxy_entry_add_child(hdr->top_proxy, f, dxpl_id, hdr) < 0) + H5E_THROW(H5E_CANTSET, "unable to add extensible array entry as child of array proxy") + } /* end if */ + + /* Set return value */ + ret_value = hdr; CATCH @@ -725,9 +742,6 @@ H5EA__hdr_delete(H5EA_hdr_t *hdr, hid_t dxpl_id)) /* Check for index block */ if(H5F_addr_defined(hdr->idx_blk_addr)) { -#ifdef QAK -HDfprintf(stderr, "%s: hdr->idx_blk_addr = %a\n", FUNC, hdr->idx_blk_addr); -#endif /* QAK */ /* Delete index block */ if(H5EA__iblock_delete(hdr, dxpl_id) < 0) H5E_THROW(H5E_CANTDELETE, "unable to delete extensible array index block") @@ -798,6 +812,13 @@ H5EA__hdr_dest(H5EA_hdr_t *hdr)) if(hdr->sblk_info) hdr->sblk_info = (H5EA_sblk_info_t *)H5FL_SEQ_FREE(H5EA_sblk_info_t, hdr->sblk_info); + /* Destroy the 'top' proxy */ + if(hdr->top_proxy) { + if(H5AC_proxy_entry_dest(hdr->top_proxy) < 0) + H5E_THROW(H5E_CANTRELEASE, "unable to destroy extensible array 'top' proxy") + hdr->top_proxy = NULL; + } /* end if */ + /* Free the shared info itself */ hdr = H5FL_FREE(H5EA_hdr_t, hdr); diff --git a/src/H5EAiblock.c b/src/H5EAiblock.c index 623ae30..a3723c5 100644 --- a/src/H5EAiblock.c +++ b/src/H5EAiblock.c @@ -183,6 +183,7 @@ H5EA__iblock_create(H5EA_hdr_t *hdr, hid_t dxpl_id, hbool_t *stats_changed)) /* Local variables */ H5EA_iblock_t *iblock = NULL; /* Extensible array index block */ haddr_t iblock_addr; /* Extensible array index block address */ + hbool_t inserted = FALSE; /* Whether the header was inserted into cache */ #ifdef QAK HDfprintf(stderr, "%s: Called\n", FUNC); @@ -233,6 +234,14 @@ HDfprintf(stderr, "%s: iblock->size = %Zu\n", FUNC, iblock->size); /* Cache the new extensible array index block */ if(H5AC_insert_entry(hdr->f, dxpl_id, H5AC_EARRAY_IBLOCK, iblock_addr, iblock, H5AC__NO_FLAGS_SET) < 0) H5E_THROW(H5E_CANTINSERT, "can't add extensible array index block to cache") + inserted = TRUE; + + /* Add index block as child of 'top' proxy */ + if(hdr->top_proxy) { + if(H5AC_proxy_entry_add_child(hdr->top_proxy, hdr->f, dxpl_id, iblock) < 0) + H5E_THROW(H5E_CANTSET, "unable to add extensible array entry as child of array proxy") + iblock->top_proxy = hdr->top_proxy; + } /* end if */ /* Update extensible array index block statistics */ HDassert(0 == hdr->stats.computed.nindex_blks); @@ -252,9 +261,14 @@ HDfprintf(stderr, "%s: iblock->size = %Zu\n", FUNC, iblock->size); CATCH if(!H5F_addr_defined(ret_value)) if(iblock) { + /* Remove from cache, if inserted */ + if(inserted) + if(H5AC_remove_entry(iblock) < 0) + H5E_THROW(H5E_CANTREMOVE, "unable to remove extensible array index block from cache") + /* Release index block's disk space */ if(H5F_addr_defined(iblock->addr) && H5MF_xfree(hdr->f, H5FD_MEM_EARRAY_IBLOCK, dxpl_id, iblock->addr, (hsize_t)iblock->size) < 0) - H5E_THROW(H5E_CANTFREE, "unable to release extensible array index block") + H5E_THROW(H5E_CANTFREE, "unable to release file space for extensible array index block") /* Destroy index block */ if(H5EA__iblock_dest(iblock) < 0) @@ -281,6 +295,9 @@ BEGIN_FUNC(PKG, ERR, H5EA_iblock_t *, NULL, NULL, H5EA__iblock_protect(H5EA_hdr_t *hdr, hid_t dxpl_id, unsigned flags)) + /* Local variables */ + H5EA_iblock_t *iblock = NULL; /* Pointer to index block */ + #ifdef QAK HDfprintf(stderr, "%s: Called\n", FUNC); #endif /* QAK */ @@ -292,10 +309,27 @@ HDfprintf(stderr, "%s: Called\n", FUNC); HDassert((flags & (unsigned)(~H5AC__READ_ONLY_FLAG)) == 0); /* Protect the index block */ - if(NULL == (ret_value = (H5EA_iblock_t *)H5AC_protect(hdr->f, dxpl_id, H5AC_EARRAY_IBLOCK, hdr->idx_blk_addr, hdr, flags))) + if(NULL == (iblock = (H5EA_iblock_t *)H5AC_protect(hdr->f, dxpl_id, H5AC_EARRAY_IBLOCK, hdr->idx_blk_addr, hdr, flags))) H5E_THROW(H5E_CANTPROTECT, "unable to protect extensible array index block, address = %llu", (unsigned long long)hdr->idx_blk_addr) + /* Create top proxy, if it doesn't exist */ + if(hdr->top_proxy && NULL == iblock->top_proxy) { + /* Add index block as child of 'top' proxy */ + if(H5AC_proxy_entry_add_child(hdr->top_proxy, hdr->f, dxpl_id, iblock) < 0) + H5E_THROW(H5E_CANTSET, "unable to add extensible array entry as child of array proxy") + iblock->top_proxy = hdr->top_proxy; + } /* end if */ + + /* Set return value */ + ret_value = iblock; + CATCH + /* Clean up on error */ + if(!ret_value) { + /* Release the index block, if it was protected */ + if(iblock && H5AC_unprotect(hdr->f, dxpl_id, H5AC_EARRAY_IBLOCK, iblock->addr, iblock, H5AC__NO_FLAGS_SET) < 0) + H5E_THROW(H5E_CANTUNPROTECT, "unable to unprotect extensible array index block, address = %llu", (unsigned long long)iblock->addr) + } /* end if */ END_FUNC(PKG) /* end H5EA__iblock_protect() */ @@ -470,6 +504,9 @@ H5EA__iblock_dest(H5EA_iblock_t *iblock)) iblock->hdr = NULL; } /* end if */ + /* Sanity check */ + HDassert(NULL == iblock->top_proxy); + /* Free the index block itself */ iblock = H5FL_FREE(H5EA_iblock_t, iblock); diff --git a/src/H5EApkg.h b/src/H5EApkg.h index 6f3ee9c..093403c 100644 --- a/src/H5EApkg.h +++ b/src/H5EApkg.h @@ -206,6 +206,28 @@ typedef struct H5EA_hdr_t { /* Client information (not stored) */ void *cb_ctx; /* Callback context */ + + /* SWMR / Flush dependency information (not stored) */ + hbool_t swmr_write; /* Flag indicating the file is opened with SWMR-write access */ + H5AC_proxy_entry_t *top_proxy; /* 'Top' proxy cache entry for all array entries */ + void *parent; /* Pointer to 'top' proxy flush dependency + * parent, if it exists, otherwise NULL. + * If the extensible array is being used + * to index a chunked dataset and the + * dataset metadata is modified by a + * SWMR writer, this field will be set + * equal to the object header proxy + * that is the flush dependency parent + * of the extensible array header. + * + * The field is used to avoid duplicate + * setups of the flush dependency + * relationship, and to allow the + * extensible array header to destroy + * the flush dependency on receipt of + * an eviction notification from the + * metadata cache. + */ } H5EA_hdr_t; /* The extensible array index block information */ @@ -223,6 +245,9 @@ typedef struct H5EA_iblock_t { haddr_t addr; /* Address of this index block on disk */ size_t size; /* Size of index block on disk */ + /* SWMR / Flush dependency information (not stored) */ + H5AC_proxy_entry_t *top_proxy; /* "Top" proxy cache entry for all array entries */ + /* Computed/cached values (not stored) */ size_t nsblks; /* # of super blocks whose data block addresses are in index block */ size_t ndblk_addrs; /* Number of pointers to data blocks in index block */ @@ -241,11 +266,14 @@ typedef struct H5EA_sblock_t { /* Internal array information (not stored) */ H5EA_hdr_t *hdr; /* Shared array header info */ - hbool_t has_hdr_depend; /* Whether this object has a flush dependency on the header */ - H5EA_iblock_t *parent; /* Parent object for super block (index block) */ haddr_t addr; /* Address of this index block on disk */ size_t size; /* Size of index block on disk */ + /* SWMR / Flush dependency information (not stored) */ + hbool_t has_hdr_depend; /* Whether this object has a flush dependency on the header */ + H5AC_proxy_entry_t *top_proxy; /* "Top" proxy cache entry for all array entries */ + H5EA_iblock_t *parent; /* Parent object for super block (index block) */ + /* Computed/cached values (not stored) */ unsigned idx; /* Super block index within the extensible array */ size_t ndblks; /* # of data block addresses that are in super block */ @@ -266,11 +294,14 @@ typedef struct H5EA_dblock_t { /* Internal array information (not stored) */ H5EA_hdr_t *hdr; /* Shared array header info */ - hbool_t has_hdr_depend; /* Whether this object has a flush dependency on the header */ - void *parent; /* Parent object for data block (index or super block) */ haddr_t addr; /* Address of this data block on disk */ size_t size; /* Size of data block on disk */ + /* SWMR / Flush dependency information (not stored) */ + hbool_t has_hdr_depend; /* Whether this object has a flush dependency on the header */ + H5AC_proxy_entry_t *top_proxy; /* 'Top' proxy cache entry for all array entries */ + void *parent; /* Parent object for data block (index or super block) */ + /* Computed/cached values (not stored) */ size_t nelmts; /* Number of elements in block */ size_t npages; /* Nummber of pages in a block (zero if not paged) */ @@ -286,11 +317,14 @@ typedef struct H5EA_dbk_page_t { /* Internal array information (not stored) */ H5EA_hdr_t *hdr; /* Shared array header info */ - hbool_t has_hdr_depend; /* Whether this object has a flush dependency on the header */ - H5EA_sblock_t *parent; /* Parent object for data block page (super block) */ haddr_t addr; /* Address of this data block page on disk */ size_t size; /* Size of data block page on disk */ + /* SWMR / Flush dependency information (not stored) */ + hbool_t has_hdr_depend; /* Whether this object has a flush dependency on the header */ + H5AC_proxy_entry_t *top_proxy; /* "Top" proxy cache entry for all array entries */ + H5EA_sblock_t *parent; /* Parent object for data block page (super block) */ + /* Computed/cached values (not stored) */ /* <none> */ } H5EA_dblk_page_t; diff --git a/src/H5EAprivate.h b/src/H5EAprivate.h index 0a1b945..66d88e0 100644 --- a/src/H5EAprivate.h +++ b/src/H5EAprivate.h @@ -142,7 +142,7 @@ H5_DLL herr_t H5EA_get_nelmts(const H5EA_t *ea, hsize_t *nelmts); H5_DLL herr_t H5EA_get_addr(const H5EA_t *ea, haddr_t *addr); H5_DLL herr_t H5EA_set(const H5EA_t *ea, hid_t dxpl_id, hsize_t idx, const void *elmt); H5_DLL herr_t H5EA_get(const H5EA_t *ea, hid_t dxpl_id, hsize_t idx, void *elmt); -H5_DLL herr_t H5EA_depend(H5AC_info_t *parent_entry, H5EA_t *ea); +H5_DLL herr_t H5EA_depend(H5EA_t *ea, hid_t dxpl_id, H5AC_proxy_entry_t *parent); H5_DLL herr_t H5EA_iterate(H5EA_t *fa, hid_t dxpl_id, H5EA_operator_t op, void *udata); H5_DLL herr_t H5EA_close(H5EA_t *ea, hid_t dxpl_id); H5_DLL herr_t H5EA_delete(H5F_t *f, hid_t dxpl_id, haddr_t ea_addr, void *ctx_udata); diff --git a/src/H5EAsblock.c b/src/H5EAsblock.c index 4f153cd..2fa87e0 100644 --- a/src/H5EAsblock.c +++ b/src/H5EAsblock.c @@ -195,6 +195,7 @@ H5EA__sblock_create(H5EA_hdr_t *hdr, hid_t dxpl_id, H5EA_iblock_t *parent, H5EA_sblock_t *sblock = NULL; /* Extensible array super block */ haddr_t sblock_addr; /* Extensible array super block address */ haddr_t tmp_addr = HADDR_UNDEF; /* Address value to fill data block addresses with */ + hbool_t inserted = FALSE; /* Whether the header was inserted into cache */ /* Sanity check */ HDassert(hdr); @@ -221,6 +222,14 @@ H5EA__sblock_create(H5EA_hdr_t *hdr, hid_t dxpl_id, H5EA_iblock_t *parent, /* Cache the new extensible array super block */ if(H5AC_insert_entry(hdr->f, dxpl_id, H5AC_EARRAY_SBLOCK, sblock_addr, sblock, H5AC__NO_FLAGS_SET) < 0) H5E_THROW(H5E_CANTINSERT, "can't add extensible array super block to cache") + inserted = TRUE; + + /* Add super block as child of 'top' proxy */ + if(hdr->top_proxy) { + if(H5AC_proxy_entry_add_child(hdr->top_proxy, hdr->f, dxpl_id, sblock) < 0) + H5E_THROW(H5E_CANTSET, "unable to add extensible array entry as child of array proxy") + sblock->top_proxy = hdr->top_proxy; + } /* end if */ /* Update extensible array super block statistics */ hdr->stats.stored.nsuper_blks++; @@ -235,6 +244,11 @@ H5EA__sblock_create(H5EA_hdr_t *hdr, hid_t dxpl_id, H5EA_iblock_t *parent, CATCH if(!H5F_addr_defined(ret_value)) if(sblock) { + /* Remove from cache, if inserted */ + if(inserted) + if(H5AC_remove_entry(sblock) < 0) + H5E_THROW(H5E_CANTREMOVE, "unable to remove extensible array super block from cache") + /* Release super block's disk space */ if(H5F_addr_defined(sblock->addr) && H5MF_xfree(hdr->f, H5FD_MEM_EARRAY_SBLOCK, dxpl_id, sblock->addr, (hsize_t)sblock->size) < 0) H5E_THROW(H5E_CANTFREE, "unable to release extensible array super block") @@ -266,6 +280,7 @@ H5EA__sblock_protect(H5EA_hdr_t *hdr, hid_t dxpl_id, H5EA_iblock_t *parent, haddr_t sblk_addr, unsigned sblk_idx, unsigned flags)) /* Local variables */ + H5EA_sblock_t *sblock = NULL; /* Pointer to super block */ H5EA_sblock_cache_ud_t udata; /* Information needed for loading super block */ /* Sanity check */ @@ -282,10 +297,27 @@ H5EA__sblock_protect(H5EA_hdr_t *hdr, hid_t dxpl_id, H5EA_iblock_t *parent, udata.sblk_addr = sblk_addr; /* Protect the super block */ - if(NULL == (ret_value = (H5EA_sblock_t *)H5AC_protect(hdr->f, dxpl_id, H5AC_EARRAY_SBLOCK, sblk_addr, &udata, flags))) + if(NULL == (sblock = (H5EA_sblock_t *)H5AC_protect(hdr->f, dxpl_id, H5AC_EARRAY_SBLOCK, sblk_addr, &udata, flags))) H5E_THROW(H5E_CANTPROTECT, "unable to protect extensible array super block, address = %llu", (unsigned long long)sblk_addr) + /* Create top proxy, if it doesn't exist */ + if(hdr->top_proxy && NULL == sblock->top_proxy) { + /* Add super block as child of 'top' proxy */ + if(H5AC_proxy_entry_add_child(hdr->top_proxy, hdr->f, dxpl_id, sblock) < 0) + H5E_THROW(H5E_CANTSET, "unable to add extensible array entry as child of array proxy") + sblock->top_proxy = hdr->top_proxy; + } /* end if */ + + /* Set return value */ + ret_value = sblock; + CATCH + /* Clean up on error */ + if(!ret_value) { + /* Release the super block, if it was protected */ + if(sblock && H5AC_unprotect(hdr->f, dxpl_id, H5AC_EARRAY_SBLOCK, sblock->addr, sblock, H5AC__NO_FLAGS_SET) < 0) + H5E_THROW(H5E_CANTUNPROTECT, "unable to unprotect extensible array super block, address = %llu", (unsigned long long)sblock->addr) + } /* end if */ END_FUNC(PKG) /* end H5EA__sblock_protect() */ @@ -409,6 +441,9 @@ H5EA__sblock_dest(H5EA_sblock_t *sblock)) sblock->hdr = NULL; } /* end if */ + /* Sanity check */ + HDassert(NULL == sblock->top_proxy); + /* Free the super block itself */ sblock = H5FL_FREE(H5EA_sblock_t, sblock); @@ -456,9 +456,9 @@ H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id) if(!filename || !*filename) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file name") /* In this routine, we only accept the following flags: - * H5F_ACC_EXCL and H5F_ACC_TRUNC + * H5F_ACC_EXCL, H5F_ACC_TRUNC and H5F_ACC_SWMR_WRITE */ - if(flags & ~(H5F_ACC_EXCL | H5F_ACC_TRUNC)) + if(flags & ~(H5F_ACC_EXCL | H5F_ACC_TRUNC | H5F_ACC_SWMR_WRITE)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid flags") /* The H5F_ACC_EXCL and H5F_ACC_TRUNC flags are mutually exclusive */ if((flags & H5F_ACC_EXCL) && (flags & H5F_ACC_TRUNC)) @@ -563,6 +563,12 @@ H5Fopen(const char *filename, unsigned flags, hid_t fapl_id) if((flags & ~H5F_ACC_PUBLIC_FLAGS) || (flags & H5F_ACC_TRUNC) || (flags & H5F_ACC_EXCL)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file open flags") + /* Asking for SWMR write access on a read-only file is invalid */ + if((flags & H5F_ACC_SWMR_WRITE) && 0 == (flags & H5F_ACC_RDWR)) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "SWMR write access on a file open for read-only access is not allowed") + /* Asking for SWMR read access on a non-read-only file is invalid */ + if((flags & H5F_ACC_SWMR_READ) && (flags & H5F_ACC_RDWR)) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "SWMR read access on a file open for read-write access is not allowed") /* Verify access property list and get correct dxpl */ if(H5P_verify_apl_and_dxpl(&fapl_id, H5P_CLS_FACC, &dxpl_id, H5I_INVALID_HID, TRUE) < 0) @@ -580,7 +586,7 @@ H5Fopen(const char *filename, unsigned flags, hid_t fapl_id) new_file->file_id = ret_value; done: - if(ret_value < 0 && new_file && H5F_try_close(new_file) < 0) + if(ret_value < 0 && new_file && H5F_try_close(new_file, NULL) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "problems closing file") FUNC_LEAVE_API(ret_value) @@ -857,10 +863,20 @@ H5Fget_intent(hid_t file_id, unsigned *intent_flags) * Simplify things for them so that they only get either H5F_ACC_RDWR * or H5F_ACC_RDONLY. */ - if(H5F_INTENT(file) & H5F_ACC_RDWR) + if(H5F_INTENT(file) & H5F_ACC_RDWR) { *intent_flags = H5F_ACC_RDWR; - else + + /* Check for SWMR write access on the file */ + if(H5F_INTENT(file) & H5F_ACC_SWMR_WRITE) + *intent_flags |= H5F_ACC_SWMR_WRITE; + } /* end if */ + else { *intent_flags = H5F_ACC_RDONLY; + + /* Check for SWMR read access on the file */ + if(H5F_INTENT(file) & H5F_ACC_SWMR_READ) + *intent_flags |= H5F_ACC_SWMR_READ; + } /* end else */ } /* end if */ done: @@ -1367,6 +1383,103 @@ done: /*------------------------------------------------------------------------- + * Function: H5Fget_metadata_read_retry_info + * + * Purpose: To retrieve the collection of read retries for metadata items with checksum. + * + * Return: Success: non-negative on success + * Failure: Negative + * + * Programmer: Vailin Choi; October 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Fget_metadata_read_retry_info(hid_t file_id, H5F_retry_info_t *info) +{ + H5F_t *file; /* File object for file ID */ + unsigned i, j; /* Local index variable */ + size_t tot_size; /* Size of each retries[i] */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*x", file_id, info); + + /* Check args */ + if(!info) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no info struct") + + /* Get the file pointer */ + if(NULL == (file = (H5F_t *)H5I_object_verify(file_id, H5I_FILE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a file ID") + + /* Copy the # of bins for "retries" array */ + info->nbins = file->shared->retries_nbins; + + /* Initialize the array of "retries" */ + HDmemset(info->retries, 0, sizeof(info->retries)); + + /* Return if there are no bins -- no retries */ + if(!info->nbins) + HGOTO_DONE(SUCCEED); + + /* Calculate size for each retries[i] */ + tot_size = info->nbins * sizeof(uint32_t); + + /* Map and copy information to info's retries for metadata items with tracking for read retries */ + j = 0; + for(i = 0; i < H5AC_NTYPES; i++) { + switch(i) { + case H5AC_OHDR_ID: + case H5AC_OHDR_CHK_ID: + case H5AC_BT2_HDR_ID: + case H5AC_BT2_INT_ID: + case H5AC_BT2_LEAF_ID: + case H5AC_FHEAP_HDR_ID: + case H5AC_FHEAP_DBLOCK_ID: + case H5AC_FHEAP_IBLOCK_ID: + case H5AC_FSPACE_HDR_ID: + case H5AC_FSPACE_SINFO_ID: + case H5AC_SOHM_TABLE_ID: + case H5AC_SOHM_LIST_ID: + case H5AC_EARRAY_HDR_ID: + case H5AC_EARRAY_IBLOCK_ID: + case H5AC_EARRAY_SBLOCK_ID: + case H5AC_EARRAY_DBLOCK_ID: + case H5AC_EARRAY_DBLK_PAGE_ID: + case H5AC_FARRAY_HDR_ID: + case H5AC_FARRAY_DBLOCK_ID: + case H5AC_FARRAY_DBLK_PAGE_ID: + case H5AC_SUPERBLOCK_ID: + HDassert(j < H5F_NUM_METADATA_READ_RETRY_TYPES); + if(file->shared->retries[i] != NULL) { + /* Allocate memory for retries[i] + * + * This memory should be released by the user with + * the H5free_memory() call. + */ + if(NULL == (info->retries[j] = (uint32_t *)H5MM_malloc(tot_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + + /* Copy the information */ + HDmemcpy(info->retries[j], file->shared->retries[i], tot_size); + } /* end if */ + + /* Increment location in info->retries[] array */ + j++; + break; + + default: + break; + } /* end switch */ + } /* end for */ + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Fget_metadata_read_retry_info() */ + + +/*------------------------------------------------------------------------- * Function: H5Fget_free_sections * * Purpose: To get free-space section information for free-space manager with @@ -1444,6 +1557,321 @@ done: /*------------------------------------------------------------------------- + * Function: H5Fstart_swmr_write + * + * Purpose: To enable SWMR writing mode for the file + * 1) Refresh opened objects: part 1 + * 2) Flush & reset accumulator + * 3) Mark the file in SWMR writing mode + * 4) Set metadata read attempts and retries info + * 5) Disable accumulator + * 6) Evict all cache entries except the superblock + * 7) Refresh opened objects (part 2) + * 8) Unlock the file + * + * Pre-conditions: + * 1) The file being opened has v3 superblock + * 2) The file is opened with H5F_ACC_RDWR + * 3) The file is not already marked for SWMR writing + * 4) Current implementaion for opened objects: + * --only allow datasets and groups without attributes + * --disallow named datatype with/without attributes + * --disallow opened attributes attached to objects + * NOTE: Currently, only opened groups and datasets are allowed + * when enabling SWMR via H5Fstart_swmr_write(). + * Will later implement a different approach-- + * set up flush dependency/proxy even for file opened without + * SWMR to resolve issues with opened objects. + * + * Return: Non-negative on success/negative on failure + * + * Programmer: + * Vailin Choi; Feb 2014 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Fstart_swmr_write(hid_t file_id) +{ + H5F_t *file = NULL; /* File info */ + size_t grp_dset_count=0; /* # of open objects: groups & datasets */ + size_t nt_attr_count=0; /* # of opened named datatypes + opened attributes */ + hid_t *obj_ids=NULL; /* List of ids */ + H5G_loc_t *obj_glocs=NULL; /* Group location of the object */ + H5O_loc_t *obj_olocs=NULL; /* Object location */ + H5G_name_t *obj_paths=NULL; /* Group hierarchy path */ + size_t u; /* Local index variable */ + hbool_t setup = FALSE; /* Boolean flag to indicate whether SWMR setting is enabled */ + H5F_io_info_t fio_info; /* I/O info for operation */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE1("e", "i", file_id); + + /* check args */ + if(NULL == (file = (H5F_t *)H5I_object_verify(file_id, H5I_FILE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file") + + /* Should have write permission */ + if((H5F_INTENT(file) & H5F_ACC_RDWR) == 0) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "no write intent on file") + + if(file->shared->sblock->super_vers < HDF5_SUPERBLOCK_VERSION_3) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "file superblock version should be at least 3") + HDassert(file->shared->latest_flags == H5F_LATEST_ALL_FLAGS); + + /* Should not be marked for SWMR writing mode already */ + if(file->shared->sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "file already in SWMR writing mode") + + HDassert(file->shared->sblock->status_flags & H5F_SUPER_WRITE_ACCESS); + + /* Flush data buffers */ + if(H5F_flush(file, H5AC_ind_read_dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush file's cached information") + + /* Get the # of opened named datatypes and attributes */ + if(H5F_get_obj_count(file, H5F_OBJ_DATATYPE|H5F_OBJ_ATTR, FALSE, &nt_attr_count) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_BADITER, FAIL, "H5F_get_obj_count failed") + if(nt_attr_count) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "named datatypes and/or attributes opened in the file") + + /* Get the # of opened datasets and groups */ + if(H5F_get_obj_count(file, H5F_OBJ_GROUP|H5F_OBJ_DATASET, FALSE, &grp_dset_count) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_BADITER, FAIL, "H5F_get_obj_count failed") + + if(grp_dset_count) { + /* Allocate space for group and object locations */ + if((obj_ids = (hid_t *) H5MM_malloc(grp_dset_count * sizeof(hid_t))) == NULL) + HGOTO_ERROR(H5E_FILE, H5E_NOSPACE, FAIL, "can't allocate buffer for hid_t") + if((obj_glocs = (H5G_loc_t *) H5MM_malloc(grp_dset_count * sizeof(H5G_loc_t))) == NULL) + HGOTO_ERROR(H5E_FILE, H5E_NOSPACE, FAIL, "can't allocate buffer for H5G_loc_t") + if((obj_olocs = (H5O_loc_t *) H5MM_malloc(grp_dset_count * sizeof(H5O_loc_t))) == NULL) + HGOTO_ERROR(H5E_FILE, H5E_NOSPACE, FAIL, "can't allocate buffer for H5O_loc_t") + if((obj_paths = (H5G_name_t *) H5MM_malloc(grp_dset_count * sizeof(H5G_name_t))) == NULL) + HGOTO_ERROR(H5E_FILE, H5E_NOSPACE, FAIL, "can't allocate buffer for H5G_name_t") + + /* Get the list of opened object ids (groups & datasets) */ + if(H5F_get_obj_ids(file, H5F_OBJ_GROUP|H5F_OBJ_DATASET, grp_dset_count, obj_ids, FALSE, &grp_dset_count) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "H5F_get_obj_ids failed") + + /* Refresh opened objects (groups, datasets) in the file */ + for(u = 0; u < grp_dset_count; u++) { + H5O_loc_t *oloc; /* object location */ + H5G_loc_t tmp_loc; + + /* Set up the id's group location */ + obj_glocs[u].oloc = &obj_olocs[u]; + obj_glocs[u].path = &obj_paths[u]; + H5G_loc_reset(&obj_glocs[u]); + + /* get the id's object location */ + if((oloc = H5O_get_loc(obj_ids[u])) == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not an object") + + /* Make deep local copy of object's location information */ + H5G_loc(obj_ids[u], &tmp_loc); + H5G_loc_copy(&obj_glocs[u], &tmp_loc, H5_COPY_DEEP); + + /* Close the object */ + if(H5I_dec_ref(obj_ids[u]) < 0) + HGOTO_ERROR(H5E_ATOM, H5E_CANTCLOSEOBJ, FAIL, "decrementing object ID failed") + } /* end for */ + } /* end if */ + + /* Set up I/O info for operation */ + fio_info.f = file; + if(NULL == (fio_info.dxpl = (H5P_genplist_t *)H5I_object(H5AC_ind_read_dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get property list") + + /* Flush and reset the accumulator */ + if(H5F__accum_reset(&fio_info, TRUE) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTRESET, FAIL, "can't reset accumulator") + + /* Turn on SWMR write in shared file open flags */ + file->shared->flags |= H5F_ACC_SWMR_WRITE; + + /* Mark the file in SWMR writing mode */ + file->shared->sblock->status_flags |= H5F_SUPER_SWMR_WRITE_ACCESS; + + /* Set up metadata read attempts */ + file->shared->read_attempts = H5F_SWMR_METADATA_READ_ATTEMPTS; + + /* Initialize "retries" and "retries_nbins" */ + if(H5F_set_retries(file) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "can't set retries and retries_nbins") + + /* Turn off usage of accumulator */ + file->shared->feature_flags &= ~(unsigned)H5FD_FEAT_ACCUMULATE_METADATA; + if(H5FD_set_feature_flags(file->shared->lf, file->shared->feature_flags) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't set feature_flags in VFD") + + setup = TRUE; + + /* Mark superblock as dirty */ + if(H5F_super_dirty(file) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty") + + /* Flush the superblock */ + if(H5F_flush_tagged_metadata(file, H5AC__SUPERBLOCK_TAG, H5AC_ind_read_dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush superblock") + + /* Evict all flushed entries in the cache except the pinned superblock */ + if(H5F__evict_cache_entries(file, H5AC_ind_read_dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to evict file's cached information") + + /* Refresh (reopen) the objects (groups & datasets) in the file */ + for(u = 0; u < grp_dset_count; u++) + if(H5O_refresh_metadata_reopen(obj_ids[u], &obj_glocs[u], H5AC_ind_read_dxpl_id, TRUE) < 0) + HGOTO_ERROR(H5E_ATOM, H5E_CLOSEERROR, FAIL, "can't refresh-close object") + + /* Unlock the file */ + if(H5FD_unlock(file->shared->lf) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to unlock the file") + +done: + if(ret_value < 0 && setup) { + HDassert(file); + + /* Re-enable accumulator */ + file->shared->feature_flags |= (unsigned)H5FD_FEAT_ACCUMULATE_METADATA; + if(H5FD_set_feature_flags(file->shared->lf, file->shared->feature_flags) < 0) + HDONE_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "can't set feature_flags in VFD") + + /* Reset the # of read attempts */ + file->shared->read_attempts = H5F_METADATA_READ_ATTEMPTS; + if(H5F_set_retries(file) < 0) + HDONE_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "can't set retries and retries_nbins") + + /* Un-set H5F_ACC_SWMR_WRITE in shared open flags */ + file->shared->flags &= ~H5F_ACC_SWMR_WRITE; + + /* Unmark the file: not in SWMR writing mode */ + file->shared->sblock->status_flags &= (uint8_t)(~H5F_SUPER_SWMR_WRITE_ACCESS); + + /* Mark superblock as dirty */ + if(H5F_super_dirty(file) < 0) + HDONE_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty") + + /* Flush the superblock */ + if(H5F_flush_tagged_metadata(file, H5AC__SUPERBLOCK_TAG, H5AC_ind_read_dxpl_id) < 0) + HDONE_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush superblock") + } /* end if */ + + /* Free memory */ + if(obj_ids) + H5MM_xfree(obj_ids); + if(obj_glocs) + H5MM_xfree(obj_glocs); + if(obj_olocs) + H5MM_xfree(obj_olocs); + if(obj_paths) + H5MM_xfree(obj_paths); + + FUNC_LEAVE_API(ret_value) +} /* end H5Fstart_swmr_write() */ + + +/*------------------------------------------------------------------------- + * Function: H5Fstart_mdc_logging + * + * Purpose: Start metadata cache logging operations for a file. + * - Logging must have been set up via the fapl. + * + * Return: Non-negative on success/Negative on errors + * + *------------------------------------------------------------------------- + */ +herr_t +H5Fstart_mdc_logging(hid_t file_id) +{ + H5F_t *file; /* File info */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE1("e", "i", file_id); + + /* Sanity check */ + if(NULL == (file = (H5F_t *)H5I_object_verify(file_id, H5I_FILE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "hid_t identifier is not a file ID") + + /* Call mdc logging function */ + if(H5C_start_logging(file->shared->cache) < 0) + HGOTO_ERROR(H5E_FILE, H5E_LOGFAIL, FAIL, "unable to start mdc logging") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Fstart_mdc_logging() */ + + +/*------------------------------------------------------------------------- + * Function: H5Fstop_mdc_logging + * + * Purpose: Stop metadata cache logging operations for a file. + * - Does not close the log file. + * - Logging must have been set up via the fapl. + * + * Return: Non-negative on success/Negative on errors + * + *------------------------------------------------------------------------- + */ +herr_t +H5Fstop_mdc_logging(hid_t file_id) +{ + H5F_t *file; /* File info */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE1("e", "i", file_id); + + /* Sanity check */ + if(NULL == (file = (H5F_t *)H5I_object_verify(file_id, H5I_FILE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "hid_t identifier is not a file ID") + + /* Call mdc logging function */ + if(H5C_stop_logging(file->shared->cache) < 0) + HGOTO_ERROR(H5E_FILE, H5E_LOGFAIL, FAIL, "unable to stop mdc logging") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Fstop_mdc_logging() */ + + +/*------------------------------------------------------------------------- + * Function: H5Fget_mdc_logging_status + * + * Purpose: Get the logging flags. is_enabled determines if logging was + * set up via the fapl. is_currently_logging determines if + * log messages are being recorded at this time. + * + * Return: Non-negative on success/Negative on errors + * + *------------------------------------------------------------------------- + */ +herr_t +H5Fget_mdc_logging_status(hid_t file_id, hbool_t *is_enabled, + hbool_t *is_currently_logging) +{ + H5F_t *file; /* File info */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "i*b*b", file_id, is_enabled, is_currently_logging); + + /* Sanity check */ + if(NULL == (file = (H5F_t *)H5I_object_verify(file_id, H5I_FILE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "hid_t identifier is not a file ID") + + /* Call mdc logging function */ + if(H5C_get_logging_status(file->shared->cache, is_enabled, is_currently_logging) < 0) + HGOTO_ERROR(H5E_FILE, H5E_LOGFAIL, FAIL, "unable to get logging status") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Fstop_mdc_logging() */ + + +/*------------------------------------------------------------------------- * Function: H5Fformat_convert_super (Internal) * * Purpose: Downgrade the superblock version to v2 and @@ -64,6 +64,8 @@ /********************/ /* Local Prototypes */ /********************/ +static H5FA_t *H5FA__new(H5F_t *f, hid_t dxpl_id, haddr_t fa_addr, + hbool_t from_open, void *ctx_udata); /*********************/ @@ -103,60 +105,53 @@ H5FL_BLK_DEFINE(fa_native_elmt); /*------------------------------------------------------------------------- - * Function: H5FA_create + * Function: H5FA__new * - * Purpose: Creates a new fixed array (header) in the file. + * Purpose: Allocate and initialize a new fixe array wrapper in memory * - * Return: Pointer to fixed array wrapper on success + * Return: Pointer to farray wrapper success * NULL on failure * - * Programmer: Vailin Choi - * Thursday, April 30, 2009 + * Programmer: Quincey Koziol + * koziol@lbl.gov + * Oct 17 2016 * *------------------------------------------------------------------------- */ -BEGIN_FUNC(PRIV, ERR, +BEGIN_FUNC(STATIC, ERR, H5FA_t *, NULL, NULL, -H5FA_create(H5F_t *f, hid_t dxpl_id, const H5FA_create_t *cparam, void *ctx_udata)) +H5FA__new(H5F_t *f, hid_t dxpl_id, haddr_t fa_addr, hbool_t from_open, void *ctx_udata)) /* Local variables */ H5FA_t *fa = NULL; /* Pointer to new fixed array */ H5FA_hdr_t *hdr = NULL; /* The fixed array header information */ - haddr_t fa_addr; /* Fixed Array header address */ - -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* H5FA_DEBUG */ /* * Check arguments. */ HDassert(f); - HDassert(cparam); - - /* H5FA interface sanity check */ - HDcompile_assert(H5FA_NUM_CLS_ID == NELMTS(H5FA_client_class_g)); - - /* Create fixed array header */ - if(HADDR_UNDEF == (fa_addr = H5FA__hdr_create(f, dxpl_id, cparam, ctx_udata))) - H5E_THROW(H5E_CANTINIT, "can't create fixed array header") + HDassert(H5F_addr_defined(fa_addr)); /* Allocate fixed array wrapper */ - if(NULL == (fa = H5FL_MALLOC(H5FA_t))) - H5E_THROW(H5E_CANTALLOC, "memory allocation failed for fixed array info") + if(NULL == (fa = H5FL_CALLOC(H5FA_t))) + H5E_THROW(H5E_CANTALLOC, "memory allocation failed for fixed array info") /* Lock the array header into memory */ - if(NULL == (hdr = H5FA__hdr_protect(f, dxpl_id, fa_addr, ctx_udata, H5AC__NO_FLAGS_SET))) - H5E_THROW(H5E_CANTPROTECT, "unable to load fixed array header") + if(NULL == (hdr = H5FA__hdr_protect(f, dxpl_id, fa_addr, ctx_udata, H5AC__READ_ONLY_FLAG))) + H5E_THROW(H5E_CANTPROTECT, "unable to load fixed array header") + + /* Check for pending array deletion */ + if(from_open && hdr->pending_delete) + H5E_THROW(H5E_CANTOPENOBJ, "can't open fixed array pending deletion") /* Point fixed array wrapper at header and bump it's ref count */ fa->hdr = hdr; if(H5FA__hdr_incr(fa->hdr) < 0) - H5E_THROW(H5E_CANTINC, "can't increment reference count on shared array header") + H5E_THROW(H5E_CANTINC, "can't increment reference count on shared array header") /* Increment # of files using this array header */ if(H5FA__hdr_fuse_incr(fa->hdr) < 0) - H5E_THROW(H5E_CANTINC, "can't increment file reference count on shared array header") + H5E_THROW(H5E_CANTINC, "can't increment file reference count on shared array header") /* Set file pointer for this array open context */ fa->f = f; @@ -167,7 +162,57 @@ HDfprintf(stderr, "%s: Called\n", FUNC); CATCH if(hdr && H5FA__hdr_unprotect(hdr, dxpl_id, H5AC__NO_FLAGS_SET) < 0) - H5E_THROW(H5E_CANTUNPROTECT, "unable to release fixed array header") + H5E_THROW(H5E_CANTUNPROTECT, "unable to release fixed array header") + if(!ret_value) + if(fa && H5FA_close(fa, dxpl_id) < 0) + H5E_THROW(H5E_CLOSEERROR, "unable to close fixed array") + +END_FUNC(STATIC) /* end H5FA__new() */ + + +/*------------------------------------------------------------------------- + * Function: H5FA_create + * + * Purpose: Creates a new fixed array (header) in the file. + * + * Return: Pointer to fixed array wrapper on success + * NULL on failure + * + * Programmer: Vailin Choi + * Thursday, April 30, 2009 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(PRIV, ERR, +H5FA_t *, NULL, NULL, +H5FA_create(H5F_t *f, hid_t dxpl_id, const H5FA_create_t *cparam, void *ctx_udata)) + + /* Local variables */ + H5FA_t *fa = NULL; /* Pointer to new fixed array */ + haddr_t fa_addr; /* Fixed array header address */ + + /* + * Check arguments. + */ + HDassert(f); + HDassert(cparam); + + /* H5FA interface sanity check */ + HDcompile_assert(H5FA_NUM_CLS_ID == NELMTS(H5FA_client_class_g)); + + /* Create fixed array header */ + if(HADDR_UNDEF == (fa_addr = H5FA__hdr_create(f, dxpl_id, cparam, ctx_udata))) + H5E_THROW(H5E_CANTINIT, "can't create fixed array header") + + /* Allocate and initialize new fixed array wrapper */ + if(NULL == (fa = H5FA__new(f, dxpl_id, fa_addr, FALSE, ctx_udata))) + H5E_THROW(H5E_CANTINIT, "allocation and/or initialization failed for fixed array wrapper") + + /* Set the return value */ + ret_value = fa; + +CATCH + if(!ret_value) if(fa && H5FA_close(fa, dxpl_id) < 0) H5E_THROW(H5E_CLOSEERROR, "unable to close fixed array") @@ -194,7 +239,6 @@ H5FA_open(H5F_t *f, hid_t dxpl_id, haddr_t fa_addr, void *ctx_udata)) /* Local variables */ H5FA_t *fa = NULL; /* Pointer to new fixed array wrapper */ - H5FA_hdr_t *hdr = NULL; /* The fixed array header information */ /* * Check arguments. @@ -202,40 +246,15 @@ H5FA_open(H5F_t *f, hid_t dxpl_id, haddr_t fa_addr, void *ctx_udata)) HDassert(f); HDassert(H5F_addr_defined(fa_addr)); - /* Load the array header into memory */ -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: fa_addr = %a\n", FUNC, fa_addr); -#endif /* H5FA_DEBUG */ - if(NULL == (hdr = H5FA__hdr_protect(f, dxpl_id, fa_addr, ctx_udata, H5AC__READ_ONLY_FLAG))) - H5E_THROW(H5E_CANTPROTECT, "unable to load fixed array header, address = %llu", (unsigned long long)fa_addr) - - /* Check for pending array deletion */ - if(hdr->pending_delete) - H5E_THROW(H5E_CANTOPENOBJ, "can't open fixed array pending deletion") - - /* Create fixed array info */ - if(NULL == (fa = H5FL_MALLOC(H5FA_t))) - H5E_THROW(H5E_CANTALLOC, "memory allocation failed for fixed array info") - - /* Point fixed array wrapper at header */ - fa->hdr = hdr; - if(H5FA__hdr_incr(fa->hdr) < 0) - H5E_THROW(H5E_CANTINC, "can't increment reference count on shared array header") - - /* Increment # of files using this array header */ - if(H5FA__hdr_fuse_incr(fa->hdr) < 0) - H5E_THROW(H5E_CANTINC, "can't increment file reference count on shared array header") - - /* Set file pointer for this array open context */ - fa->f = f; + /* Allocate and initialize new fixed array wrapper */ + if(NULL == (fa = H5FA__new(f, dxpl_id, fa_addr, TRUE, ctx_udata))) + H5E_THROW(H5E_CANTINIT, "allocation and/or initialization failed for fixed array wrapper") /* Set the return value */ ret_value = fa; CATCH - if(hdr && H5FA__hdr_unprotect(hdr, dxpl_id, H5AC__NO_FLAGS_SET) < 0) - H5E_THROW(H5E_CANTUNPROTECT, "unable to release fixed array header") if(!ret_value) if(fa && H5FA_close(fa, dxpl_id) < 0) H5E_THROW(H5E_CLOSEERROR, "unable to close fixed array") @@ -261,10 +280,6 @@ H5FA_get_nelmts(const H5FA_t *fa, hsize_t *nelmts)) /* Local variables */ -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* H5FA_DEBUG */ - /* * Check arguments. */ @@ -295,10 +310,6 @@ H5FA_get_addr(const H5FA_t *fa, haddr_t *addr)) /* Local variables */ -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* H5FA_DEBUG */ - /* * Check arguments. */ @@ -336,11 +347,6 @@ H5FA_set(const H5FA_t *fa, hid_t dxpl_id, hsize_t idx, const void *elmt)) unsigned dblk_page_cache_flags = H5AC__NO_FLAGS_SET; /* Flags to unprotecting FIxed Array Data block page */ hbool_t hdr_dirty = FALSE; /* Whether header information changed */ -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: Called\n", FUNC); -HDfprintf(stderr, "%s: Index %Hu\n", FUNC, idx); -#endif /* H5FA_DEBUG */ - /* * Check arguments. */ @@ -352,9 +358,6 @@ HDfprintf(stderr, "%s: Index %Hu\n", FUNC, idx); /* Check if we need to create the fixed array data block */ if(!H5F_addr_defined(hdr->dblk_addr)) { -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: fixed array data block address not defined!\n", FUNC, idx); -#endif /* H5FA_DEBUG */ /* Create the data block */ hdr->dblk_addr = H5FA__dblock_create(hdr, dxpl_id, &hdr_dirty); if(!H5F_addr_defined(hdr->dblk_addr)) @@ -449,11 +452,6 @@ H5FA_get(const H5FA_t *fa, hid_t dxpl_id, hsize_t idx, void *elmt)) H5FA_dblock_t *dblock = NULL; /* Pointer to data block for FA */ H5FA_dblk_page_t *dblk_page = NULL; /* Pointer to data block page for FA */ -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: Called\n", FUNC); -HDfprintf(stderr, "%s: Index %Hu\n", FUNC, idx); -#endif /* H5FA_DEBUG */ - /* * Check arguments. */ @@ -550,35 +548,33 @@ H5FA_close(H5FA_t *fa, hid_t dxpl_id)) hbool_t pending_delete = FALSE; /* Whether the array is pending deletion */ haddr_t fa_addr = HADDR_UNDEF; /* Address of array (for deletion) */ -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* H5FA_DEBUG */ - /* * Check arguments. */ HDassert(fa); - /* Decrement file reference & check if this is the last open fixed array using the shared array header */ - if(0 == H5FA__hdr_fuse_decr(fa->hdr)) { - /* Set the shared array header's file context for this operation */ - fa->hdr->f = fa->f; - - /* Shut down anything that can't be put in the header's 'flush' callback */ - - /* Check for pending array deletion */ - if(fa->hdr->pending_delete) { - /* Set local info, so array deletion can occur after decrementing the - * header's ref count - */ - pending_delete = TRUE; - fa_addr = fa->hdr->addr; + /* Close the header, if it was set */ + if(fa->hdr) { + /* Decrement file reference & check if this is the last open fixed array using the shared array header */ + if(0 == H5FA__hdr_fuse_decr(fa->hdr)) { + /* Set the shared array header's file context for this operation */ + fa->hdr->f = fa->f; + + /* Shut down anything that can't be put in the header's 'flush' callback */ + + /* Check for pending array deletion */ + if(fa->hdr->pending_delete) { + /* Set local info, so array deletion can occur after decrementing the + * header's ref count + */ + pending_delete = TRUE; + fa_addr = fa->hdr->addr; + } /* end if */ } /* end if */ - } /* end if */ - /* Check for pending array deletion */ - if(pending_delete) { - H5FA_hdr_t *hdr; /* Another pointer to fixed array header */ + /* Check for pending array deletion */ + if(pending_delete) { + H5FA_hdr_t *hdr; /* Another pointer to fixed array header */ #ifndef NDEBUG { @@ -595,33 +591,34 @@ HDfprintf(stderr, "%s: Called\n", FUNC); } #endif /* NDEBUG */ - /* Lock the array header into memory */ - /* (OK to pass in NULL for callback context, since we know the header must be in the cache) */ - if(NULL == (hdr = H5FA__hdr_protect(fa->f, dxpl_id, fa_addr, NULL, H5AC__NO_FLAGS_SET))) - H5E_THROW(H5E_CANTLOAD, "unable to load fixed array header") + /* Lock the array header into memory */ + /* (OK to pass in NULL for callback context, since we know the header must be in the cache) */ + if(NULL == (hdr = H5FA__hdr_protect(fa->f, dxpl_id, fa_addr, NULL, H5AC__NO_FLAGS_SET))) + H5E_THROW(H5E_CANTLOAD, "unable to load fixed array header") - /* Set the shared array header's file context for this operation */ - hdr->f = fa->f; + /* Set the shared array header's file context for this operation */ + hdr->f = fa->f; - /* Decrement the reference count on the array header */ - /* (don't put in H5FA_hdr_fuse_decr() as the array header may be evicted - * immediately -QAK) - */ - if(H5FA__hdr_decr(fa->hdr) < 0) - H5E_THROW(H5E_CANTDEC, "can't decrement reference count on shared array header") + /* Decrement the reference count on the array header */ + /* (don't put in H5FA_hdr_fuse_decr() as the array header may be evicted + * immediately -QAK) + */ + if(H5FA__hdr_decr(fa->hdr) < 0) + H5E_THROW(H5E_CANTDEC, "can't decrement reference count on shared array header") - /* Delete array, starting with header (unprotects header) */ - if(H5FA__hdr_delete(hdr, dxpl_id) < 0) - H5E_THROW(H5E_CANTDELETE, "unable to delete fixed array") + /* Delete array, starting with header (unprotects header) */ + if(H5FA__hdr_delete(hdr, dxpl_id) < 0) + H5E_THROW(H5E_CANTDELETE, "unable to delete fixed array") + } /* end if */ + else { + /* Decrement the reference count on the array header */ + /* (don't put in H5FA_hdr_fuse_decr() as the array header may be evicted + * immediately -QAK) + */ + if(H5FA__hdr_decr(fa->hdr) < 0) + H5E_THROW(H5E_CANTDEC, "can't decrement reference count on shared array header") + } /* end else */ } /* end if */ - else { - /* Decrement the reference count on the array header */ - /* (don't put in H5FA_hdr_fuse_decr() as the array header may be evicted - * immediately -QAK) - */ - if(H5FA__hdr_decr(fa->hdr) < 0) - H5E_THROW(H5E_CANTDEC, "can't decrement reference count on shared array header") - } /* end else */ /* Release the fixed array wrapper */ fa = H5FL_FREE(H5FA_t, fa); @@ -657,9 +654,6 @@ H5FA_delete(H5F_t *f, hid_t dxpl_id, haddr_t fa_addr, void *ctx_udata)) HDassert(H5F_addr_defined(fa_addr)); /* Lock the array header into memory */ -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: fa_addr = %a\n", FUNC, fa_addr); -#endif /* H5FA_DEBUG */ if(NULL == (hdr = H5FA__hdr_protect(f, dxpl_id, fa_addr, ctx_udata, H5AC__NO_FLAGS_SET))) H5E_THROW(H5E_CANTPROTECT, "unable to protect fixed array header, address = %llu", (unsigned long long)fa_addr) @@ -743,6 +737,56 @@ END_FUNC(PRIV) /* end H5FA_iterate() */ /*------------------------------------------------------------------------- + * Function: H5FA_depend + * + * Purpose: Make a child flush dependency between the fixed array + * and another piece of metadata in the file. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Fall 2012 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(PRIV, ERR, +herr_t, SUCCEED, FAIL, +H5FA_depend(H5FA_t *fa, hid_t dxpl_id, H5AC_proxy_entry_t *parent)) + + /* Local variables */ + H5FA_hdr_t *hdr = fa->hdr; /* Header for FA */ + + /* + * Check arguments. + */ + HDassert(fa); + HDassert(hdr); + HDassert(parent); + + /* + * Check to see if a flush dependency between the fixed array + * and another data structure in the file has already been set up. + * If it hasn't, do so now. + */ + if(NULL == hdr->parent) { + /* Sanity check */ + HDassert(hdr->top_proxy); + + /* Set the shared array header's file context for this operation */ + hdr->f = fa->f; + + /* Add the fixed array as a child of the parent (proxy) */ + if(H5AC_proxy_entry_add_child(parent, hdr->f, dxpl_id, hdr->top_proxy) < 0) + H5E_THROW(H5E_CANTSET, "unable to add fixed array as child of proxy") + hdr->parent = parent; + } /* end if */ + +CATCH + +END_FUNC(PRIV) /* end H5FA_depend() */ + + +/*------------------------------------------------------------------------- * Function: H5FA_patch_file * * Purpose: Patch the top-level file pointer contained in fa @@ -760,10 +804,6 @@ H5FA_patch_file(H5FA_t *fa, H5F_t *f)) /* Local variables */ -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* H5FA_DEBUG */ - /* * Check arguments. */ diff --git a/src/H5FAcache.c b/src/H5FAcache.c index 540004a..11e8571 100644 --- a/src/H5FAcache.c +++ b/src/H5FAcache.c @@ -71,34 +71,35 @@ /********************/ /* Metadata cache (H5AC) callbacks */ -static herr_t H5FA__cache_hdr_get_load_size(const void *udata, size_t *image_len); +static herr_t H5FA__cache_hdr_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5FA__cache_hdr_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5FA__cache_hdr_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5FA__cache_hdr_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5FA__cache_hdr_image_len(const void *thing, size_t *image_len); static herr_t H5FA__cache_hdr_serialize(const H5F_t *f, void *image, size_t len, void *thing); +static herr_t H5FA__cache_hdr_notify(H5AC_notify_action_t action, void *thing); static herr_t H5FA__cache_hdr_free_icr(void *thing); -static herr_t H5FA__cache_dblock_get_load_size(const void *udata, size_t *image_len); +static herr_t H5FA__cache_dblock_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5FA__cache_dblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5FA__cache_dblock_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5FA__cache_dblock_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5FA__cache_dblock_image_len(const void *thing, size_t *image_len); static herr_t H5FA__cache_dblock_serialize(const H5F_t *f, void *image, size_t len, void *thing); +static herr_t H5FA__cache_dblock_notify(H5AC_notify_action_t action, void *thing); static herr_t H5FA__cache_dblock_free_icr(void *thing); static herr_t H5FA__cache_dblock_fsf_size(const void *thing, size_t *fsf_size); -static herr_t H5FA__cache_dblk_page_get_load_size(const void *udata, size_t *image_len); +static herr_t H5FA__cache_dblk_page_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5FA__cache_dblk_page_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5FA__cache_dblk_page_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5FA__cache_dblk_page_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5FA__cache_dblk_page_image_len(const void *thing, size_t *image_len); static herr_t H5FA__cache_dblk_page_serialize(const H5F_t *f, void *image, size_t len, void *thing); +static herr_t H5FA__cache_dblk_page_notify(H5AC_notify_action_t action, void *thing); static herr_t H5FA__cache_dblk_page_free_icr(void *thing); @@ -112,14 +113,15 @@ const H5AC_class_t H5AC_FARRAY_HDR[1] = {{ "Fixed-array Header", /* Metadata client name (for debugging) */ H5FD_MEM_FARRAY_HDR, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5FA__cache_hdr_get_load_size, /* 'get_load_size' callback */ + H5FA__cache_hdr_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5FA__cache_hdr_verify_chksum, /* 'verify_chksum' callback */ H5FA__cache_hdr_deserialize, /* 'deserialize' callback */ H5FA__cache_hdr_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5FA__cache_hdr_serialize, /* 'serialize' callback */ - NULL, /* 'notify' callback */ + H5FA__cache_hdr_notify, /* 'notify' callback */ H5FA__cache_hdr_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -129,14 +131,15 @@ const H5AC_class_t H5AC_FARRAY_DBLOCK[1] = {{ "Fixed Array Data Block", /* Metadata client name (for debugging) */ H5FD_MEM_FARRAY_DBLOCK, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5FA__cache_dblock_get_load_size, /* 'get_load_size' callback */ + H5FA__cache_dblock_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5FA__cache_dblock_verify_chksum, /* 'verify_chksum' callback */ H5FA__cache_dblock_deserialize, /* 'deserialize' callback */ H5FA__cache_dblock_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5FA__cache_dblock_serialize, /* 'serialize' callback */ - NULL, /* 'notify' callback */ + H5FA__cache_dblock_notify, /* 'notify' callback */ H5FA__cache_dblock_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ H5FA__cache_dblock_fsf_size, /* 'fsf_size' callback */ }}; @@ -146,14 +149,15 @@ const H5AC_class_t H5AC_FARRAY_DBLK_PAGE[1] = {{ "Fixed Array Data Block Page", /* Metadata client name (for debugging) */ H5FD_MEM_FARRAY_DBLK_PAGE, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5FA__cache_dblk_page_get_load_size, /* 'get_load_size' callback */ + H5FA__cache_dblk_page_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5FA__cache_dblk_page_verify_chksum, /* 'verify_chksum' callback */ H5FA__cache_dblk_page_deserialize, /* 'deserialize' callback */ H5FA__cache_dblk_page_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5FA__cache_dblk_page_serialize, /* 'serialize' callback */ - NULL, /* 'notify' callback */ + H5FA__cache_dblk_page_notify, /* 'notify' callback */ H5FA__cache_dblk_page_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -170,7 +174,7 @@ const H5AC_class_t H5AC_FARRAY_DBLK_PAGE[1] = {{ /*------------------------------------------------------------------------- - * Function: H5FA__cache_hdr_get_load_size + * Function: H5FA__cache_hdr_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -184,10 +188,10 @@ const H5AC_class_t H5AC_FARRAY_DBLK_PAGE[1] = {{ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5FA__cache_hdr_get_load_size(const void *_udata, size_t *image_len)) +H5FA__cache_hdr_get_initial_load_size(void *_udata, size_t *image_len)) /* Local variables */ - const H5FA_hdr_cache_ud_t *udata = (const H5FA_hdr_cache_ud_t *)_udata; /* User data for callback */ + H5FA_hdr_cache_ud_t *udata = (H5FA_hdr_cache_ud_t *)_udata; /* User data for callback */ /* Check arguments */ HDassert(udata); @@ -197,7 +201,41 @@ H5FA__cache_hdr_get_load_size(const void *_udata, size_t *image_len)) /* Set the image length size */ *image_len = (size_t)H5FA_HEADER_SIZE_FILE(udata->f); -END_FUNC(STATIC) /* end H5FA__cache_hdr_get_load_size() */ +END_FUNC(STATIC) /* end H5FA__cache_hdr_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5FA__cache_hdr_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, NOERR, +htri_t, TRUE, -, +H5FA__cache_hdr_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)) + + /* Local variables */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + +END_FUNC(STATIC) /* end H5FA__cache_hdr_verify_chksum() */ /*------------------------------------------------------------------------- @@ -225,7 +263,6 @@ H5FA__cache_hdr_deserialize(const void *_image, size_t len, H5FA_hdr_cache_ud_t *udata = (H5FA_hdr_cache_ud_t *)_udata; const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ /* Check arguments */ HDassert(udata); @@ -289,9 +326,7 @@ H5FA__cache_hdr_deserialize(const void *_image, size_t len, /* (allow for checksum not decoded yet) */ HDassert((size_t)(image - (const uint8_t *)_image) == (len - H5FA_SIZEOF_CHKSUM)); - /* Compute checksum on entire header */ - /* (including the filter information, if present) */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -299,10 +334,6 @@ H5FA__cache_hdr_deserialize(const void *_image, size_t len, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == len); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for fixed array header") - /* Finish initializing fixed array header */ if(H5FA__hdr_init(hdr, udata->ctx_udata) < 0) H5E_THROW(H5E_CANTINIT, "initialization failed for fixed array header") @@ -336,8 +367,7 @@ END_FUNC(STATIC) /* end H5FA__cache_hdr_deserialize() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5FA__cache_hdr_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr)) +H5FA__cache_hdr_image_len(const void *_thing, size_t *image_len)) /* Local variables */ const H5FA_hdr_t *hdr = (const H5FA_hdr_t *)_thing; /* Pointer to the object */ @@ -413,6 +443,80 @@ END_FUNC(STATIC) /* end H5FA__cache_hdr_serialize() */ /*------------------------------------------------------------------------- + * Function: H5FA__cache_hdr_notify + * + * Purpose: Handle cache action notifications + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Dana Robinson + * December 2015 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, ERR, +herr_t, SUCCEED, FAIL, +H5FA__cache_hdr_notify(H5AC_notify_action_t action, void *_thing)) + + /* Local variables */ + H5FA_hdr_t *hdr = (H5FA_hdr_t *)_thing; /* Pointer to the object */ + + /* Sanity check */ + HDassert(hdr); + + /* Check if the file was opened with SWMR-write access */ + if(hdr->swmr_write) { + /* Determine which action to take */ + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + case H5AC_NOTIFY_ACTION_AFTER_LOAD: + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + /* If hdr->parent != NULL, hdr->parent is used to destroy + * the flush dependency before the header is evicted. + */ + if(hdr->parent) { + /* Sanity check */ + HDassert(hdr->top_proxy); + + /* Destroy flush dependency on object header proxy */ + if(H5AC_proxy_entry_remove_child((H5AC_proxy_entry_t *)hdr->parent, (void *)hdr->top_proxy) < 0) + H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between fixed array and proxy") + hdr->parent = NULL; + } /* end if */ + + /* Detach from 'top' proxy for fixed array */ + if(hdr->top_proxy) { + if(H5AC_proxy_entry_remove_child(hdr->top_proxy, hdr) < 0) + H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between header and fixed array 'top' proxy") + /* Don't reset hdr->top_proxy here, it's destroyed when the header is freed -QAK */ + } /* end if */ + break; + + default: +#ifdef NDEBUG + H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Unknown action?!?"); +#endif /* NDEBUG */ + } /* end switch */ + } /* end if */ + else + HDassert(NULL == hdr->parent); + +CATCH + +END_FUNC(STATIC) /* end H5FA__cache_hdr_notify() */ + + +/*------------------------------------------------------------------------- * Function: H5FA__cache_hdr_free_icr * * Purpose: Destroy/release an "in core representation" of a data @@ -443,7 +547,7 @@ END_FUNC(STATIC) /* end H5FA__cache_hdr_free_icr() */ /*------------------------------------------------------------------------- - * Function: H5FA__cache_dblock_get_load_size + * Function: H5FA__cache_dblock_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -457,7 +561,7 @@ END_FUNC(STATIC) /* end H5FA__cache_hdr_free_icr() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5FA__cache_dblock_get_load_size(const void *_udata, size_t *image_len)) +H5FA__cache_dblock_get_initial_load_size(void *_udata, size_t *image_len)) /* Local variables */ H5FA_dblock_cache_ud_t *udata = (H5FA_dblock_cache_ud_t *)_udata; /* User data */ @@ -479,7 +583,6 @@ H5FA__cache_dblock_get_load_size(const void *_udata, size_t *image_len)) * dblock->npages * dblock->dblk_page_init_size */ - dblock.hdr = udata->hdr; dblk_page_nelmts = (size_t)1 << udata->hdr->cparam.max_dblk_page_nelmts_bits; if(udata->hdr->cparam.nelmts > dblk_page_nelmts) { @@ -493,7 +596,41 @@ H5FA__cache_dblock_get_load_size(const void *_udata, size_t *image_len)) else *image_len = (size_t)H5FA_DBLOCK_PREFIX_SIZE(&dblock); -END_FUNC(STATIC) /* end H5FA__cache_dblock_get_load_size() */ +END_FUNC(STATIC) /* end H5FA__cache_dblock_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5FA__cache_dblock_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, NOERR, +htri_t, TRUE, -, +H5FA__cache_dblock_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)) + + /* Local variables */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + +END_FUNC(STATIC) /* end H5FA__cache_dblock_verify_chksum() */ /*------------------------------------------------------------------------- @@ -520,7 +657,6 @@ H5FA__cache_dblock_deserialize(const void *_image, size_t len, H5FA_dblock_cache_ud_t *udata = (H5FA_dblock_cache_ud_t *)_udata; /* User data for loading data block */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ haddr_t arr_addr; /* Address of array header in the file */ /* Sanity check */ @@ -577,8 +713,7 @@ H5FA__cache_dblock_deserialize(const void *_image, size_t len, /* Set the data block's size */ dblock->size = H5FA_DBLOCK_SIZE(dblock); - /* Compute checksum on data block */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -586,10 +721,6 @@ H5FA__cache_dblock_deserialize(const void *_image, size_t len, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == len); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for fixed array data block") - /* Set return value */ ret_value = dblock; @@ -618,8 +749,7 @@ END_FUNC(STATIC) /* end H5FA__cache_dblock_deserialize() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5FA__cache_dblock_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr)) +H5FA__cache_dblock_image_len(const void *_thing, size_t *image_len)) /* Local variables */ const H5FA_dblock_t *dblock = (const H5FA_dblock_t *)_thing; /* Pointer to the object */ @@ -712,6 +842,75 @@ END_FUNC(STATIC) /* end H5FA__cache_dblock_serialize() */ /*------------------------------------------------------------------------- + * Function: H5FA__cache_dblock_notify + * + * Purpose: Handle cache action notifications + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Fall 2012 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, ERR, +herr_t, SUCCEED, FAIL, +H5FA__cache_dblock_notify(H5AC_notify_action_t action, void *_thing)) + + /* Local variables */ + H5FA_dblock_t *dblock = (H5FA_dblock_t *)_thing; + + /* Sanity check */ + HDassert(dblock); + + /* Check if the file was opened with SWMR-write access */ + if(dblock->hdr->swmr_write) { + /* Determine which action to take */ + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + case H5AC_NOTIFY_ACTION_AFTER_LOAD: + /* Create flush dependency on parent */ + if(H5FA__create_flush_depend((H5AC_info_t *)dblock->hdr, (H5AC_info_t *)dblock) < 0) + H5E_THROW(H5E_CANTDEPEND, "unable to create flush dependency between data block and header, address = %llu", (unsigned long long)dblock->addr) + break; + + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + /* Destroy flush dependency on parent */ + if(H5FA__destroy_flush_depend((H5AC_info_t *)dblock->hdr, (H5AC_info_t *)dblock) < 0) + H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency") + + /* Detach from 'top' proxy for fixed array */ + if(dblock->top_proxy) { + if(H5AC_proxy_entry_remove_child(dblock->top_proxy, dblock) < 0) + H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between data block and fixed array 'top' proxy") + dblock->top_proxy = NULL; + } /* end if */ + break; + + default: +#ifdef NDEBUG + H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Unknown action?!?"); +#endif /* NDEBUG */ + } /* end switch */ + } /* end if */ + +CATCH + +END_FUNC(STATIC) /* end H5FA__cache_dblock_notify() */ + + + +/*------------------------------------------------------------------------- * Function: H5FA__cache_dblock_free_icr * * Purpose: Destroy/release an "in core representation" of a data @@ -790,7 +989,7 @@ END_FUNC(STATIC) /* end H5FA__cache_dblock_fsf_size() */ /*------------------------------------------------------------------------- - * Function: H5FA__cache_dblk_page_get_load_size + * Function: H5FA__cache_dblk_page_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -804,10 +1003,10 @@ END_FUNC(STATIC) /* end H5FA__cache_dblock_fsf_size() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5FA__cache_dblk_page_get_load_size(const void *_udata, size_t *image_len)) +H5FA__cache_dblk_page_get_initial_load_size(void *_udata, size_t *image_len)) /* Local variables */ - const H5FA_dblk_page_cache_ud_t *udata = (const H5FA_dblk_page_cache_ud_t *)_udata; /* User data */ + H5FA_dblk_page_cache_ud_t *udata = (H5FA_dblk_page_cache_ud_t *)_udata; /* User data */ /* Check arguments */ HDassert(udata); @@ -815,9 +1014,44 @@ H5FA__cache_dblk_page_get_load_size(const void *_udata, size_t *image_len)) HDassert(udata->nelmts > 0); HDassert(image_len); + /* Set the image length size */ *image_len = (size_t)H5FA_DBLK_PAGE_SIZE(udata->hdr, udata->nelmts); -END_FUNC(STATIC) /* end H5FA__cache_dblk_page_get_load_size() */ +END_FUNC(STATIC) /* end H5FA__cache_dblk_page_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5FA__cache_dblk_page_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, NOERR, +htri_t, TRUE, -, +H5FA__cache_dblk_page_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)) + + /* Local variables */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + +END_FUNC(STATIC) /* end H5FA__cache_dblk_page_verify_chksum() */ /*------------------------------------------------------------------------- @@ -844,7 +1078,6 @@ H5FA__cache_dblk_page_deserialize(const void *_image, size_t len, H5FA_dblk_page_cache_ud_t *udata = (H5FA_dblk_page_cache_ud_t *)_udata; /* User data for loading data block page */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ /* Sanity check */ HDassert(udata); @@ -874,8 +1107,7 @@ H5FA__cache_dblk_page_deserialize(const void *_image, size_t len, /* Set the data block page's size */ dblk_page->size = len; - /* Compute checksum on data block */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -883,10 +1115,6 @@ H5FA__cache_dblk_page_deserialize(const void *_image, size_t len, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == dblk_page->size); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for fixed array data block page") - /* Set return value */ ret_value = dblk_page; @@ -915,8 +1143,7 @@ END_FUNC(STATIC) /* end H5FA__cache_dblk_page_deserialize() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5FA__cache_dblk_page_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr)) +H5FA__cache_dblk_page_image_len(const void *_thing, size_t *image_len)) /* Local variables */ const H5FA_dblk_page_t *dblk_page = (const H5FA_dblk_page_t *)_thing; /* Pointer to the object */ @@ -984,6 +1211,66 @@ END_FUNC(STATIC) /* end H5FA__cache_dblk_page_serialize() */ /*------------------------------------------------------------------------- + * Function: H5FA__cache_dblk_page_notify + * + * Purpose: Handle cache action notifications + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@lbl.gov + * Oct 17 2016 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, ERR, +herr_t, SUCCEED, FAIL, +H5FA__cache_dblk_page_notify(H5AC_notify_action_t action, void *_thing)) + + /* Local variables */ + H5FA_dblk_page_t *dblk_page = (H5FA_dblk_page_t *)_thing; /* Pointer to the object */ + + /* Sanity check */ + HDassert(dblk_page); + + /* Determine which action to take */ + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + case H5AC_NOTIFY_ACTION_AFTER_LOAD: + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + /* Detach from 'top' proxy for fixed array */ + if(dblk_page->top_proxy) { + if(H5AC_proxy_entry_remove_child(dblk_page->top_proxy, dblk_page) < 0) + H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between data block page and fixed array 'top' proxy") + dblk_page->top_proxy = NULL; + } /* end if */ + break; + + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + /* do nothing */ + break; + + default: +#ifdef NDEBUG + H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Unknown action?!?"); +#endif /* NDEBUG */ + } /* end switch */ + +CATCH + +END_FUNC(STATIC) /* end H5FA__cache_dblk_page_notify() */ + + +/*------------------------------------------------------------------------- * Function: H5FA__cache_dblk_page_free_icr * * Purpose: Destroy/release an "in core representation" of a data diff --git a/src/H5FAdblkpage.c b/src/H5FAdblkpage.c index bba4439..09278f8 100644 --- a/src/H5FAdblkpage.c +++ b/src/H5FAdblkpage.c @@ -152,7 +152,8 @@ herr_t, SUCCEED, FAIL, H5FA__dblk_page_create(H5FA_hdr_t *hdr, hid_t dxpl_id, haddr_t addr, size_t nelmts)) /* Local variables */ - H5FA_dblk_page_t *dblk_page = NULL; /* Fixed array data block page */ + H5FA_dblk_page_t *dblk_page = NULL; /* Fixed array data block page */ + hbool_t inserted = FALSE; /* Whether the header was inserted into cache */ #ifdef H5FA_DEBUG HDfprintf(stderr, "%s: Called, addr = %a\n", FUNC, addr); @@ -179,10 +180,23 @@ HDfprintf(stderr, "%s: dblk_page->size = %Zu\n", FUNC, dblk_page->size); /* Cache the new fixed array data block page */ if(H5AC_insert_entry(hdr->f, dxpl_id, H5AC_FARRAY_DBLK_PAGE, dblk_page->addr, dblk_page, H5AC__NO_FLAGS_SET) < 0) H5E_THROW(H5E_CANTINSERT, "can't add fixed array data block page to cache") + inserted = TRUE; + + /* Add data block page as child of 'top' proxy */ + if(hdr->top_proxy) { + if(H5AC_proxy_entry_add_child(hdr->top_proxy, hdr->f, dxpl_id, dblk_page) < 0) + H5E_THROW(H5E_CANTSET, "unable to add fixed array entry as child of array proxy") + dblk_page->top_proxy = hdr->top_proxy; + } /* end if */ CATCH if(ret_value < 0) if(dblk_page) { + /* Remove from cache, if inserted */ + if(inserted) + if(H5AC_remove_entry(dblk_page) < 0) + H5E_THROW(H5E_CANTREMOVE, "unable to remove fixed array data block page from cache") + /* Destroy data block page */ if(H5FA__dblk_page_dest(dblk_page) < 0) H5E_THROW(H5E_CANTFREE, "unable to destroy fixed array data block page") @@ -210,7 +224,8 @@ H5FA__dblk_page_protect(H5FA_hdr_t *hdr, hid_t dxpl_id, haddr_t dblk_page_addr, size_t dblk_page_nelmts, unsigned flags)) /* Local variables */ - H5FA_dblk_page_cache_ud_t udata; /* Information needed for loading data block page */ + H5FA_dblk_page_t *dblk_page = NULL; /* Fixed array data block page */ + H5FA_dblk_page_cache_ud_t udata; /* Information needed for loading data block page */ #ifdef H5FA_DEBUG HDfprintf(stderr, "%s: Called\n", FUNC); @@ -229,11 +244,29 @@ HDfprintf(stderr, "%s: Called\n", FUNC); udata.dblk_page_addr = dblk_page_addr; /* Protect the data block page */ - if(NULL == (ret_value = (H5FA_dblk_page_t *)H5AC_protect(hdr->f, dxpl_id, H5AC_FARRAY_DBLK_PAGE, dblk_page_addr, &udata, flags))) + if(NULL == (dblk_page = (H5FA_dblk_page_t *)H5AC_protect(hdr->f, dxpl_id, H5AC_FARRAY_DBLK_PAGE, dblk_page_addr, &udata, flags))) H5E_THROW(H5E_CANTPROTECT, "unable to protect fixed array data block page, address = %llu", (unsigned long long)dblk_page_addr) + /* Create top proxy, if it doesn't exist */ + if(hdr->top_proxy && NULL == dblk_page->top_proxy) { + /* Add data block page as child of 'top' proxy */ + if(H5AC_proxy_entry_add_child(hdr->top_proxy, hdr->f, dxpl_id, dblk_page) < 0) + H5E_THROW(H5E_CANTSET, "unable to add fixed array entry as child of array proxy") + dblk_page->top_proxy = hdr->top_proxy; + } /* end if */ + + /* Set return value */ + ret_value = dblk_page; + CATCH + /* Clean up on error */ + if(!ret_value) { + /* Release the data block page, if it was protected */ + if(dblk_page && H5AC_unprotect(hdr->f, dxpl_id, H5AC_FARRAY_DBLK_PAGE, dblk_page->addr, dblk_page, H5AC__NO_FLAGS_SET) < 0) + H5E_THROW(H5E_CANTUNPROTECT, "unable to unprotect fixed array data block page, address = %llu", (unsigned long long)dblk_page->addr) + } /* end if */ + END_FUNC(PKG) /* end H5FA__dblk_page_protect() */ @@ -306,6 +339,9 @@ H5FA__dblk_page_dest(H5FA_dblk_page_t *dblk_page)) dblk_page->hdr = NULL; } /* end if */ + /* Sanity check */ + HDassert(NULL == dblk_page->top_proxy); + /* Free the data block page itself */ dblk_page = H5FL_FREE(H5FA_dblk_page_t, dblk_page); diff --git a/src/H5FAdblock.c b/src/H5FAdblock.c index 95419f8..440447d 100644 --- a/src/H5FAdblock.c +++ b/src/H5FAdblock.c @@ -189,12 +189,9 @@ haddr_t, HADDR_UNDEF, HADDR_UNDEF, H5FA__dblock_create(H5FA_hdr_t *hdr, hid_t dxpl_id, hbool_t *hdr_dirty)) /* Local variables */ - H5FA_dblock_t *dblock = NULL; /* fixed array data block */ - haddr_t dblock_addr; /* fixed array data block address */ - -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: Called, hdr->stats.nelmts = %Zu, nelmts = %Zu\n", FUNC, hdr->stats.nelmts, hdr->cparam.nelmts); -#endif /* H5FA_DEBUG */ + H5FA_dblock_t *dblock = NULL; /* Fixed array data block */ + haddr_t dblock_addr; /* Fixed array data block address */ + hbool_t inserted = FALSE; /* Whether the header was inserted into cache */ /* Sanity check */ HDassert(hdr); @@ -206,10 +203,6 @@ HDfprintf(stderr, "%s: Called, hdr->stats.nelmts = %Zu, nelmts = %Zu\n", FUNC, h /* Set size of data block on disk */ hdr->stats.dblk_size = dblock->size = H5FA_DBLOCK_SIZE(dblock); -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: dblock->size = %Zu\n", FUNC, dblock->size); -#endif /* H5FA_DEBUG */ - /* Allocate space for the data block on disk */ if(HADDR_UNDEF == (dblock_addr = H5MF_alloc(hdr->f, H5FD_MEM_FARRAY_DBLOCK, dxpl_id, (hsize_t)dblock->size))) @@ -225,6 +218,14 @@ HDfprintf(stderr, "%s: dblock->size = %Zu\n", FUNC, dblock->size); /* Cache the new fixed array data block */ if(H5AC_insert_entry(hdr->f, dxpl_id, H5AC_FARRAY_DBLOCK, dblock_addr, dblock, H5AC__NO_FLAGS_SET) < 0) H5E_THROW(H5E_CANTINSERT, "can't add fixed array data block to cache") + inserted = TRUE; + + /* Add data block as child of 'top' proxy */ + if(hdr->top_proxy) { + if(H5AC_proxy_entry_add_child(hdr->top_proxy, hdr->f, dxpl_id, dblock) < 0) + H5E_THROW(H5E_CANTSET, "unable to add fixed array entry as child of array proxy") + dblock->top_proxy = hdr->top_proxy; + } /* end if */ /* Mark the header dirty (for updating statistics) */ *hdr_dirty = TRUE; @@ -236,6 +237,11 @@ CATCH if(!H5F_addr_defined(ret_value)) if(dblock) { + /* Remove from cache, if inserted */ + if(inserted) + if(H5AC_remove_entry(dblock) < 0) + H5E_THROW(H5E_CANTREMOVE, "unable to remove fixed array data block from cache") + /* Release data block's disk space */ if(H5F_addr_defined(dblock->addr) && H5MF_xfree(hdr->f, H5FD_MEM_FARRAY_DBLOCK, dxpl_id, dblock->addr, (hsize_t)dblock->size) < 0) H5E_THROW(H5E_CANTFREE, "unable to release fixed array data block") @@ -266,11 +272,8 @@ H5FA__dblock_protect(H5FA_hdr_t *hdr, hid_t dxpl_id, haddr_t dblk_addr, unsigned flags)) /* Local variables */ - H5FA_dblock_cache_ud_t udata; /* Information needed for loading data block */ - -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* H5FA_DEBUG */ + H5FA_dblock_t *dblock; /* Fixed array data block */ + H5FA_dblock_cache_ud_t udata; /* Information needed for loading data block */ /* Sanity check */ HDassert(hdr); @@ -284,11 +287,28 @@ HDfprintf(stderr, "%s: Called\n", FUNC); udata.dblk_addr = dblk_addr; /* Protect the data block */ - if(NULL == (ret_value = (H5FA_dblock_t *)H5AC_protect(hdr->f, dxpl_id, H5AC_FARRAY_DBLOCK, dblk_addr, &udata, flags))) + if(NULL == (dblock = (H5FA_dblock_t *)H5AC_protect(hdr->f, dxpl_id, H5AC_FARRAY_DBLOCK, dblk_addr, &udata, flags))) H5E_THROW(H5E_CANTPROTECT, "unable to protect fixed array data block, address = %llu", (unsigned long long)dblk_addr) + /* Create top proxy, if it doesn't exist */ + if(hdr->top_proxy && NULL == dblock->top_proxy) { + /* Add data block as child of 'top' proxy */ + if(H5AC_proxy_entry_add_child(hdr->top_proxy, hdr->f, dxpl_id, dblock) < 0) + H5E_THROW(H5E_CANTSET, "unable to add fixed array entry as child of array proxy") + dblock->top_proxy = hdr->top_proxy; + } /* end if */ + + /* Set return value */ + ret_value = dblock; + CATCH + /* Clean up on error */ + if(!ret_value) + /* Release the data block, if it was protected */ + if(dblock && H5AC_unprotect(hdr->f, dxpl_id, H5AC_FARRAY_DBLOCK, dblock->addr, dblock, H5AC__NO_FLAGS_SET) < 0) + H5E_THROW(H5E_CANTUNPROTECT, "unable to unprotect fixed array data block, address = %llu", (unsigned long long)dblock->addr) + END_FUNC(PKG) /* end H5FA__dblock_protect() */ @@ -310,10 +330,6 @@ H5FA__dblock_unprotect(H5FA_dblock_t *dblock, hid_t dxpl_id, unsigned cache_flag /* Local variables */ -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* H5FA_DEBUG */ - /* Sanity check */ HDassert(dblock); @@ -345,10 +361,6 @@ H5FA__dblock_delete(H5FA_hdr_t *hdr, hid_t dxpl_id, haddr_t dblk_addr)) /* Local variables */ H5FA_dblock_t *dblock = NULL; /* Pointer to data block */ -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* H5FA_DEBUG */ - /* Sanity check */ HDassert(hdr); HDassert(H5F_addr_defined(dblk_addr)); @@ -367,16 +379,10 @@ HDfprintf(stderr, "%s: Called\n", FUNC); /* Iterate over pages in data block */ for(u = 0; u < dblock->npages; u++) { -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: Expunging data block page from cache\n", FUNC); -#endif /* H5FA_DEBUG */ /* Evict the data block page from the metadata cache */ /* (OK to call if it doesn't exist in the cache) */ if(H5AC_expunge_entry(hdr->f, dxpl_id, H5AC_FARRAY_DBLK_PAGE, dblk_page_addr, H5AC__NO_FLAGS_SET) < 0) H5E_THROW(H5E_CANTEXPUNGE, "unable to remove array data block page from metadata cache") -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: Done expunging data block page from cache\n", FUNC); -#endif /* H5FA_DEBUG */ /* Advance to next page address */ dblk_page_addr += dblock->dblk_page_size; @@ -434,6 +440,9 @@ H5FA__dblock_dest(H5FA_dblock_t *dblock)) dblock->hdr = NULL; } /* end if */ + /* Sanity check */ + HDassert(NULL == dblock->top_proxy); + /* Free the data block itself */ dblock = H5FL_FREE(H5FA_dblock_t, dblock); diff --git a/src/H5FAhdr.c b/src/H5FAhdr.c index ac9a103..52b90d1 100644 --- a/src/H5FAhdr.c +++ b/src/H5FAhdr.c @@ -112,6 +112,7 @@ H5FA__hdr_alloc(H5F_t *f)) /* Set the internal parameters for the array */ hdr->f = f; + hdr->swmr_write = (H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) > 0; hdr->sizeof_addr = H5F_SIZEOF_ADDR(f); hdr->sizeof_size = H5F_SIZEOF_SIZE(f); @@ -119,6 +120,7 @@ H5FA__hdr_alloc(H5F_t *f)) ret_value = hdr; CATCH + if(!ret_value) if(hdr && H5FA__hdr_dest(hdr) < 0) H5E_THROW(H5E_CANTFREE, "unable to destroy fixed array header") @@ -183,10 +185,7 @@ H5FA__hdr_create(H5F_t *f, hid_t dxpl_id, const H5FA_create_t *cparam, /* Local variables */ H5FA_hdr_t *hdr = NULL; /* Fixed array header */ - -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* H5FA_DEBUG */ + hbool_t inserted = FALSE; /* Whether the header was inserted into cache */ /* Check arguments */ HDassert(f); @@ -221,16 +220,33 @@ HDfprintf(stderr, "%s: Called\n", FUNC); if(HADDR_UNDEF == (hdr->addr = H5MF_alloc(f, H5FD_MEM_FARRAY_HDR, dxpl_id, (hsize_t)hdr->size))) H5E_THROW(H5E_CANTALLOC, "file allocation failed for Fixed Array header") + /* Create 'top' proxy for extensible array entries */ + if(hdr->swmr_write) + if(NULL == (hdr->top_proxy = H5AC_proxy_entry_create())) + H5E_THROW(H5E_CANTCREATE, "can't create fixed array entry proxy") + /* Cache the new Fixed Array header */ if(H5AC_insert_entry(f, dxpl_id, H5AC_FARRAY_HDR, hdr->addr, hdr, H5AC__NO_FLAGS_SET) < 0) H5E_THROW(H5E_CANTINSERT, "can't add fixed array header to cache") + inserted = TRUE; + + /* Add header as child of 'top' proxy */ + if(hdr->top_proxy) + if(H5AC_proxy_entry_add_child(hdr->top_proxy, f, dxpl_id, hdr) < 0) + H5E_THROW(H5E_CANTSET, "unable to add fixed array entry as child of array proxy") /* Set address of array header to return */ ret_value = hdr->addr; CATCH + if(!H5F_addr_defined(ret_value)) if(hdr) { + /* Remove from cache, if inserted */ + if(inserted) + if(H5AC_remove_entry(hdr) < 0) + H5E_THROW(H5E_CANTREMOVE, "unable to remove fixed array header from cache") + /* Release header's disk space */ if(H5F_addr_defined(hdr->addr) && H5MF_xfree(f, H5FD_MEM_FARRAY_HDR, dxpl_id, hdr->addr, (hsize_t)hdr->size) < 0) H5E_THROW(H5E_CANTFREE, "unable to free Fixed Array header") @@ -411,6 +427,7 @@ H5FA__hdr_protect(H5F_t *f, hid_t dxpl_id, haddr_t fa_addr, void *ctx_udata, unsigned flags)) /* Local variables */ + H5FA_hdr_t *hdr; /* Fixed array header */ H5FA_hdr_cache_ud_t udata; /* User data for cache callbacks */ /* Sanity check */ @@ -426,9 +443,23 @@ H5FA__hdr_protect(H5F_t *f, hid_t dxpl_id, haddr_t fa_addr, void *ctx_udata, udata.ctx_udata = ctx_udata; /* Protect the header */ - if(NULL == (ret_value = (H5FA_hdr_t *)H5AC_protect(f, dxpl_id, H5AC_FARRAY_HDR, fa_addr, &udata, flags))) + if(NULL == (hdr = (H5FA_hdr_t *)H5AC_protect(f, dxpl_id, H5AC_FARRAY_HDR, fa_addr, &udata, flags))) H5E_THROW(H5E_CANTPROTECT, "unable to protect fixed array header, address = %llu", (unsigned long long)fa_addr) - ret_value->f = f; /* (Must be set again here, in case the header was already in the cache -QAK) */ + hdr->f = f; /* (Must be set again here, in case the header was already in the cache -QAK) */ + + /* Create top proxy, if it doesn't exist */ + if(hdr->swmr_write && NULL == hdr->top_proxy) { + /* Create 'top' proxy for fixed array entries */ + if(NULL == (hdr->top_proxy = H5AC_proxy_entry_create())) + H5E_THROW(H5E_CANTCREATE, "can't create fixed array entry proxy") + + /* Add header as child of 'top' proxy */ + if(H5AC_proxy_entry_add_child(hdr->top_proxy, f, dxpl_id, hdr) < 0) + H5E_THROW(H5E_CANTSET, "unable to add fixed array entry as child of array proxy") + } /* end if */ + + /* Set return value */ + ret_value = hdr; CATCH @@ -505,10 +536,6 @@ H5FA__hdr_delete(H5FA_hdr_t *hdr, hid_t dxpl_id)) /* Check for Fixed Array Data block */ if(H5F_addr_defined(hdr->dblk_addr)) { -#ifdef H5FA_DEBUG -HDfprintf(stderr, "%s: hdr->dblk_addr = %a\n", FUNC, hdr->dblk_addr); -#endif /* H5FA_DEBUG */ - /* Delete Fixed Array Data block */ if(H5FA__dblock_delete(hdr, dxpl_id, hdr->dblk_addr) < 0) H5E_THROW(H5E_CANTDELETE, "unable to delete fixed array data block") @@ -553,6 +580,13 @@ H5FA__hdr_dest(H5FA_hdr_t *hdr)) } /* end if */ hdr->cb_ctx = NULL; + /* Destroy the 'top' proxy */ + if(hdr->top_proxy) { + if(H5AC_proxy_entry_dest(hdr->top_proxy) < 0) + H5E_THROW(H5E_CANTRELEASE, "unable to destroy fixed array 'top' proxy") + hdr->top_proxy = NULL; + } /* end if */ + /* Free the shared info itself */ hdr = H5FL_FREE(H5FA_hdr_t, hdr); diff --git a/src/H5FAint.c b/src/H5FAint.c new file mode 100644 index 0000000..331227b --- /dev/null +++ b/src/H5FAint.c @@ -0,0 +1,139 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5FAint.c + * Fall 2012 + * Dana Robinson <derobins@hdfgroup.org> + * + * Purpose: Internal routines for fixed arrays. + * + *------------------------------------------------------------------------- + */ + +/**********************/ +/* Module Declaration */ +/**********************/ + +#include "H5FAmodule.h" /* This source code file is part of the H5FA module */ + + +/***********************/ +/* Other Packages Used */ +/***********************/ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error Handling */ +#include "H5FApkg.h" /* Fixed Arrays */ + + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + + + +/*------------------------------------------------------------------------- + * Function: H5FA__create_flush_depend + * + * Purpose: Create a flush dependency between two data structure components + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Fall 2012 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(PKG, ERR, +herr_t, SUCCEED, FAIL, +H5FA__create_flush_depend(H5AC_info_t *parent_entry, H5AC_info_t *child_entry)) + + /* Sanity check */ + HDassert(parent_entry); + HDassert(child_entry); + + /* Create a flush dependency between parent and child entry */ + if(H5AC_create_flush_dependency(parent_entry, child_entry) < 0) + H5E_THROW(H5E_CANTDEPEND, "unable to create flush dependency") + +CATCH + +END_FUNC(PKG) /* end H5FA__create_flush_depend() */ + + +/*------------------------------------------------------------------------- + * Function: H5FA__destroy_flush_depend + * + * Purpose: Destroy a flush dependency between two data structure components + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Fall 2012 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(PKG, ERR, +herr_t, SUCCEED, FAIL, +H5FA__destroy_flush_depend(H5AC_info_t *parent_entry, H5AC_info_t *child_entry)) + + /* Sanity check */ + HDassert(parent_entry); + HDassert(child_entry); + + /* Destroy a flush dependency between parent and child entry */ + if(H5AC_destroy_flush_dependency(parent_entry, child_entry) < 0) + H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency") + +CATCH + +END_FUNC(PKG) /* end H5FA__destroy_flush_depend() */ + diff --git a/src/H5FApkg.h b/src/H5FApkg.h index c29322a..ccef562 100644 --- a/src/H5FApkg.h +++ b/src/H5FApkg.h @@ -144,6 +144,28 @@ typedef struct H5FA_hdr_t { /* Client information (not stored) */ void *cb_ctx; /* Callback context */ + + /* SWMR / Flush dependency information (not stored) */ + hbool_t swmr_write; /* Flag indicating the file is opened with SWMR-write access */ + H5AC_proxy_entry_t *top_proxy; /* 'Top' proxy cache entry for all array entries */ + void *parent; /* Pointer to 'top' proxy flush dependency + * parent, if it exists, otherwise NULL. + * If the fixed array is being used + * to index a chunked dataset and the + * dataset metadata is modified by a + * SWMR writer, this field will be set + * equal to the object header proxy + * that is the flush dependency parent + * of the fixed array header. + * + * The field is used to avoid duplicate + * setups of the flush dependency + * relationship, and to allow the + * fixed array header to destroy + * the flush dependency on receipt of + * an eviction notification from the + * metadata cache. + */ } H5FA_hdr_t; /* The fixed array data block information */ @@ -158,6 +180,9 @@ typedef struct H5FA_dblock_t { /* Internal array information (not stored) */ H5FA_hdr_t *hdr; /* Shared array header info */ + /* SWMR / Flush dependency information (not stored) */ + H5AC_proxy_entry_t *top_proxy; /* 'Top' proxy cache entry for all array entries */ + /* Computed/cached values (not stored) */ haddr_t addr; /* Address of this data block on disk */ hsize_t size; /* Size of data block on disk */ @@ -181,6 +206,9 @@ typedef struct H5FA_dbk_page_t { /* Internal array information (not stored) */ H5FA_hdr_t *hdr; /* Shared array header info */ + /* SWMR / Flush dependency information (not stored) */ + H5AC_proxy_entry_t *top_proxy; /* 'Top' proxy cache entry for all array entries */ + /* Computed/cached values (not stored) */ haddr_t addr; /* Address of this data block page on disk */ size_t size; /* Size of data block page on disk */ @@ -241,6 +269,12 @@ H5_DLLVAR const H5FA_class_t *const H5FA_client_class_g[H5FA_NUM_CLS_ID]; /* Package Private Prototypes */ /******************************/ +/* Generic routines */ +H5_DLL herr_t H5FA__create_flush_depend(H5AC_info_t *parent_entry, + H5AC_info_t *child_entry); +H5_DLL herr_t H5FA__destroy_flush_depend(H5AC_info_t *parent_entry, + H5AC_info_t *child_entry); + /* Header routines */ H5_DLL H5FA_hdr_t *H5FA__hdr_alloc(H5F_t *f); H5_DLL herr_t H5FA__hdr_init(H5FA_hdr_t *hdr, void *ctx_udata); diff --git a/src/H5FAprivate.h b/src/H5FAprivate.h index 1e76468..612f3a2 100644 --- a/src/H5FAprivate.h +++ b/src/H5FAprivate.h @@ -32,6 +32,7 @@ #endif /* NOT_YET */ /* Private headers needed by this file */ +#include "H5ACprivate.h" /* Metadata cache */ #include "H5Fprivate.h" /* File access */ @@ -126,6 +127,7 @@ H5_DLL herr_t H5FA_get_nelmts(const H5FA_t *fa, hsize_t *nelmts); H5_DLL herr_t H5FA_get_addr(const H5FA_t *fa, haddr_t *addr); H5_DLL herr_t H5FA_set(const H5FA_t *fa, hid_t dxpl_id, hsize_t idx, const void *elmt); H5_DLL herr_t H5FA_get(const H5FA_t *fa, hid_t dxpl_id, hsize_t idx, void *elmt); +H5_DLL herr_t H5FA_depend(H5FA_t *fa, hid_t dxpl_id, H5AC_proxy_entry_t *parent); H5_DLL herr_t H5FA_iterate(H5FA_t *fa, hid_t dxpl_id, H5FA_operator_t op, void *udata); H5_DLL herr_t H5FA_close(H5FA_t *fa, hid_t dxpl_id); H5_DLL herr_t H5FA_delete(H5F_t *f, hid_t dxpl_id, haddr_t fa_addr, void *ctx_udata); @@ -811,6 +811,9 @@ H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) if(NULL == (file = (driver->open)(name, flags, fapl_id, maxaddr))) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "open failed") + /* Set the file access flags */ + file->access_flags = flags; + /* * Fill in public fields. We must increment the reference count on the * driver ID to prevent it from being freed while this file is open. @@ -1441,6 +1444,32 @@ H5FD_get_feature_flags(const H5FD_t *file, unsigned long *feature_flags) /*------------------------------------------------------------------------- + * Function: H5FD_set_feature_flags + * + * Purpose: Set the feature flags for the VFD + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Vailin Choi; Oct 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5FD_set_feature_flags(H5FD_t *file, unsigned long feature_flags) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(file); + + /* Set the file's feature flags */ + file->feature_flags = feature_flags; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD_set_feature_flags() */ + + +/*------------------------------------------------------------------------- * Function: H5FD_get_fs_type_map * * Purpose: Retrieve the free space type mapping for the VFD diff --git a/src/H5FDcore.c b/src/H5FDcore.c index f29e231..9090679 100644 --- a/src/H5FDcore.c +++ b/src/H5FDcore.c @@ -1479,23 +1479,31 @@ done: static herr_t H5FD_core_lock(H5FD_t *_file, hbool_t rw) { - H5FD_core_t *file = (H5FD_core_t*)_file; /* VFD file struct */ - int lock; /* The type of lock */ - herr_t ret_value = SUCCEED; /* Return value */ + H5FD_core_t *file = (H5FD_core_t*)_file; /* VFD file struct */ + int lock_flags; /* file locking flags */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT HDassert(file); + + /* Only set the lock if there is a file descriptor. If no file + * descriptor, this is a no-op. + */ if(file->fd >= 0) { - /* Determine the type of lock */ - lock = rw ? LOCK_EX : LOCK_SH; - - /* Place the lock with non-blocking */ - if(HDflock(file->fd, lock | LOCK_NB) < 0) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to flock file") - } - /* Otherwise a noop */ + /* Set exclusive or shared lock based on rw status */ + lock_flags = rw ? LOCK_EX : LOCK_SH; + + /* Place a non-blocking lock on the file */ + if(HDflock(file->fd, lock_flags | LOCK_NB) < 0) { + if(ENOSYS == errno) + HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + else + HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") + } /* end if */ + + } /* end if */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -1525,11 +1533,16 @@ H5FD_core_unlock(H5FD_t *_file) if(file->fd >= 0) { - if(HDflock(file->fd, LOCK_UN) < 0) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to flock (unlock) file") - } - /* Otherwise a noop */ + if(HDflock(file->fd, LOCK_UN) < 0) { + if(ENOSYS == errno) + HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + else + HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") + } /* end if */ + + } /* end if */ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_core_unlock() */ + diff --git a/src/H5FDfamily.c b/src/H5FDfamily.c index 7ec8751..f0d4dba 100644 --- a/src/H5FDfamily.c +++ b/src/H5FDfamily.c @@ -1339,33 +1339,37 @@ done: static herr_t H5FD_family_lock(H5FD_t *_file, hbool_t rw) { - H5FD_family_t *file = (H5FD_family_t *)_file; /* VFD file struct */ - unsigned u, i; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ + H5FD_family_t *file = (H5FD_family_t *)_file; /* VFD file struct */ + unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT /* Place the lock on all the member files */ - for(u = 0; u < file->nmembs; u++) { - if(file->memb[u]) { + for(u = 0; u < file->nmembs; u++) + if(file->memb[u]) if(H5FD_lock(file->memb[u], rw) < 0) - break; - } /* end if */ - } /* end for */ + break; - if(u < file->nmembs) { /* Try to unlock the member files done before */ - for(i = 0; i < u; i++) { - if(H5FD_unlock(file->memb[i]) < 0) - /* Push error, but keep going*/ - HDONE_ERROR(H5E_IO, H5E_CANTUNLOCK, FAIL, "unable to unlock member files") - } - HGOTO_ERROR(H5E_IO, H5E_CANTLOCK, FAIL, "unable to lock member files") + /* If one of the locks failed, try to unlock the locked member files + * in an attempt to return to a fully unlocked state. + */ + if(u < file->nmembs) { + unsigned v; /* Local index variable */ + + for(v = 0; v < v; v++) { + if(H5FD_unlock(file->memb[v]) < 0) + /* Push error, but keep going */ + HDONE_ERROR(H5E_IO, H5E_CANTUNLOCK, FAIL, "unable to unlock member files") + } /* end for */ + HGOTO_ERROR(H5E_IO, H5E_CANTLOCK, FAIL, "unable to lock member files") } /* end if */ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_family_lock() */ + /*------------------------------------------------------------------------- * Function: H5FD_family_unlock * @@ -1381,19 +1385,18 @@ static herr_t H5FD_family_unlock(H5FD_t *_file) { H5FD_family_t *file = (H5FD_family_t *)_file; /* VFD file struct */ - unsigned u; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ + unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT /* Remove the lock on the member files */ - for(u = 0; u < file->nmembs; u++) { - if(file->memb[u]) { + for(u = 0; u < file->nmembs; u++) + if(file->memb[u]) if(H5FD_unlock(file->memb[u]) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTUNLOCK, FAIL, "unable to unlock member files") - } /* end if */ - } /* end for */ + HGOTO_ERROR(H5E_IO, H5E_CANTUNLOCK, FAIL, "unable to unlock member files") done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_family_unlock() */ + diff --git a/src/H5FDint.c b/src/H5FDint.c index 128f30f..744c3d1 100644 --- a/src/H5FDint.c +++ b/src/H5FDint.c @@ -93,7 +93,11 @@ *------------------------------------------------------------------------- */ herr_t -H5FD_locate_signature(H5FD_t *file, const H5P_genplist_t *dxpl, haddr_t *sig_addr) +H5FD_locate_signature(H5FD_t *file, +#ifndef H5_DEBUG_BUILD +const +#endif /* H5_DEBUG_BUILD */ +H5P_genplist_t *dxpl, haddr_t *sig_addr) { haddr_t addr, eoa, eof; uint8_t buf[H5F_SIGNATURE_LEN]; @@ -158,7 +162,11 @@ done: *------------------------------------------------------------------------- */ herr_t -H5FD_read(H5FD_t *file, const H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t addr, +H5FD_read(H5FD_t *file, +#ifndef H5_DEBUG_BUILD +const +#endif /* H5_DEBUG_BUILD */ +H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/) { haddr_t eoa = HADDR_UNDEF; @@ -199,9 +207,16 @@ H5FD_read(H5FD_t *file, const H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t add if(HADDR_UNDEF == (eoa = (file->cls->get_eoa)(file, type))) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver get_eoa request failed") - if((addr + file->base_addr + size) > eoa) - HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, addr = %llu, size=%llu, eoa=%llu", - (unsigned long long)(addr+ file->base_addr), (unsigned long long)size, (unsigned long long)eoa) + + /* + * If the file is open for SWMR read access, allow access to data past + * the end of the allocated space (the 'eoa'). This is done because the + * eoa stored in the file's superblock might be out of sync with the + * objects being written within the file by the application performing + * SWMR write operations. + */ + if(!(file->access_flags & H5F_ACC_SWMR_READ) && ((addr + file->base_addr + size) > eoa)) + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, addr = %llu, size = %llu, eoa = %llu", (unsigned long long)(addr + file->base_addr), (unsigned long long)size, (unsigned long long)eoa) /* Dispatch to driver */ if((file->cls->read)(file, type, H5P_PLIST_ID(dxpl), addr + file->base_addr, size, buf) < 0) @@ -226,7 +241,11 @@ done: *------------------------------------------------------------------------- */ herr_t -H5FD_write(H5FD_t *file, const H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t addr, +H5FD_write(H5FD_t *file, +#ifndef H5_DEBUG_BUILD +const +#endif /* H5_DEBUG_BUILD */ +H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf) { haddr_t eoa = HADDR_UNDEF; diff --git a/src/H5FDlog.c b/src/H5FDlog.c index d0c4647..03228d2 100644 --- a/src/H5FDlog.c +++ b/src/H5FDlog.c @@ -132,6 +132,7 @@ typedef struct H5FD_log_t { double total_read_time; /* Total time spent in read operations */ double total_write_time; /* Total time spent in write operations */ double total_seek_time; /* Total time spent in seek operations */ + double total_truncate_time; /* Total time spent in truncate operations */ size_t iosize; /* Size of I/O information buffers */ FILE *logfp; /* Log file pointer */ H5FD_log_fapl_t fa; /* Driver-specific file access properties */ @@ -170,6 +171,8 @@ static herr_t H5FD_log_close(H5FD_t *_file); static int H5FD_log_cmp(const H5FD_t *_f1, const H5FD_t *_f2); static herr_t H5FD_log_query(const H5FD_t *_f1, unsigned long *flags); static haddr_t H5FD_log_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size); +static herr_t H5FD__log_free(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, + hsize_t size); static haddr_t H5FD_log_get_eoa(const H5FD_t *_file, H5FD_mem_t type); static herr_t H5FD_log_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); static haddr_t H5FD_log_get_eof(const H5FD_t *_file, H5FD_mem_t type); @@ -203,7 +206,7 @@ static const H5FD_class_t H5FD_log_g = { H5FD_log_query, /*query */ NULL, /*get_type_map */ H5FD_log_alloc, /*alloc */ - NULL, /*free */ + H5FD__log_free, /*free */ H5FD_log_get_eoa, /*get_eoa */ H5FD_log_set_eoa, /*set_eoa */ H5FD_log_get_eof, /*get_eof */ @@ -735,6 +738,8 @@ H5FD_log_close(H5FD_t *_file) HDfprintf(file->logfp, "Total time in write operations: %f s\n", file->total_write_time); if(file->fa.flags & H5FD_LOG_TIME_SEEK) HDfprintf(file->logfp, "Total time in seek operations: %f s\n", file->total_seek_time); + if(file->fa.flags & H5FD_LOG_TIME_TRUNCATE) + HDfprintf(file->logfp, "Total time in truncate operations: %f s\n", file->total_truncate_time); /* Dump the write I/O information */ if(file->fa.flags & H5FD_LOG_FILE_WRITE) { @@ -893,6 +898,7 @@ H5FD_log_query(const H5FD_t *_file, unsigned long *flags /* out */) *flags |= H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */ *flags |= H5FD_FEAT_AGGREGATE_SMALLDATA; /* OK to aggregate "small" raw data allocations */ *flags |= H5FD_FEAT_POSIX_COMPAT_HANDLE; /* VFD handle is POSIX I/O call compatible */ + *flags |= H5FD_FEAT_SUPPORTS_SWMR_IO; /* VFD supports the single-writer/multiple-readers (SWMR) pattern */ /* Check for flags that are set by h5repart */ if(file && file->fam_to_sec2) @@ -957,6 +963,43 @@ H5FD_log_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hsi /*------------------------------------------------------------------------- + * Function: H5FD__log_free + * + * Purpose: Release file memory. + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * Wednesday, September 28, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__log_free(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, + haddr_t addr, hsize_t size) +{ + H5FD_log_t *file = (H5FD_log_t *)_file; + + FUNC_ENTER_STATIC_NOERR + + if(file->fa.flags != 0) { + /* Reset the flavor of the information in the file */ + if(file->fa.flags & H5FD_LOG_FLAVOR) { + HDassert(addr < file->iosize); + H5_CHECK_OVERFLOW(size, hsize_t, size_t); + HDmemset(&file->flavor[addr], H5FD_MEM_DEFAULT, (size_t)size); + } /* end if */ + + /* Log the file memory freed */ + if(file->fa.flags & H5FD_LOG_FREE) + HDfprintf(file->logfp, "%10a-%10a (%10Hu bytes) (%s) Freed\n", addr, (addr + size) - 1, size, flavors[type]); + } /* end if */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD__log_free() */ + + +/*------------------------------------------------------------------------- * Function: H5FD_log_get_eoa * * Purpose: Gets the end-of-address marker for the file. The EOA marker @@ -1004,6 +1047,7 @@ H5FD_log_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr) FUNC_ENTER_NOAPI_NOINIT_NOERR if(file->fa.flags != 0) { + /* Check for increasing file size */ if(H5F_addr_gt(addr, file->eoa) && H5F_addr_gt(addr, 0)) { hsize_t size = addr - file->eoa; @@ -1018,6 +1062,22 @@ H5FD_log_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr) if(file->fa.flags & H5FD_LOG_ALLOC) HDfprintf(file->logfp, "%10a-%10a (%10Hu bytes) (%s) Allocated\n", file->eoa, addr, size, flavors[type]); } /* end if */ + + /* Check for decreasing file size */ + if(H5F_addr_lt(addr, file->eoa) && H5F_addr_gt(addr, 0)) { + hsize_t size = file->eoa - addr; + + /* Reset the flavor of the space freed by the shrink */ + if(file->fa.flags & H5FD_LOG_FLAVOR) { + HDassert((addr + size) < file->iosize); + H5_CHECK_OVERFLOW(size, hsize_t, size_t); + HDmemset(&file->flavor[addr], H5FD_MEM_DEFAULT, (size_t)size); + } /* end if */ + + /* Log the shrink like a free */ + if(file->fa.flags & H5FD_LOG_FREE) + HDfprintf(file->logfp, "%10a-%10a (%10Hu bytes) (%s) Freed\n", file->eoa, addr, size, flavors[type]); + } /* end if */ } /* end if */ file->eoa = addr; @@ -1167,7 +1227,7 @@ H5FD_log_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hadd timeval_diff.tv_sec--; } /* end if */ time_diff = (double)timeval_diff.tv_sec + ((double)timeval_diff.tv_usec / (double)1000000.0f); - HDfprintf(file->logfp, " (%f s)\n", time_diff); + HDfprintf(file->logfp, " (%fs @ %.6lu.%.6llu)\n", time_diff, (unsigned long long)timeval_start.tv_sec, (unsigned long long)timeval_start.tv_usec); /* Add to total seek time */ file->total_seek_time += time_diff; @@ -1241,7 +1301,12 @@ H5FD_log_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hadd if(file->fa.flags & H5FD_LOG_LOC_READ) { HDfprintf(file->logfp, "%10a-%10a (%10Zu bytes) (%s) Read", orig_addr, (orig_addr + orig_size) - 1, orig_size, flavors[type]); - /* XXX: Verify the flavor information, if we have it? */ + /* Verify that we are reading in the type of data we allocated in this location */ + if(file->flavor) { + HDassert(type == H5FD_MEM_DEFAULT || type == (H5FD_mem_t)file->flavor[orig_addr] || (H5FD_mem_t)file->flavor[orig_addr] == H5FD_MEM_DEFAULT); + HDassert(type == H5FD_MEM_DEFAULT || type == (H5FD_mem_t)file->flavor[(orig_addr + orig_size) - 1] || (H5FD_mem_t)file->flavor[(orig_addr + orig_size) - 1] == H5FD_MEM_DEFAULT); + } /* end if */ + #ifdef H5_HAVE_GETTIMEOFDAY if(file->fa.flags & H5FD_LOG_TIME_READ) { @@ -1256,7 +1321,7 @@ H5FD_log_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hadd timeval_diff.tv_sec--; } /* end if */ time_diff = (double)timeval_diff.tv_sec + ((double)timeval_diff.tv_usec / (double)1000000.0f); - HDfprintf(file->logfp, " (%f s)\n", time_diff); + HDfprintf(file->logfp, " (%fs @ %.6lu.%.6llu)\n", time_diff, (unsigned long long)timeval_start.tv_sec, (unsigned long long)timeval_start.tv_usec); /* Add to total read time */ file->total_read_time += time_diff; @@ -1369,7 +1434,7 @@ H5FD_log_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, had timeval_diff.tv_sec--; } /* end if */ time_diff = (double)timeval_diff.tv_sec + ((double)timeval_diff.tv_usec / (double)1000000.0f); - HDfprintf(file->logfp, " (%f s)\n", time_diff); + HDfprintf(file->logfp, " (%fs @ %.6lu.%.6llu)\n", time_diff, (unsigned long long)timeval_start.tv_sec, (unsigned long long)timeval_start.tv_usec); /* Add to total seek time */ file->total_seek_time += time_diff; @@ -1438,8 +1503,10 @@ H5FD_log_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, had /* Check if this is the first write into a "default" section, grabbed by the metadata agregation algorithm */ if(file->fa.flags & H5FD_LOG_FLAVOR) { - if((H5FD_mem_t)file->flavor[orig_addr] == H5FD_MEM_DEFAULT) + if((H5FD_mem_t)file->flavor[orig_addr] == H5FD_MEM_DEFAULT) { HDmemset(&file->flavor[orig_addr], (int)type, orig_size); + HDfprintf(file->logfp, " (fresh)"); + } /* end if */ } /* end if */ #ifdef H5_HAVE_GETTIMEOFDAY @@ -1455,7 +1522,7 @@ H5FD_log_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, had timeval_diff.tv_sec--; } /* end if */ time_diff = (double)timeval_diff.tv_sec + ((double)timeval_diff.tv_usec / (double)1000000.0f); - HDfprintf(file->logfp, " (%f s)\n", time_diff); + HDfprintf(file->logfp, " (%fs @ %.6lu.%.6llu)\n", time_diff, (unsigned long long)timeval_start.tv_sec, (unsigned long long)timeval_start.tv_usec); /* Add to total write time */ file->total_write_time += time_diff; @@ -1509,6 +1576,9 @@ H5FD_log_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_U /* Extend the file to make sure it's large enough */ if(!H5F_addr_eq(file->eoa, file->eof)) { +#ifdef H5_HAVE_GETTIMEOFDAY + struct timeval timeval_start, timeval_stop; +#endif /* H5_HAVE_GETTIMEOFDAY */ #ifdef H5_HAVE_WIN32_API LARGE_INTEGER li; /* 64-bit (union) integer for SetFilePointer() call */ DWORD dwPtrLow; /* Low-order pointer bits from SetFilePointer() @@ -1516,7 +1586,13 @@ H5FD_log_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_U */ DWORD dwError; /* DWORD error code from GetLastError() */ BOOL bError; /* Boolean error flag */ +#endif /* H5_HAVE_WIN32_API */ +#ifdef H5_HAVE_GETTIMEOFDAY + if(file->fa.flags & H5FD_LOG_TIME_TRUNCATE) + HDgettimeofday(&timeval_start, NULL); +#endif /* H5_HAVE_GETTIMEOFDAY */ +#ifdef H5_HAVE_WIN32_API /* Windows uses this odd QuadPart union for 32/64-bit portability */ li.QuadPart = (__int64)file->eoa; @@ -1539,10 +1615,40 @@ H5FD_log_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_U if(-1 == HDftruncate(file->fd, (HDoff_t)file->eoa)) HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to extend file properly") #endif /* H5_HAVE_WIN32_API */ +#ifdef H5_HAVE_GETTIMEOFDAY + if(file->fa.flags & H5FD_LOG_TIME_TRUNCATE) + HDgettimeofday(&timeval_stop, NULL); +#endif /* H5_HAVE_GETTIMEOFDAY */ /* Log information about the truncate */ if(file->fa.flags & H5FD_LOG_NUM_TRUNCATE) file->total_truncate_ops++; + if(file->fa.flags & H5FD_LOG_TRUNCATE) { + HDfprintf(file->logfp, "Truncate: To %10a", file->eoa); +#ifdef H5_HAVE_GETTIMEOFDAY + if(file->fa.flags & H5FD_LOG_TIME_TRUNCATE) { + struct timeval timeval_diff; + double time_diff; + + /* Calculate the elapsed gettimeofday time */ + timeval_diff.tv_usec = timeval_stop.tv_usec - timeval_start.tv_usec; + timeval_diff.tv_sec = timeval_stop.tv_sec - timeval_start.tv_sec; + if(timeval_diff.tv_usec < 0) { + timeval_diff.tv_usec += 1000000; + timeval_diff.tv_sec--; + } /* end if */ + time_diff = (double)timeval_diff.tv_sec + ((double)timeval_diff.tv_usec / (double)1000000.0f); + HDfprintf(file->logfp, " (%fs @ %.6lu.%.6llu)\n", time_diff, (unsigned long long)timeval_start.tv_sec, (unsigned long long)timeval_start.tv_usec); + + /* Add to total truncate time */ + file->total_truncate_time += time_diff; + } /* end if */ + else + HDfprintf(file->logfp, "\n"); +#else /* H5_HAVE_GETTIMEOFDAY */ + HDfprintf(file->logfp, "\n"); +#endif /* H5_HAVE_GETTIMEOFDAY */ + } /* end if */ /* Update the eof value */ file->eof = file->eoa; @@ -1572,21 +1678,25 @@ done: static herr_t H5FD_log_lock(H5FD_t *_file, hbool_t rw) { - H5FD_log_t *file = (H5FD_log_t *)_file; - int lock; - herr_t ret_value = SUCCEED; /* Return value */ + H5FD_log_t *file = (H5FD_log_t *)_file; /* VFD file struct */ + int lock_flags; /* file locking flags */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT /* Sanity check */ HDassert(file); - /* Determine the type of lock */ - lock = rw ? LOCK_EX : LOCK_SH; + /* Set exclusive or shared lock based on rw status */ + lock_flags = rw ? LOCK_EX : LOCK_SH; - /* Place the lock with non-blocking */ - if(HDflock(file->fd, lock | LOCK_NB) < 0) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to flock file") + /* Place a non-blocking lock on the file */ + if(HDflock(file->fd, lock_flags | LOCK_NB) < 0) { + if(ENOSYS == errno) + HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + else + HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") + } /* end if */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -1607,15 +1717,19 @@ done: static herr_t H5FD_log_unlock(H5FD_t *_file) { - H5FD_log_t *file = (H5FD_log_t *)_file; /* VFD file struct */ - herr_t ret_value = SUCCEED; /* Return value */ + H5FD_log_t *file = (H5FD_log_t *)_file; /* VFD file struct */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT HDassert(file); - if(HDflock(file->fd, LOCK_UN) < 0) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to flock (unlock) file") + if(HDflock(file->fd, LOCK_UN) < 0) { + if(ENOSYS == errno) + HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + else + HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") + } /* end if */ done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDlog.h b/src/H5FDlog.h index 2f1544e..11044e2 100644 --- a/src/H5FDlog.h +++ b/src/H5FDlog.h @@ -25,34 +25,39 @@ #define H5FD_LOG (H5FD_log_init()) /* Flags for H5Pset_fapl_log() */ +/* Flags for tracking 'meta' operations (truncate) */ +#define H5FD_LOG_TRUNCATE 0x00000001 +#define H5FD_LOG_META_IO (H5FD_LOG_TRUNCATE) /* Flags for tracking where reads/writes/seeks occur */ -#define H5FD_LOG_LOC_READ 0x00000001 -#define H5FD_LOG_LOC_WRITE 0x00000002 -#define H5FD_LOG_LOC_SEEK 0x00000004 +#define H5FD_LOG_LOC_READ 0x00000002 +#define H5FD_LOG_LOC_WRITE 0x00000004 +#define H5FD_LOG_LOC_SEEK 0x00000008 #define H5FD_LOG_LOC_IO (H5FD_LOG_LOC_READ|H5FD_LOG_LOC_WRITE|H5FD_LOG_LOC_SEEK) /* Flags for tracking number of times each byte is read/written */ -#define H5FD_LOG_FILE_READ 0x00000008 -#define H5FD_LOG_FILE_WRITE 0x00000010 +#define H5FD_LOG_FILE_READ 0x00000010 +#define H5FD_LOG_FILE_WRITE 0x00000020 #define H5FD_LOG_FILE_IO (H5FD_LOG_FILE_READ|H5FD_LOG_FILE_WRITE) /* Flag for tracking "flavor" (type) of information stored at each byte */ -#define H5FD_LOG_FLAVOR 0x00000020 +#define H5FD_LOG_FLAVOR 0x00000040 /* Flags for tracking total number of reads/writes/seeks/truncates */ -#define H5FD_LOG_NUM_READ 0x00000040 -#define H5FD_LOG_NUM_WRITE 0x00000080 -#define H5FD_LOG_NUM_SEEK 0x00000100 -#define H5FD_LOG_NUM_TRUNCATE 0x00000200 +#define H5FD_LOG_NUM_READ 0x00000080 +#define H5FD_LOG_NUM_WRITE 0x00000100 +#define H5FD_LOG_NUM_SEEK 0x00000200 +#define H5FD_LOG_NUM_TRUNCATE 0x00000400 #define H5FD_LOG_NUM_IO (H5FD_LOG_NUM_READ|H5FD_LOG_NUM_WRITE|H5FD_LOG_NUM_SEEK|H5FD_LOG_NUM_TRUNCATE) -/* Flags for tracking time spent in open/stat/read/write/seek/close */ -#define H5FD_LOG_TIME_OPEN 0x00000400 -#define H5FD_LOG_TIME_STAT 0x00000800 -#define H5FD_LOG_TIME_READ 0x00001000 -#define H5FD_LOG_TIME_WRITE 0x00002000 -#define H5FD_LOG_TIME_SEEK 0x00004000 -#define H5FD_LOG_TIME_CLOSE 0x00008000 -#define H5FD_LOG_TIME_IO (H5FD_LOG_TIME_OPEN|H5FD_LOG_TIME_STAT|H5FD_LOG_TIME_READ|H5FD_LOG_TIME_WRITE|H5FD_LOG_TIME_SEEK|H5FD_LOG_TIME_CLOSE) -/* Flag for tracking allocation of space in file */ -#define H5FD_LOG_ALLOC 0x00010000 -#define H5FD_LOG_ALL (H5FD_LOG_ALLOC|H5FD_LOG_TIME_IO|H5FD_LOG_NUM_IO|H5FD_LOG_FLAVOR|H5FD_LOG_FILE_IO|H5FD_LOG_LOC_IO) +/* Flags for tracking time spent in open/stat/read/write/seek/truncate/close */ +#define H5FD_LOG_TIME_OPEN 0x00000800 +#define H5FD_LOG_TIME_STAT 0x00001000 +#define H5FD_LOG_TIME_READ 0x00002000 +#define H5FD_LOG_TIME_WRITE 0x00004000 +#define H5FD_LOG_TIME_SEEK 0x00008000 +#define H5FD_LOG_TIME_TRUNCATE 0x00010000 +#define H5FD_LOG_TIME_CLOSE 0x00020000 +#define H5FD_LOG_TIME_IO (H5FD_LOG_TIME_OPEN|H5FD_LOG_TIME_STAT|H5FD_LOG_TIME_READ|H5FD_LOG_TIME_WRITE|H5FD_LOG_TIME_SEEK|H5FD_LOG_TIME_TRUNCATE|H5FD_LOG_TIME_CLOSE) +/* Flags for tracking allocation/release of space in file */ +#define H5FD_LOG_ALLOC 0x00040000 +#define H5FD_LOG_FREE 0x00080000 +#define H5FD_LOG_ALL (H5FD_LOG_FREE|H5FD_LOG_ALLOC|H5FD_LOG_TIME_IO|H5FD_LOG_NUM_IO|H5FD_LOG_FLAVOR|H5FD_LOG_FILE_IO|H5FD_LOG_LOC_IO|H5FD_LOG_META_IO) #ifdef __cplusplus extern "C" { diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index ed70e20..a3a404f 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -90,7 +90,7 @@ static herr_t H5FD_mpio_read(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, hadd size_t size, void *buf); static herr_t H5FD_mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, const void *buf); -static herr_t H5FD_mpio_flush(H5FD_t *_file, hid_t dxpl_id, unsigned closing); +static herr_t H5FD_mpio_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); static herr_t H5FD_mpio_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); static int H5FD_mpio_mpi_rank(const H5FD_t *_file); static int H5FD_mpio_mpi_size(const H5FD_t *_file); @@ -1866,7 +1866,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_mpio_flush(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, unsigned closing) +H5FD_mpio_flush(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t closing) { H5FD_mpio_t *file = (H5FD_mpio_t*)_file; int mpi_code; /* mpi return code */ diff --git a/src/H5FDmulti.c b/src/H5FDmulti.c index d0dfab3..5e27c53 100644 --- a/src/H5FDmulti.c +++ b/src/H5FDmulti.c @@ -534,7 +534,7 @@ H5Pget_fapl_multi(hid_t fapl_id, H5FD_mem_t *memb_map/*out*/, hid_t *memb_fapl/*out*/, char **memb_name/*out*/, haddr_t *memb_addr/*out*/, hbool_t *relax) { - H5FD_multi_fapl_t *fa; + const H5FD_multi_fapl_t *fa; H5FD_mem_t mt; static const char *func="H5FDget_fapl_multi"; /* Function Name for error reporting */ @@ -548,7 +548,7 @@ H5Pget_fapl_multi(hid_t fapl_id, H5FD_mem_t *memb_map/*out*/, H5Epush_ret(func, H5E_ERR_CLS, H5E_PLIST, H5E_BADTYPE, "not an access list", -1) if(H5FD_MULTI != H5Pget_driver(fapl_id)) H5Epush_ret(func, H5E_ERR_CLS, H5E_PLIST, H5E_BADVALUE, "incorrect VFL driver", -1) - if(NULL == (fa= (H5FD_multi_fapl_t *)H5Pget_driver_info(fapl_id))) + if(NULL == (fa= (const H5FD_multi_fapl_t *)H5Pget_driver_info(fapl_id))) H5Epush_ret(func, H5E_ERR_CLS, H5E_PLIST, H5E_BADVALUE, "bad VFL driver info", -1) if (memb_map) @@ -992,7 +992,7 @@ H5FD_multi_open(const char *name, unsigned flags, hid_t fapl_id, { H5FD_multi_t *file=NULL; hid_t close_fapl=-1; - H5FD_multi_fapl_t *fa; + const H5FD_multi_fapl_t *fa; H5FD_mem_t m; static const char *func="H5FD_multi_open"; /* Function Name for error reporting */ @@ -1018,7 +1018,7 @@ H5FD_multi_open(const char *name, unsigned flags, hid_t fapl_id, if(H5Pset_fapl_multi(fapl_id, NULL, NULL, NULL, NULL, TRUE)<0) H5Epush_goto(func, H5E_ERR_CLS, H5E_FILE, H5E_CANTSET, "can't set property value", error) } - fa = (H5FD_multi_fapl_t *)H5Pget_driver_info(fapl_id); + fa = (const H5FD_multi_fapl_t *)H5Pget_driver_info(fapl_id); assert(fa); ALL_MEMBERS(mt) { file->fa.memb_map[mt] = fa->memb_map[mt]; @@ -1290,7 +1290,7 @@ H5FD_multi_get_eoa(const H5FD_t *_file, H5FD_mem_t type) } else { H5FD_mem_t mmt = file->fa.memb_map[type]; - if(H5FD_MEM_DEFAULT==mmt) + if(H5FD_MEM_DEFAULT == mmt) mmt = type; if(file->memb[mmt]) { @@ -1358,18 +1358,24 @@ H5FD_multi_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t eoa) H5Eclear2(H5E_DEFAULT); mmt = file->fa.memb_map[type]; - if(H5FD_MEM_DEFAULT == mmt) - mmt = type; - - /* Handle backward compatibility in a quick and simple way. v1.6 library had EOA for the entire virtual - * file. But it wasn't meaningful. So v1.8 library doesn't have it anymore. It saves the EOA for the - * metadata file, instead. Here we try to figure out whether the EOA is from a v1.6 file by comparing its - * value. If it is a big value, we assume it's from v1.6 and simply discard it. This is the normal case - * when the metadata file has the smallest starting address. If the metadata file has the biggest address, - * the EOAs of v1.6 and v1.8 files are the same. It won't cause any trouble. (Please see Issue 2598 - * in Jira) SLU - 2011/6/21 + if(H5FD_MEM_DEFAULT == mmt) { + if(H5FD_MEM_DEFAULT == type) + mmt = H5FD_MEM_SUPER; + else + mmt = type; + } /* end if */ + + /* Handle backward compatibility in a quick and simple way. v1.6 library + * had EOA for the entire virtual file. But it wasn't meaningful. So v1.8 + * library doesn't have it anymore. It saves the EOA for the metadata file, + * instead. Here we try to figure out whether the EOA is from a v1.6 file + * by comparing its value. If it is a big value, we assume it's from v1.6 + * and simply discard it. This is the normal case when the metadata file + * has the smallest starting address. If the metadata file has the biggest + * address, the EOAs of v1.6 and v1.8 files are the same. It won't cause + * any trouble. (Please see Issue 2598 in Jira) SLU - 2011/6/21 */ - if(H5FD_MEM_SUPER == type && file->memb_eoa[H5FD_MEM_SUPER] > 0 && eoa > file->memb_eoa[H5FD_MEM_SUPER]) + if(H5FD_MEM_SUPER == mmt && file->memb_eoa[H5FD_MEM_SUPER] > 0 && eoa > (file->memb_next[H5FD_MEM_SUPER] / 2)) return 0; assert(eoa >= file->fa.memb_addr[mmt]); @@ -1378,7 +1384,7 @@ H5FD_multi_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t eoa) H5E_BEGIN_TRY { status = H5FDset_eoa(file->memb[mmt], mmt, (eoa - file->fa.memb_addr[mmt])); } H5E_END_TRY; - if (status<0) + if(status < 0) H5Epush_ret(func, H5E_ERR_CLS, H5E_FILE, H5E_BADVALUE, "member H5FDset_eoa failed", -1) return 0; diff --git a/src/H5FDpkg.h b/src/H5FDpkg.h index a0c1b3a..45bcfd8 100644 --- a/src/H5FDpkg.h +++ b/src/H5FDpkg.h @@ -59,6 +59,7 @@ H5_DLL herr_t H5FD_free_real(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, /* Testing functions */ #ifdef H5FD_TESTING +H5_DLL hbool_t H5FD_supports_swmr_test(const char *vfd_name); #endif /* H5FD_TESTING */ #endif /* _H5FDpkg_H */ diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h index 915f7e1..427b42c 100644 --- a/src/H5FDprivate.h +++ b/src/H5FDprivate.h @@ -130,7 +130,11 @@ struct H5P_genplist_t; struct H5F_t; H5_DLL int H5FD_term_interface(void); -H5_DLL herr_t H5FD_locate_signature(H5FD_t *file, const H5P_genplist_t *dxpl, haddr_t *sig_addr); +H5_DLL herr_t H5FD_locate_signature(H5FD_t *file, +#ifndef H5_DEBUG_BUILD +const +#endif /* H5_DEBUG_BUILD */ +H5P_genplist_t *dxpl, haddr_t *sig_addr); H5_DLL H5FD_class_t *H5FD_get_class(hid_t id); H5_DLL hsize_t H5FD_sb_size(H5FD_t *file); H5_DLL herr_t H5FD_sb_encode(H5FD_t *file, char *name/*out*/, uint8_t *buf); @@ -153,10 +157,19 @@ H5_DLL herr_t H5FD_set_eoa(H5FD_t *file, H5FD_mem_t type, haddr_t addr); H5_DLL haddr_t H5FD_get_eof(const H5FD_t *file, H5FD_mem_t type); H5_DLL haddr_t H5FD_get_maxaddr(const H5FD_t *file); H5_DLL herr_t H5FD_get_feature_flags(const H5FD_t *file, unsigned long *feature_flags); +H5_DLL herr_t H5FD_set_feature_flags(H5FD_t *file, unsigned long feature_flags); H5_DLL herr_t H5FD_get_fs_type_map(const H5FD_t *file, H5FD_mem_t *type_map); -H5_DLL herr_t H5FD_read(H5FD_t *file, const H5P_genplist_t *dxpl, H5FD_mem_t type, +H5_DLL herr_t H5FD_read(H5FD_t *file, +#ifndef H5_DEBUG_BUILD +const +#endif /* H5_DEBUG_BUILD */ +H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/); -H5_DLL herr_t H5FD_write(H5FD_t *file, const H5P_genplist_t *dxpl, H5FD_mem_t type, +H5_DLL herr_t H5FD_write(H5FD_t *file, +#ifndef H5_DEBUG_BUILD +const +#endif /* H5_DEBUG_BUILD */ +H5P_genplist_t *dxpl, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf); H5_DLL herr_t H5FD_flush(H5FD_t *file, hid_t dxpl_id, hbool_t closing); H5_DLL herr_t H5FD_truncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing); diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h index 4931e0f..436be10 100644 --- a/src/H5FDpublic.h +++ b/src/H5FDpublic.h @@ -234,6 +234,11 @@ typedef enum H5F_mem_t H5FD_mem_t; * image to store in memory. */ #define H5FD_FEAT_CAN_USE_FILE_IMAGE_CALLBACKS 0x00000800 + /* + * Defining H5FD_FEAT_SUPPORTS_SWMR_IO for a VFL driver means that the + * driver supports the single-writer/multiple-readers I/O pattern. + */ +#define H5FD_FEAT_SUPPORTS_SWMR_IO 0x00001000 /* Forward declaration */ typedef struct H5FD_t H5FD_t; @@ -294,6 +299,7 @@ struct H5FD_t { hid_t driver_id; /*driver ID for this file */ const H5FD_class_t *cls; /*constant class info */ unsigned long fileno; /* File 'serial' number */ + unsigned access_flags; /* File access flags (from create or open) */ unsigned long feature_flags; /* VFL Driver feature Flags */ haddr_t maxaddr; /* For this file, overrides class */ haddr_t base_addr; /* Base address for HDF5 data w/in file */ diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c index bb8f004..0ca5efb 100644 --- a/src/H5FDsec2.c +++ b/src/H5FDsec2.c @@ -530,6 +530,7 @@ H5FD_sec2_query(const H5FD_t *_file, unsigned long *flags /* out */) *flags |= H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */ *flags |= H5FD_FEAT_AGGREGATE_SMALLDATA; /* OK to aggregate "small" raw data allocations */ *flags |= H5FD_FEAT_POSIX_COMPAT_HANDLE; /* VFD handle is POSIX I/O call compatible */ + *flags |= H5FD_FEAT_SUPPORTS_SWMR_IO; /* VFD supports the single-writer/multiple-readers (SWMR) pattern */ /* Check for flags that are set by h5repart */ if(file && file->fam_to_sec2) @@ -922,20 +923,24 @@ done: static herr_t H5FD_sec2_lock(H5FD_t *_file, hbool_t rw) { - H5FD_sec2_t *file = (H5FD_sec2_t *)_file; /* VFD file struct */ - int lock; /* The type of lock */ - herr_t ret_value = SUCCEED; /* Return value */ + H5FD_sec2_t *file = (H5FD_sec2_t *)_file; /* VFD file struct */ + int lock_flags; /* file locking flags */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT HDassert(file); - /* Determine the type of lock */ - lock = rw ? LOCK_EX : LOCK_SH; - - /* Place the lock with non-blocking */ - if(HDflock(file->fd, lock | LOCK_NB) < 0) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to flock file") + /* Set exclusive or shared lock based on rw status */ + lock_flags = rw ? LOCK_EX : LOCK_SH; + + /* Place a non-blocking lock on the file */ + if(HDflock(file->fd, lock_flags | LOCK_NB) < 0) { + if(ENOSYS == errno) + HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + else + HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") + } /* end if */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -956,15 +961,19 @@ done: static herr_t H5FD_sec2_unlock(H5FD_t *_file) { - H5FD_sec2_t *file = (H5FD_sec2_t *)_file; /* VFD file struct */ - herr_t ret_value = SUCCEED; /* Return value */ + H5FD_sec2_t *file = (H5FD_sec2_t *)_file; /* VFD file struct */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT HDassert(file); - if(HDflock(file->fd, LOCK_UN) < 0) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to flock (unlock) file") + if(HDflock(file->fd, LOCK_UN) < 0) { + if(ENOSYS == errno) + HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + else + HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") + } /* end if */ done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDstdio.c b/src/H5FDstdio.c index 0168ff4..4c62bcc 100644 --- a/src/H5FDstdio.c +++ b/src/H5FDstdio.c @@ -24,6 +24,7 @@ * and is not intended for production use! */ #include <assert.h> +#include <errno.h> #include <stdio.h> #include <stdlib.h> #include <string.h> @@ -549,7 +550,11 @@ H5FD_stdio_query(const H5FD_t *_f, unsigned long /*OUT*/ *flags) /* Quiet the compiler */ _f=_f; - /* Set the VFL feature flags that this driver supports */ + /* Set the VFL feature flags that this driver supports. + * + * Note that this VFD does not support SWMR due to the unpredictable + * nature of the buffering layer. + */ if(flags) { *flags = 0; *flags|=H5FD_FEAT_AGGREGATE_METADATA; /* OK to aggregate metadata allocations */ @@ -1078,8 +1083,8 @@ H5FD_stdio_truncate(H5FD_t *_file, hid_t /*UNUSED*/ dxpl_id, * Function: H5FD_stdio_lock * * Purpose: Lock a file via flock - * * NOTE: This function is a no-op if flock() is not present. + * * Errors: * IO FCNTL flock failed. * @@ -1093,21 +1098,27 @@ static herr_t H5FD_stdio_lock(H5FD_t *_file, hbool_t rw) { #ifdef H5_HAVE_FLOCK - H5FD_stdio_t *file = (H5FD_stdio_t*)_file; - int lock; /* The type of lock */ - static const char *func = "H5FD_stdio_lock"; /* Function Name for error reporting */ + H5FD_stdio_t *file = (H5FD_stdio_t*)_file; /* VFD file struct */ + int lock_flags; /* file locking flags */ + static const char *func = "H5FD_stdio_lock"; /* Function Name for error reporting */ /* Clear the error stack */ H5Eclear2(H5E_DEFAULT); assert(file); - /* Determine the type of lock */ - lock = rw ? LOCK_EX : LOCK_SH; + /* Set exclusive or shared lock based on rw status */ + lock_flags = rw ? LOCK_EX : LOCK_SH; + + /* Place a non-blocking lock on the file */ + if(flock(file->fd, lock_flags | LOCK_NB) < 0) { + if(ENOSYS == errno) + H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)", -1) + else + H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file lock failed", -1) + } /* end if */ - /* Place the lock with non-blocking */ - if(flock(file->fd, lock | LOCK_NB) < 0) - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "flock failed", -1) + /* Flush the stream */ if(fflush(file->fp) < 0) H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_WRITEERROR, "fflush failed", -1) @@ -1117,16 +1128,15 @@ H5FD_stdio_lock(H5FD_t *_file, hbool_t rw) } /* end H5FD_stdio_lock() */ /*------------------------------------------------------------------------- - * Function: H5F_stdio_unlock - * - * Purpose: Unlock a file via flock + * Function: H5F_stdio_unlock * + * Purpose: Unlock a file via flock + * NOTE: This function is a no-op if flock() is not present. * - * NOTE: This function is a no-op if flock() is not present. * Errors: * IO FCNTL flock failed. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * * Programmer: Vailin Choi; March 2015 * @@ -1136,18 +1146,25 @@ static herr_t H5FD_stdio_unlock(H5FD_t *_file) { #ifdef H5_HAVE_FLOCK - H5FD_stdio_t *file = (H5FD_stdio_t*)_file; - static const char *func = "H5FD_stdio_unlock"; /* Function Name for error reporting */ + H5FD_stdio_t *file = (H5FD_stdio_t*)_file; /* VFD file struct */ + static const char *func = "H5FD_stdio_unlock"; /* Function Name for error reporting */ /* Clear the error stack */ H5Eclear2(H5E_DEFAULT); assert(file); + /* Flush the stream */ if(fflush(file->fp) < 0) H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_WRITEERROR, "fflush failed", -1) - if(flock(file->fd, LOCK_UN) < 0) - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "flock (unlock) failed", -1) + + /* Place a non-blocking lock on the file */ + if(flock(file->fd, LOCK_UN) < 0) { + if(ENOSYS == errno) + H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)", -1) + else + H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file unlock failed", -1) + } /* end if */ #endif /* H5_HAVE_FLOCK */ diff --git a/src/H5FDtest.c b/src/H5FDtest.c new file mode 100644 index 0000000..fc9188b --- /dev/null +++ b/src/H5FDtest.c @@ -0,0 +1,118 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5FDtest.c + * Fall 2014 + * + * Purpose: File driver testing routines. + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5FDmodule.h" /* This source code file is part of the H5FD module */ +#define H5FD_TESTING /* Suppress warning about H5FD testing funcs */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5FDpkg.h" /* File Drivers */ + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + + +/*------------------------------------------------------------------------- + * Function: H5FD_supports_swmr_test() + * + * Purpose: Determines if a VFD supports SWMR. + * + * The function determines SWMR support by inspecting the + * HDF5_DRIVER environment variable, not by checking the + * VFD feature flags (which do not exist until the driver + * is instantiated). + * + * See test/Makefile.am for a list of the VFD strings. + * + * This function is only intended for use in the test code. + * + * Return: TRUE (1) if the VFD supports SWMR I/O or vfd_name is + * NULL or the empty string (which implies the default VFD). + * + * FALSE (0) if it does not + * + * This function cannot fail at this time so there is no + * error return value. + * + * Programmer: Dana Robinson + * Fall 2014 + * + *------------------------------------------------------------------------- + */ +hbool_t +H5FD_supports_swmr_test(const char *vfd_name) +{ + hbool_t ret_value = FALSE; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + if(!vfd_name || !HDstrcmp(vfd_name, "")) + ret_value = TRUE; + else + ret_value = !HDstrcmp(vfd_name, "log") + || !HDstrcmp(vfd_name, "sec2"); + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_supports_swmr_test() */ + @@ -109,7 +109,7 @@ H5FS_create(H5F_t *f, hid_t dxpl_id, haddr_t *fs_addr, const H5FS_create_t *fs_c H5FS_t *fspace = NULL; /* New free space structure */ H5FS_t *ret_value = NULL; /* Return value */ - FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, NULL) + FUNC_ENTER_NOAPI(NULL) #ifdef H5FS_DEBUG HDfprintf(stderr, "%s: Creating free space manager, nclasses = %Zu\n", FUNC, nclasses); #endif /* H5FS_DEBUG */ @@ -132,6 +132,7 @@ HDfprintf(stderr, "%s: Creating free space manager, nclasses = %Zu\n", FUNC, ncl fspace->expand_percent = fs_create->expand_percent; fspace->max_sect_addr = fs_create->max_sect_addr; fspace->max_sect_size = fs_create->max_sect_size; + fspace->swmr_write = (H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) > 0; fspace->alignment = alignment; fspace->threshold = threshold; @@ -167,7 +168,7 @@ done: #ifdef H5FS_DEBUG HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value); #endif /* H5FS_DEBUG */ - FUNC_LEAVE_NOAPI_TAG(ret_value, NULL) + FUNC_LEAVE_NOAPI(ret_value) } /* H5FS_create() */ @@ -192,7 +193,7 @@ H5FS_open(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr, uint16_t nclasses, H5FS_hdr_cache_ud_t cache_udata; /* User-data for metadata cache callback */ H5FS_t *ret_value = NULL; /* Return value */ - FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, NULL) + FUNC_ENTER_NOAPI(NULL) #ifdef H5FS_DEBUG HDfprintf(stderr, "%s: Opening free space manager, fs_addr = %a, nclasses = %Zu\n", FUNC, fs_addr, nclasses); #endif /* H5FS_DEBUG */ @@ -236,7 +237,7 @@ HDfprintf(stderr, "%s: fspace->rc = %u\n", FUNC, fspace->rc); ret_value = fspace; done: - FUNC_LEAVE_NOAPI_TAG(ret_value, NULL) + FUNC_LEAVE_NOAPI(ret_value) } /* H5FS_open() */ @@ -259,7 +260,7 @@ H5FS_delete(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr) H5FS_hdr_cache_ud_t cache_udata; /* User-data for metadata cache callback */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + FUNC_ENTER_NOAPI(FAIL) #ifdef H5FS_DEBUG HDfprintf(stderr, "%s: Deleting free space manager, fs_addr = %a\n", FUNC, fs_addr); #endif /* H5FS_DEBUG */ @@ -386,7 +387,7 @@ done: if(fspace && H5AC_unprotect(f, dxpl_id, H5AC_FSPACE_HDR, fs_addr, fspace, H5AC__DELETED_FLAG | H5AC__FREE_FILE_SPACE_FLAG) < 0) HDONE_ERROR(H5E_FSPACE, H5E_CANTUNPROTECT, FAIL, "unable to release free space header") - FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) + FUNC_LEAVE_NOAPI(ret_value) } /* H5FS_delete() */ @@ -408,7 +409,7 @@ H5FS_close(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + FUNC_ENTER_NOAPI(FAIL) /* Check arguments. */ HDassert(f); @@ -569,7 +570,7 @@ done: #ifdef H5FS_DEBUG HDfprintf(stderr, "%s: Leaving, ret_value = %d, fspace->rc = %u\n", FUNC, ret_value, fspace->rc); #endif /* H5FS_DEBUG */ - FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) + FUNC_LEAVE_NOAPI(ret_value) } /* H5FS_close() */ @@ -824,7 +825,7 @@ H5FS_alloc_hdr(H5F_t *f, H5FS_t *fspace, haddr_t *fs_addr, hid_t dxpl_id) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + FUNC_ENTER_NOAPI(FAIL) /* Check arguments. */ HDassert(f); @@ -844,7 +845,7 @@ H5FS_alloc_hdr(H5F_t *f, H5FS_t *fspace, haddr_t *fs_addr, hid_t dxpl_id) *fs_addr = fspace->addr; done: - FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) + FUNC_LEAVE_NOAPI(ret_value) } /* H5FS_alloc_hdr() */ @@ -865,7 +866,7 @@ H5FS_alloc_sect(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + FUNC_ENTER_NOAPI(FAIL) /* Check arguments. */ HDassert(f); @@ -891,7 +892,7 @@ H5FS_alloc_sect(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id) } /* end if */ done: - FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) + FUNC_LEAVE_NOAPI(ret_value) } /* H5FS_alloc_sect() */ @@ -914,7 +915,7 @@ H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id) unsigned cache_flags; /* Flags for unprotecting cache entries */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + FUNC_ENTER_NOAPI(FAIL) /* Check arguments. */ HDassert(f); @@ -1000,7 +1001,7 @@ H5FS_free(H5F_t *f, H5FS_t *fspace, hid_t dxpl_id) } /* end if */ done: - FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) + FUNC_LEAVE_NOAPI(ret_value) } /* H5FS_free() */ @@ -1212,3 +1213,4 @@ HDfprintf(stderr, "%s: fspace->tot_sect_count = %Hu\n", "H5FS_assert", fspace->t FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5FS_assert() */ #endif /* H5FS_DEBUG_ASSERT */ + diff --git a/src/H5FScache.c b/src/H5FScache.c index 1d27972..42eccff 100644 --- a/src/H5FScache.c +++ b/src/H5FScache.c @@ -78,30 +78,29 @@ static herr_t H5FS__sinfo_serialize_sect_cb(void *_item, void H5_ATTR_UNUSED *ke static herr_t H5FS__sinfo_serialize_node_cb(void *_item, void H5_ATTR_UNUSED *key, void *_udata); /* Metadata cache callbacks */ -static herr_t H5FS__cache_hdr_get_load_size(const void *udata, size_t *image_len); +static herr_t H5FS__cache_hdr_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5FS__cache_hdr_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5FS__cache_hdr_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5FS__cache_hdr_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5FS__cache_hdr_image_len(const void *thing, size_t *image_len); static herr_t H5FS__cache_hdr_pre_serialize(const H5F_t *f, hid_t dxpl_id, - void *thing, haddr_t addr, size_t len, size_t compressed_len, - haddr_t *new_addr, size_t *new_len, size_t *new_compressed_len, + void *thing, haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len, unsigned *flags); static herr_t H5FS__cache_hdr_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5FS__cache_hdr_free_icr(void *thing); -static herr_t H5FS__cache_sinfo_get_load_size(const void *udata, size_t *image_len); +static herr_t H5FS__cache_sinfo_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5FS__cache_sinfo_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5FS__cache_sinfo_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5FS__cache_sinfo_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5FS__cache_sinfo_image_len(const void *thing, size_t *image_len); static herr_t H5FS__cache_sinfo_pre_serialize(const H5F_t *f, hid_t dxpl_id, - void *thing, haddr_t addr, size_t len, size_t compressed_len, - haddr_t *new_addr, size_t *new_len, size_t *new_compressed_len, + void *thing, haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len, unsigned *flags); static herr_t H5FS__cache_sinfo_serialize(const H5F_t *f, void *image, size_t len, void *thing); +static herr_t H5FS__cache_sinfo_notify(H5AC_notify_action_t action, void *thing); static herr_t H5FS__cache_sinfo_free_icr(void *thing); @@ -115,14 +114,15 @@ const H5AC_class_t H5AC_FSPACE_HDR[1] = {{ "Free Space Header", /* Metadata client name (for debugging) */ H5FD_MEM_FSPACE_HDR, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5FS__cache_hdr_get_load_size, /* 'get_load_size' callback */ + H5FS__cache_hdr_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5FS__cache_hdr_verify_chksum, /* 'verify_chksum' callback */ H5FS__cache_hdr_deserialize, /* 'deserialize' callback */ H5FS__cache_hdr_image_len, /* 'image_len' callback */ H5FS__cache_hdr_pre_serialize, /* 'pre_serialize' callback */ H5FS__cache_hdr_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5FS__cache_hdr_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -132,14 +132,15 @@ const H5AC_class_t H5AC_FSPACE_SINFO[1] = {{ "Free Space Section Info", /* Metadata client name (for debugging) */ H5FD_MEM_FSPACE_SINFO, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5FS__cache_sinfo_get_load_size, /* 'get_load_size' callback */ + H5FS__cache_sinfo_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5FS__cache_sinfo_verify_chksum, /* 'verify_chksum' callback */ H5FS__cache_sinfo_deserialize, /* 'deserialize' callback */ H5FS__cache_sinfo_image_len, /* 'image_len' callback */ H5FS__cache_sinfo_pre_serialize, /* 'pre_serialize' callback */ H5FS__cache_sinfo_serialize, /* 'serialize' callback */ - NULL, /* 'notify' callback */ + H5FS__cache_sinfo_notify, /* 'notify' callback */ H5FS__cache_sinfo_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -156,7 +157,7 @@ const H5AC_class_t H5AC_FSPACE_SINFO[1] = {{ /*------------------------------------------------------------------------- - * Function: H5FS__cache_hdr_get_load_size + * Function: H5FS__cache_hdr_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -169,9 +170,9 @@ const H5AC_class_t H5AC_FSPACE_SINFO[1] = {{ *------------------------------------------------------------------------- */ static herr_t -H5FS__cache_hdr_get_load_size(const void *_udata, size_t *image_len) +H5FS__cache_hdr_get_initial_load_size(void *_udata, size_t *image_len) { - const H5FS_hdr_cache_ud_t *udata = (const H5FS_hdr_cache_ud_t *)_udata; /* User-data for metadata cache callback */ + H5FS_hdr_cache_ud_t *udata = (H5FS_hdr_cache_ud_t *)_udata; /* User-data for metadata cache callback */ FUNC_ENTER_STATIC_NOERR @@ -184,7 +185,43 @@ H5FS__cache_hdr_get_load_size(const void *_udata, size_t *image_len) *image_len = (size_t)H5FS_HEADER_SIZE(udata->f); FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5FS__cache_hdr_get_load_size() */ +} /* end H5FS__cache_hdr_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5FS__cache_hdr_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +htri_t +H5FS__cache_hdr_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FS__cache_hdr_verify_chksum() */ /*------------------------------------------------------------------------- @@ -212,7 +249,6 @@ H5FS__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, H5FS_hdr_cache_ud_t *udata = (H5FS_hdr_cache_ud_t *)_udata; /* User data for callback */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ unsigned nclasses; /* Number of section classes */ H5FS_t *ret_value = NULL; /* Return value */ @@ -285,8 +321,7 @@ H5FS__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, /* Allocated size of serialized free space sections */ H5F_DECODE_LENGTH(udata->f, image, fspace->alloc_sect_size); - /* Compute checksum on indirect block */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -294,10 +329,6 @@ H5FS__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) <= len); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_FSPACE, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap indirect block") - /* Set return value */ ret_value = fspace; @@ -326,8 +357,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FS__cache_hdr_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5FS__cache_hdr_image_len(const void *_thing, size_t *image_len) { const H5FS_t *fspace = (const H5FS_t *)_thing; /* Pointer to the object */ @@ -375,8 +405,7 @@ H5FS__cache_hdr_image_len(const void *_thing, size_t *image_len, */ static herr_t H5FS__cache_hdr_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, - haddr_t addr, size_t H5_ATTR_UNUSED len, size_t H5_ATTR_UNUSED compressed_len, - haddr_t *new_addr, size_t *new_len, size_t H5_ATTR_UNUSED *new_compressed_len, + haddr_t addr, size_t H5_ATTR_UNUSED len, haddr_t *new_addr, size_t *new_len, unsigned *flags) { H5FS_t *fspace = (H5FS_t *)_thing; /* Pointer to the object */ @@ -384,7 +413,7 @@ H5FS__cache_hdr_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + FUNC_ENTER_STATIC /* Sanity check */ HDassert(f); @@ -410,7 +439,7 @@ H5FS__cache_hdr_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, /* This implies that the header "owns" the section info. * * Unfortunately, the comments in the code are not clear as to - * what this means, but from reviewing the code (most particularly + * what this means, but from reviewing the code (most particularly * H5FS_close(), H5FS_sinfo_lock, and H5FS_sinfo_unlock()), I * gather that it means that the header is maintaining a pointer to * an instance of H5FS_sinfo_t in which free space data is @@ -454,7 +483,7 @@ H5FS__cache_hdr_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, * disk if it doesn't exist. Similarly, since the section info * will not be stored to disk unless the header is, * H5F_addr_defined(fspace->addr) must hold as well. - * + * * As the objective is to touch up the free space manager header * so that it contains sensical data on the size and location of * the section information, we have to handle each of the above @@ -468,8 +497,8 @@ H5FS__cache_hdr_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, * Otherwise, allocate space for the section info in real * file space, insert the section info at this location, and * set fspace->sect_addr, fspace->sect_size, and - * fspace->alloc_sect_size to reflect the new location - * of the section info. Note that it is not necessary to + * fspace->alloc_sect_size to reflect the new location + * of the section info. Note that it is not necessary to * force a write of the section info. * * Case 2) Allocate space for the section info in real file space, @@ -483,22 +512,31 @@ H5FS__cache_hdr_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, * Note that while we may alter the contents of the free space * header in cases 1) and 2), there is no need to mark the header * as dirty, as the metadata cache would not be attempting to - * serialize the header if it though it was clean. + * serialize the header if it thought it was clean. */ if(fspace->serial_sect_count > 0 && H5F_addr_defined(fspace->addr)) { - /* Sanity check */ + /* Sanity check */ HDassert(fspace->sect_size > 0); if(!H5F_addr_defined(fspace->sect_addr)) { /* case 1 */ - /* allocate file space for the section info, and insert it + + haddr_t tag = HADDR_UNDEF; + + /* allocate file space for the section info, and insert it * into the metadata cache. */ if(HADDR_UNDEF == (fspace->sect_addr = H5MF_alloc((H5F_t *)f, H5FD_MEM_FSPACE_SINFO, dxpl_id, fspace->sect_size))) HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, FAIL, "file allocation failed for free space sections") fspace->alloc_sect_size = (size_t)fspace->sect_size; - if(H5AC_insert_entry((H5F_t *)f, dxpl_id, H5AC_FSPACE_SINFO, fspace->sect_addr, fspace->sinfo, H5AC__NO_FLAGS_SET) < 0) - HGOTO_ERROR(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space sections to cache") + + /* Get the tag for this free space manager and use it to insert the entry */ + if(H5AC_get_tag((const void *)fspace, &tag) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTTAG, FAIL, "can't get tag for metadata cache object") + H5_BEGIN_TAG(dxpl_id, tag, FAIL) + if(H5AC_insert_entry((H5F_t *)f, dxpl_id, H5AC_FSPACE_SINFO, fspace->sect_addr, fspace->sinfo, H5AC__NO_FLAGS_SET) < 0) + HGOTO_ERROR_TAG(H5E_FSPACE, H5E_CANTINIT, FAIL, "can't add free space sections to cache") + H5_END_TAG(FAIL) HDassert(fspace->sinfo->cache_info.size == fspace->alloc_sect_size); @@ -510,7 +548,7 @@ H5FS__cache_hdr_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, else if(H5F_IS_TMP_ADDR(f, fspace->sect_addr)) { /* case 2 */ haddr_t new_sect_addr; - /* move the section info from temporary (AKA imaginary) file + /* move the section info from temporary (AKA imaginary) file * space to real file space. */ @@ -566,7 +604,7 @@ H5FS__cache_hdr_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, unsigned sect_status = 0; haddr_t new_sect_addr; - /* we have work to do -- must relocate section info into + /* we have work to do -- must relocate section info into * real file space. * * Since the section info address is in temporary space (AKA @@ -616,7 +654,7 @@ done: if(H5AC_reset_ring(dxpl, orig_ring) < 0) HDONE_ERROR(H5E_FSPACE, H5E_CANTSET, FAIL, "unable to set property value") - FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) + FUNC_LEAVE_NOAPI(ret_value) } /* end H5FS__cache_hdr_pre_serialize() */ @@ -729,10 +767,6 @@ H5FS__cache_hdr_serialize(const H5F_t *f, void *_image, size_t len, FUNC_LEAVE_NOAPI(ret_value) } /* H5FS__cache_hdr_serialize() */ -/***************************************/ -/* no H5FS__cache_hdr_notify() function */ -/***************************************/ - /*------------------------------------------------------------------------- * Function: H5FS__cache_hdr_free_icr @@ -776,13 +810,9 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FS__cache_hdr_free_icr() */ -/********************************************************/ -/* metadata cache callback definitions for section info */ -/********************************************************/ - /*------------------------------------------------------------------------- - * Function: H5FS__cache_sinfo_get_load_size() + * Function: H5FS__cache_sinfo_get_initial_load_size() * * Purpose: Compute the size of the on disk image of the free space * manager section info, and place this value in *image_len. @@ -796,10 +826,10 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FS__cache_sinfo_get_load_size(const void *_udata, size_t *image_len) +H5FS__cache_sinfo_get_initial_load_size(void *_udata, size_t *image_len) { - const H5FS_t *fspace; /* free space manager */ - const H5FS_sinfo_cache_ud_t *udata = (const H5FS_sinfo_cache_ud_t *)_udata; /* User data for callback */ + const H5FS_t *fspace; /* free space manager */ + H5FS_sinfo_cache_ud_t *udata = (H5FS_sinfo_cache_ud_t *)_udata; /* User data for callback */ FUNC_ENTER_STATIC_NOERR @@ -810,10 +840,47 @@ H5FS__cache_sinfo_get_load_size(const void *_udata, size_t *image_len) HDassert(fspace->sect_size > 0); HDassert(image_len); + /* Set the image length size */ *image_len = (size_t)(fspace->sect_size); FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5FS__cache_sinfo_get_load_size() */ +} /* end H5FS__cache_sinfo_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5FS__cache_sinfo_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +htri_t +H5FS__cache_sinfo_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FS__cache_sinfo_verify_chksum() */ /*------------------------------------------------------------------------- @@ -843,7 +910,6 @@ H5FS__cache_sinfo_deserialize(const void *_image, size_t len, void *_udata, size_t old_sect_size; /* Old section size */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum */ - uint32_t computed_chksum; /* Computed metadata checksum */ void * ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC @@ -950,16 +1016,11 @@ H5FS__cache_sinfo_deserialize(const void *_image, size_t len, void *_udata, HDassert(old_tot_space == fspace->tot_space); } /* end if */ - /* Compute checksum on indirect block */ - computed_chksum = H5_checksum_metadata((const uint8_t *)_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_FSPACE, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap indirect block") - /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == old_sect_size); @@ -990,8 +1051,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FS__cache_sinfo_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5FS__cache_sinfo_image_len(const void *_thing, size_t *image_len) { const H5FS_sinfo_t *sinfo = (const H5FS_sinfo_t *)_thing; /* Pointer to the object */ const H5FS_t *fspace; /* Free space header */ @@ -1033,8 +1093,7 @@ H5FS__cache_sinfo_image_len(const void *_thing, size_t *image_len, */ static herr_t H5FS__cache_sinfo_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, - haddr_t addr, size_t len, size_t H5_ATTR_UNUSED compressed_len, haddr_t *new_addr, - size_t *new_len, size_t H5_ATTR_UNUSED *new_compressed_len, unsigned *flags) + haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len, unsigned *flags) { H5FS_sinfo_t *sinfo = (H5FS_sinfo_t *)_thing; /* Pointer to the object */ H5FS_t *fspace; /* Free space header */ @@ -1186,6 +1245,68 @@ done: /*------------------------------------------------------------------------- + * Function: H5FS__cache_sinfo_notify + * + * Purpose: Handle cache action notifications + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Fall 2012 + * + *------------------------------------------------------------------------- + */ +herr_t +H5FS__cache_sinfo_notify(H5AC_notify_action_t action, void *_thing) +{ + H5FS_sinfo_t *sinfo = (H5FS_sinfo_t *)_thing; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Sanity check */ + HDassert(sinfo); + + /* Check if the file was opened with SWMR-write access */ + if(sinfo->fspace->swmr_write) { + /* Determine which action to take */ + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + case H5AC_NOTIFY_ACTION_AFTER_LOAD: + /* Create flush dependency on parent */ + if(H5FS__create_flush_depend((H5AC_info_t *)sinfo->fspace, (H5AC_info_t *)sinfo) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTDEPEND, FAIL, "unable to create flush dependency between data block and header, address = %llu", (unsigned long long)sinfo->fspace->sect_addr) + break; + + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + /* Destroy flush dependency on parent */ + if(H5FS__destroy_flush_depend((H5AC_info_t *)sinfo->fspace, (H5AC_info_t *)sinfo) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTUNDEPEND, FAIL, "unable to destroy flush dependency") + break; + + default: +#ifdef NDEBUG + HGOTO_ERROR(H5E_FSPACE, H5E_BADVALUE, FAIL, "unknown action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Unknown action?!?"); +#endif /* NDEBUG */ + } /* end switch */ + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FS__cache_sinfo_notify() */ + + +/*------------------------------------------------------------------------- * Function: H5FS__cache_sinfo_free_icr * * Purpose: Free the memory used for the in core representation of the diff --git a/src/H5FSint.c b/src/H5FSint.c new file mode 100644 index 0000000..60cedd5 --- /dev/null +++ b/src/H5FSint.c @@ -0,0 +1,145 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5FSint.c + * Fall 2012 + * Dana Robinson <derobins@hdfgroup.org> + * + * Purpose: Internal routines for free space managers. + * + *------------------------------------------------------------------------- + */ + +/**********************/ +/* Module Declaration */ +/**********************/ + +#include "H5FSmodule.h" /* This source code file is part of the H5FS module */ + + +/***********************/ +/* Other Packages Used */ +/***********************/ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error Handling */ +#include "H5FSpkg.h" /* Free Space Managers */ + + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + + + +/*------------------------------------------------------------------------- + * Function: H5FS__create_flush_depend + * + * Purpose: Create a flush dependency between two data structure components + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Fall 2012 + * + *------------------------------------------------------------------------- + */ +herr_t +H5FS__create_flush_depend(H5AC_info_t *parent_entry, H5AC_info_t *child_entry) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Sanity check */ + HDassert(parent_entry); + HDassert(child_entry); + + /* Create a flush dependency between parent and child entry */ + if(H5AC_create_flush_dependency(parent_entry, child_entry) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTDEPEND, FAIL, "unable to create flush dependency") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FS__create_flush_depend() */ + + +/*------------------------------------------------------------------------- + * Function: H5FS__destroy_flush_depend + * + * Purpose: Destroy a flush dependency between two data structure components + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Fall 2012 + * + *------------------------------------------------------------------------- + */ +herr_t +H5FS__destroy_flush_depend(H5AC_info_t *parent_entry, H5AC_info_t *child_entry) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Sanity check */ + HDassert(parent_entry); + HDassert(child_entry); + + /* Destroy a flush dependency between parent and child entry */ + if(H5AC_destroy_flush_dependency(parent_entry, child_entry) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTUNDEPEND, FAIL, "unable to destroy flush dependency") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FS__destroy_flush_depend() */ + diff --git a/src/H5FSpkg.h b/src/H5FSpkg.h index 7e52b5f..b3b1548 100644 --- a/src/H5FSpkg.h +++ b/src/H5FSpkg.h @@ -179,6 +179,7 @@ struct H5FS_t { haddr_t addr; /* Address of free space header on disk */ size_t hdr_size; /* Size of free space header on disk */ H5FS_sinfo_t *sinfo; /* Section information */ + hbool_t swmr_write; /* Flag indicating the file is opened with SWMR-write access */ unsigned sinfo_lock_count; /* # of times the section info has been locked */ hbool_t sinfo_protected; /* Whether the section info was protected when locked */ hbool_t sinfo_modified; /* Whether the section info has been modified while locked */ @@ -222,6 +223,12 @@ H5FL_EXTERN(H5FS_t); /* Package Private Prototypes */ /******************************/ +/* Generic routines */ +H5_DLL herr_t H5FS__create_flush_depend(H5AC_info_t *parent_entry, + H5AC_info_t *child_entry); +H5_DLL herr_t H5FS__destroy_flush_depend(H5AC_info_t *parent_entry, + H5AC_info_t *child_entry); + /* Free space manager header routines */ H5_DLL H5FS_t *H5FS__new(const H5F_t *f, uint16_t nclasses, const H5FS_section_class_t *classes[], void *cls_init_udata); diff --git a/src/H5FSsection.c b/src/H5FSsection.c index 4697bd50..766a823 100644 --- a/src/H5FSsection.c +++ b/src/H5FSsection.c @@ -208,7 +208,7 @@ H5FS_sinfo_lock(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, unsigned accmode) H5FS_sinfo_cache_ud_t cache_udata; /* User-data for cache callback */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) + FUNC_ENTER_NOAPI_NOINIT #ifdef H5FS_SINFO_DEBUG HDfprintf(stderr, "%s: Called, fspace->addr = %a, fspace->sinfo = %p, fspace->sect_addr = %a\n", FUNC, fspace->addr, fspace->sinfo, fspace->sect_addr); @@ -298,7 +298,7 @@ done: HDfprintf(stderr, "%s: Leaving, fspace->addr = %a, fspace->sinfo = %p, fspace->sect_addr = %a\n", FUNC, fspace->addr, fspace->sinfo, fspace->sect_addr); HDfprintf(stderr, "%s: fspace->alloc_sect_size = %Hu, fspace->sect_size = %Hu\n", FUNC, fspace->alloc_sect_size, fspace->sect_size); #endif /* H5FS_SINFO_DEBUG */ - FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) + FUNC_LEAVE_NOAPI(ret_value) } /* H5FS_sinfo_lock() */ diff --git a/src/H5Faccum.c b/src/H5Faccum.c index 3fac184..48f9bdd 100644 --- a/src/H5Faccum.c +++ b/src/H5Faccum.c @@ -726,6 +726,12 @@ H5F__accum_write(const H5F_io_info_t *fio_info, H5FD_mem_t map_type, haddr_t add } /* end else */ } /* end if */ else { + /* Make certain that data in accumulator is visible before new write */ + if((H5F_INTENT(fio_info->f) & H5F_ACC_SWMR_WRITE) > 0) + /* Flush if dirty and reset accumulator */ + if(H5F__accum_reset(fio_info, TRUE) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTRESET, FAIL, "can't reset accumulator") + /* Write the data */ if(H5FD_write(fio_info->f->shared->lf, fio_info->dxpl, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") diff --git a/src/H5Fefc.c b/src/H5Fefc.c index 5e3deb1..42bf5d8 100644 --- a/src/H5Fefc.c +++ b/src/H5Fefc.c @@ -301,7 +301,7 @@ done: if(ent) { if(open_file) { ent->file->nopen_objs--; - if(H5F_try_close(ent->file) < 0) + if(H5F_try_close(ent->file, NULL) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, NULL, "can't close external file") } /* end if */ ent->name = (char *)H5MM_xfree(ent->name); @@ -350,7 +350,7 @@ H5F_efc_close(H5F_t *parent, H5F_t *file) * on the state of the efc. */ if(!efc) { file->nopen_objs--; - if(H5F_try_close(file) < 0) + if(H5F_try_close(file, NULL) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "can't close external file") HGOTO_DONE(SUCCEED) @@ -364,7 +364,7 @@ H5F_efc_close(H5F_t *parent, H5F_t *file) for(ent = efc->LRU_head; ent && ent->file != file; ent = ent->LRU_next); if(!ent) { file->nopen_objs--; - if(H5F_try_close(file) < 0) + if(H5F_try_close(file, NULL) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "can't close external file") } /* end if */ else @@ -572,7 +572,7 @@ H5F_efc_remove_ent(H5F_efc_t *efc, H5F_efc_ent_t *ent) * However we must still manipulate the nopen_objs field to prevent the file * from being closed out from under us. */ ent->file->nopen_objs--; - if(H5F_try_close(ent->file) < 0) + if(H5F_try_close(ent->file, NULL) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "can't close external file") ent->file = NULL; diff --git a/src/H5Fint.c b/src/H5Fint.c index 80c593b..fd79dc2 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -171,6 +171,8 @@ H5F_get_access_plist(H5F_t *f, hbool_t app_ref) latest_format = TRUE; if(H5P_set(new_plist, H5F_ACS_LATEST_FORMAT_NAME, &latest_format) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set 'latest format' flag") + if(H5P_set(new_plist, H5F_ACS_METADATA_READ_ATTEMPTS_NAME, &(f->shared->read_attempts)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set 'read attempts ' flag") if(H5P_set(new_plist, H5F_ACS_OBJECT_FLUSH_CB_NAME, &(f->shared->object_flush)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set object flush callback") @@ -639,9 +641,13 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get sieve buffer size") if(H5P_get(plist, H5F_ACS_LATEST_FORMAT_NAME, &latest_format) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get 'latest format' flag") - /* For latest format, activate all latest version support */ - if(latest_format) + /* For latest format or SWMR_WRITE, activate all latest version support */ + if(latest_format || (H5F_INTENT(f) & H5F_ACC_SWMR_WRITE)) f->shared->latest_flags |= H5F_LATEST_ALL_FLAGS; + if(H5P_get(plist, H5F_ACS_USE_MDC_LOGGING_NAME, &(f->shared->use_mdc_logging)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get 'use mdc logging' flag") + if(H5P_get(plist, H5F_ACS_START_MDC_LOG_ON_ACCESS_NAME, &(f->shared->start_mdc_log_on_access)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get 'start mdc log on access' flag") if(H5P_get(plist, H5F_ACS_META_BLOCK_SIZE_NAME, &(f->shared->meta_aggr.alloc_size)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get metadata cache size") f->shared->meta_aggr.feature_flag = H5FD_FEAT_AGGREGATE_METADATA; @@ -666,6 +672,15 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad maximum address from VFD") if(H5FD_get_feature_flags(lf, &f->shared->feature_flags) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get feature flags from VFD") + + /* Require the SWMR feature flag if SWMR I/O is desired */ + if(!H5F_HAS_FEATURE(f, H5FD_FEAT_SUPPORTS_SWMR_IO) && (H5F_INTENT(f) & (H5F_ACC_SWMR_WRITE | H5F_ACC_SWMR_READ))) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "must use a SWMR-compatible VFD when SWMR is specified") + + /* Require a POSIX compatible VFD to use SWMR feature */ + /* (It's reasonable to try to expand this to other VFDs eventually -QAK) */ + if(!H5F_HAS_FEATURE(f, H5FD_FEAT_POSIX_COMPAT_HANDLE) && (H5F_INTENT(f) & (H5F_ACC_SWMR_WRITE | H5F_ACC_SWMR_READ))) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "must use POSIX compatible VFD with SWMR write access") if(H5FD_get_fs_type_map(lf, f->shared->fs_type_map) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get free space type mapping from VFD") if(H5MF_init_merge_flags(f) < 0) @@ -682,6 +697,48 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t */ f->shared->use_tmp_space = !H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI); + /* Retrieve the # of read attempts here so that sohm in superblock will get the correct # of attempts */ + if(H5P_get(plist, H5F_ACS_METADATA_READ_ATTEMPTS_NAME, &f->shared->read_attempts) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get the # of read attempts") + + /* When opening file with SWMR access, the # of read attempts is H5F_SWMR_METADATA_READ_ATTEMPTS if not set */ + /* When opening file without SWMR access, the # of read attempts is always H5F_METADATA_READ_ATTEMPTS (set or not set) */ + if(H5F_INTENT(f) & (H5F_ACC_SWMR_READ | H5F_ACC_SWMR_WRITE)) { + /* If no value for read attempts has been set, use the default */ + if(!f->shared->read_attempts) + f->shared->read_attempts = H5F_SWMR_METADATA_READ_ATTEMPTS; + + /* Turn off accumulator with SWMR */ + f->shared->feature_flags &= ~(unsigned)H5FD_FEAT_ACCUMULATE_METADATA; + if(H5FD_set_feature_flags(f->shared->lf, f->shared->feature_flags) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, NULL, "can't set feature_flags in VFD") + } /* end if */ + else { + /* If no value for read attempts has been set, use the default */ + if(!f->shared->read_attempts) + f->shared->read_attempts = H5F_METADATA_READ_ATTEMPTS; + } /* end else */ + + /* Determine the # of bins for metdata read retries */ + if(H5F_set_retries(f) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't set retries and retries_nbins") + + /* Get the metadata cache log location (if we're logging) */ + { + char *mdc_log_location = NULL; /* location of metadata cache log location */ + + if(H5P_get(plist, H5F_ACS_MDC_LOG_LOCATION_NAME, &mdc_log_location) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get mdc log location") + if(mdc_log_location != NULL) { + size_t len = HDstrlen(mdc_log_location); + if(NULL == (f->shared->mdc_log_location = (char *)H5MM_calloc((len + 1) * sizeof(char)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "can't allocate memory for mdc log file name") + HDstrncpy(f->shared->mdc_log_location, mdc_log_location, len); + } + else + f->shared->mdc_log_location = NULL; + } /* end block */ + /* Get object flush callback information */ if(H5P_get(plist, H5F_ACS_OBJECT_FLUSH_CB_NAME, &(f->shared->object_flush)) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get object flush cb info") @@ -760,6 +817,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) HDassert(f->shared); if(1 == f->shared->nrefs) { + int actype; /* metadata cache type (enum value) */ H5F_io_info_t fio_info; /* I/O info for operation */ /* Flush at this point since the file will be closed. @@ -796,10 +854,19 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) /* Flush the file again (if requested), as shutting down the * free space manager may dirty some data structures again. */ - if(flush) + if(flush) { + /* Clear status_flags */ + f->shared->sblock->status_flags &= (uint8_t)(~H5F_SUPER_WRITE_ACCESS); + f->shared->sblock->status_flags &= (uint8_t)(~H5F_SUPER_SWMR_WRITE_ACCESS); + /* Mark superblock dirty in cache, so change will get encoded */ + /* Push error, but keep going*/ + if(H5F_super_dirty(f) < 0) + HDONE_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty") + if(H5F_flush(f, dxpl_id, TRUE) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache") + } /* end if */ } /* end if */ /* if it exists, unpin the driver information block cache entry, @@ -827,6 +894,10 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing file") + /* Clean up the metadata cache log location string */ + if(f->shared->mdc_log_location) + f->shared->mdc_log_location = (char *)H5MM_xfree(f->shared->mdc_log_location); + /* * Do not close the root group since we didn't count it, but free * the memory associated with it. @@ -873,6 +944,11 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) f->shared->mtab.child = (H5F_mount_t *)H5MM_xfree(f->shared->mtab.child); f->shared->mtab.nalloc = 0; + /* Clean up the metadata retries array */ + for(actype = 0; actype < (int)H5AC_NTYPES; actype++) + if(f->shared->retries[actype]) + f->shared->retries[actype] = (uint32_t *)H5MM_xfree(f->shared->retries[actype]); + /* Destroy shared file struct */ f->shared = (H5F_file_t *)H5FL_FREE(H5F_file_t, f->shared); @@ -935,6 +1011,37 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) * The ACCESS_PARMS argument is optional. A null pointer will * cause the default file access parameters to be used. * + * The following two tables show results of file opens for single and concurrent access: + * + * SINGLE PROCESS ACCESS CONCURRENT ACCESS + * + * #1st open# #1st open# + * -- SR SR -- -- SR SR -- -- SR SR -- -- SR SR -- + * -- -- SW SW SW SW -- -- -- -- SW SW SW SW -- -- + * W W W W R R R R W W W W R R R R + * #2nd open# #2nd open# + * -------------------------- -------------------------- + * -- -- W | s x x s x x f f | -- -- W | f x x f x x f f | + * SR -- W | x x x x x x x x | SR -- W | x x x x x x x x | + * SR SW W | x x x x x x x x | SR SW W | x x x x x x x x | + * -- SW W | f x x s x x f f | -- SW W | f x x f x x f f | + * -- SW R | x x x x x x x x | -- SW R | x x x x x x x x | + * SR SW R | x x x x x x x x | SR SW R | x x x x x x x x | + * SR -- R | s x x s x x s f | SR -- R | f x x s x x s s | + * -- -- R | s x x s x x s s | -- -- R | f x x f x x s s | + * -------------------------- -------------------------- + * + * Notations: + * W: H5F_ACC_RDWR + * R: H5F_ACC_RDONLY + * SW: H5F_ACC_SWMR_WRITE + * SR: H5F_ACC_SWMR_READ + * + * x: the first open or second open itself fails due to invalid flags combination + * f: the open fails with flags combination from both the first and second opens + * s: the open succeeds with flags combination from both the first and second opens + * + * * Return: Success: A new file pointer. * Failure: NULL * @@ -954,7 +1061,12 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_class_t *drvr; /*file driver class info */ H5P_genplist_t *a_plist; /*file access property list */ H5F_close_degree_t fc_degree; /*file close degree */ + hbool_t set_flag = FALSE; /*set the status_flags in the superblock */ + hbool_t clear = FALSE; /*clear the status_flags */ + hbool_t evict_on_close; /* evict on close value from plist */ H5F_t *ret_value = NULL; /*actual return value */ + char *lock_env_var = NULL;/*env var pointer */ + hbool_t use_file_locking; /*read from env var */ FUNC_ENTER_NOAPI(NULL) @@ -969,6 +1081,16 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, if(NULL == (drvr = H5FD_get_class(fapl_id))) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to retrieve VFL class") + /* Check the environment variable that determines if we care + * about file locking. File locking should be used unless explicitly + * disabled. + */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && !HDstrcmp(lock_env_var, "FALSE")) + use_file_locking = FALSE; + else + use_file_locking = TRUE; + /* * Opening a file is a two step process. First we try to open the * file in a way which doesn't affect its state (like not truncating @@ -981,52 +1103,58 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, * way for us to detect it here anyway). */ if(drvr->cmp) - tent_flags = flags & ~(H5F_ACC_CREAT|H5F_ACC_TRUNC|H5F_ACC_EXCL); + tent_flags = flags & ~(H5F_ACC_CREAT|H5F_ACC_TRUNC|H5F_ACC_EXCL); else - tent_flags = flags; + tent_flags = flags; if(NULL == (lf = H5FD_open(name, tent_flags, fapl_id, HADDR_UNDEF))) { - if(tent_flags == flags) { + if(tent_flags == flags) { #ifndef H5_USING_MEMCHECKER time_t mytime = HDtime(NULL); - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file: time = %s, name = '%s', tent_flags = %x", HDctime(&mytime), name, tent_flags) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file: time = %s, name = '%s', tent_flags = %x", HDctime(&mytime), name, tent_flags) #else /* H5_USING_MEMCHECKER */ - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file: name = '%s', tent_flags = %x", name, tent_flags) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file: name = '%s', tent_flags = %x", name, tent_flags) #endif /* H5_USING_MEMCHECKER */ } /* end if */ H5E_clear_stack(NULL); - tent_flags = flags; - if(NULL == (lf = H5FD_open(name, tent_flags, fapl_id, HADDR_UNDEF))) { + tent_flags = flags; + if(NULL == (lf = H5FD_open(name, tent_flags, fapl_id, HADDR_UNDEF))) { #ifndef H5_USING_MEMCHECKER time_t mytime = HDtime(NULL); - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file: time = %s, name = '%s', tent_flags = %x", HDctime(&mytime), name, tent_flags) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file: time = %s, name = '%s', tent_flags = %x", HDctime(&mytime), name, tent_flags) #else /* H5_USING_MEMCHECKER */ - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file: name = '%s', tent_flags = %x", name, tent_flags) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file: name = '%s', tent_flags = %x", name, tent_flags) #endif /* H5_USING_MEMCHECKER */ } /* end if */ } /* end if */ /* Is the file already open? */ if((shared = H5F_sfile_search(lf)) != NULL) { - /* - * The file is already open, so use that one instead of the one we - * just opened. We only one one H5FD_t* per file so one doesn't - * confuse the other. But fail if this request was to truncate the - * file (since we can't do that while the file is open), or if the - * request was to create a non-existent file (since the file already - * exists), or if the new request adds write access (since the - * readers don't expect the file to change under them). - */ - if(H5FD_close(lf) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to close low-level file info") - if(flags & H5F_ACC_TRUNC) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to truncate a file which is already open") - if(flags & H5F_ACC_EXCL) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "file exists") - if((flags & H5F_ACC_RDWR) && 0 == (shared->flags & H5F_ACC_RDWR)) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "file is already open for read-only") + /* + * The file is already open, so use that one instead of the one we + * just opened. We only one one H5FD_t* per file so one doesn't + * confuse the other. But fail if this request was to truncate the + * file (since we can't do that while the file is open), or if the + * request was to create a non-existent file (since the file already + * exists), or if the new request adds write access (since the + * readers don't expect the file to change under them), or if the + * SWMR write/read access flags don't agree. + */ + if(H5FD_close(lf) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to close low-level file info") + if(flags & H5F_ACC_TRUNC) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to truncate a file which is already open") + if(flags & H5F_ACC_EXCL) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "file exists") + if((flags & H5F_ACC_RDWR) && 0 == (shared->flags & H5F_ACC_RDWR)) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "file is already open for read-only") + + if((flags & H5F_ACC_SWMR_WRITE) && 0 == (shared->flags & H5F_ACC_SWMR_WRITE)) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "SWMR write access flag not the same for file that is already open") + if((flags & H5F_ACC_SWMR_READ) && !((shared->flags & H5F_ACC_SWMR_WRITE) || (shared->flags & H5F_ACC_SWMR_READ) || (shared->flags & H5F_ACC_RDWR))) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "SWMR read access flag not the same for file that is already open") /* Allocate new "high-level" file struct */ if((file = H5F_new(shared, flags, fcpl_id, fapl_id, NULL)) == NULL) @@ -1040,18 +1168,36 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, * open it are different than the desired flags. Close the tentative * file and open it for real. */ - if(H5FD_close(lf) < 0) { - file = NULL; /*to prevent destruction of wrong file*/ + if(H5FD_close(lf) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to close low-level file info") - } /* end if */ - if(NULL == (lf = H5FD_open(name, flags, fapl_id, HADDR_UNDEF))) { - file = NULL; /*to prevent destruction of wrong file*/ + + if(NULL == (lf = H5FD_open(name, flags, fapl_id, HADDR_UNDEF))) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file") + } /* end if */ + + /* Place an advisory lock on the file */ + if(use_file_locking) + if(H5FD_lock(lf, (hbool_t)((flags & H5F_ACC_RDWR) ? TRUE : FALSE)) < 0) { + /* Locking failed - Closing will remove the lock */ + if(H5FD_close(lf) < 0) + HDONE_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to close low-level file info") + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to lock the file") } /* end if */ + + /* Create the 'top' file structure */ + if(NULL == (file = H5F_new(NULL, flags, fcpl_id, fapl_id, lf))) { + /* If this is the only time the file has been opened and the struct + * returned is NULL, H5FD_close() will never be called via H5F_dest() + * so we have to close lf here before heading to the error handling. + */ + if(H5FD_close(lf) < 0) + HDONE_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to close low-level file info") + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to initialize file structure") } /* end if */ - if(NULL == (file = H5F_new(NULL, flags, fcpl_id, fapl_id, lf))) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to create new file object") + /* Need to set status_flags in the superblock if the driver has a 'lock' method */ + if(drvr->lock) + set_flag = TRUE; } /* end else */ /* Retain the name the file was opened with */ @@ -1082,15 +1228,16 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, */ if(H5G_mkroot(file, dxpl_id, TRUE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to create/open root group") - } else if (1 == shared->nrefs) { + } /* end if */ + else if (1 == shared->nrefs) { - /* Read the superblock if it hasn't been read before. */ + /* Read the superblock if it hasn't been read before. */ if(H5F__super_read(file, dxpl_id, TRUE) < 0) - HGOTO_ERROR(H5E_FILE, H5E_READERROR, NULL, "unable to read superblock") + HGOTO_ERROR(H5E_FILE, H5E_READERROR, NULL, "unable to read superblock") - /* Open the root group */ - if(H5G_mkroot(file, dxpl_id, FALSE) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to read root group") + /* Open the root group */ + if(H5G_mkroot(file, dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to read root group") } /* end if */ /* Get the file access property list, for future queries */ @@ -1106,34 +1253,105 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, if(H5P_get(a_plist, H5F_ACS_CLOSE_DEGREE_NAME, &fc_degree) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get file close degree") + /* This is a private property to clear the status_flags in the super block */ + /* Use by h5clear and a routine in test/flush2.c to clear the test file's status_flags */ + if(H5P_exist_plist(a_plist, H5F_ACS_CLEAR_STATUS_FLAGS_NAME) > 0) { + if(H5P_get(a_plist, H5F_ACS_CLEAR_STATUS_FLAGS_NAME, &clear) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get clearance for status_flags") + else if(clear) + file->shared->sblock->status_flags = 0; + } /* end if */ + if(shared->nrefs == 1) { if(fc_degree == H5F_CLOSE_DEFAULT) shared->fc_degree = lf->cls->fc_degree; else shared->fc_degree = fc_degree; - } else if(shared->nrefs > 1) { + } /* end if */ + else if(shared->nrefs > 1) { if(fc_degree == H5F_CLOSE_DEFAULT && shared->fc_degree != lf->cls->fc_degree) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "file close degree doesn't match") if(fc_degree != H5F_CLOSE_DEFAULT && fc_degree != shared->fc_degree) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "file close degree doesn't match") } /* end if */ + /* Record the evict-on-close MDC behavior. If it's the first time opening + * the file, set it to access property list value; if it's the second time + * or later, verify that the access property list value matches the value + * in shared file structure. + */ + if(H5P_get(a_plist, H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME, &evict_on_close) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get evict on close value") + + if(shared->nrefs == 1) { + shared->evict_on_close = evict_on_close; + } else if(shared->nrefs > 1) { + if(shared->evict_on_close != evict_on_close) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "file evict-on-close value doesn't match") + } /* end if */ + /* Formulate the absolute path for later search of target file for external links */ if(H5_build_extpath(name, &file->extpath) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to build extpath") + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to build extpath") /* Formulate the actual file name, after following symlinks, etc. */ if(H5F_build_actual_name(file, a_plist, name, &file->actual_name) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to build actual name") + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to build actual name") + + if(set_flag) { + if(H5F_INTENT(file) & H5F_ACC_RDWR) { /* Set and check consistency of status_flags */ + /* Skip check of status_flags for file with < superblock version 3 */ + if(file->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_3) { + + if(file->shared->sblock->status_flags & H5F_SUPER_WRITE_ACCESS || + file->shared->sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "file is already open for write/SWMR write (may use <h5clear file> to clear file consistency flags)") + } /* version 3 superblock */ + + file->shared->sblock->status_flags |= H5F_SUPER_WRITE_ACCESS; + if(H5F_INTENT(file) & H5F_ACC_SWMR_WRITE) + file->shared->sblock->status_flags |= H5F_SUPER_SWMR_WRITE_ACCESS; + + /* Flush the superblock */ + if(H5F_super_dirty(file) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, NULL, "unable to mark superblock as dirty") + if(H5F_flush_tagged_metadata(file, H5AC__SUPERBLOCK_TAG, H5AC_ind_read_dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, NULL, "unable to flush superblock") + + /* Remove the file lock for SWMR_WRITE */ + if(use_file_locking && (H5F_INTENT(file) & H5F_ACC_SWMR_WRITE)) { + if(H5FD_unlock(file->shared->lf) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to unlock the file") + } /* end if */ + } /* end if */ + else { /* H5F_ACC_RDONLY: check consistency of status_flags */ + /* Skip check of status_flags for file with < superblock version 3 */ + if(file->shared->sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_3) { + + if(H5F_INTENT(file) & H5F_ACC_SWMR_READ) { + if((file->shared->sblock->status_flags & H5F_SUPER_WRITE_ACCESS && + !(file->shared->sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS)) + || + (!(file->shared->sblock->status_flags & H5F_SUPER_WRITE_ACCESS) && + file->shared->sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS)) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "file is not already open for SWMR writing") + + } /* end if */ + else if((file->shared->sblock->status_flags & H5F_SUPER_WRITE_ACCESS) || + (file->shared->sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS)) + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "file is already open for write (may use <h5clear file> to clear file consistency flags)") + + } /* version 3 superblock */ + } /* end else */ + } /* end if set_flag */ /* Success */ ret_value = file; done: - if(!ret_value && file) - if(H5F_dest(file, dxpl_id, FALSE) < 0) - HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, NULL, "problems closing file") - + if((NULL == ret_value) && file) + if(H5F_dest(file, dxpl_id, FALSE) < 0) + HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, NULL, "problems closing file") FUNC_LEAVE_NOAPI(ret_value) } /* end H5F_open() */ @@ -1268,7 +1486,7 @@ H5F_close(H5F_t *f) f->file_id = -1; /* Attempt to close the file/mount hierarchy */ - if(H5F_try_close(f) < 0) + if(H5F_try_close(f, NULL) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "can't close file") done: @@ -1292,7 +1510,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F_try_close(H5F_t *f) +H5F_try_close(H5F_t *f, hbool_t *was_closed /*out*/) { unsigned nopen_files = 0; /* Number of open files in file/mount hierarchy */ unsigned nopen_objs = 0; /* Number of open objects in file/mount hierarchy */ @@ -1304,9 +1522,21 @@ H5F_try_close(H5F_t *f) HDassert(f); HDassert(f->shared); + /* Set the was_closed flag to the default value. + * This flag lets downstream code know if the file struct is + * still accessible and/or likely to contain useful data. + * It's needed by the evict-on-close code. Clients can ignore + * this value by passing in NULL. + */ + if(was_closed) + *was_closed = FALSE; + /* Check if this file is already in the process of closing */ - if(f->closing) + if(f->closing) { + if(was_closed) + *was_closed = TRUE; HGOTO_DONE(SUCCEED) + } /* end if */ /* Get the number of open objects and open files on this file/mount hierarchy */ if(H5F_mount_count_ids(f, &nopen_files, &nopen_objs) < 0) @@ -1407,7 +1637,7 @@ H5F_try_close(H5F_t *f) * hierarchy if so. */ if(f->parent) - if(H5F_try_close(f->parent) < 0) + if(H5F_try_close(f->parent, NULL) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "can't close parent file") /* Unmount and close each child before closing the current file. */ @@ -1433,6 +1663,9 @@ H5F_try_close(H5F_t *f) if(H5F_dest(f, H5AC_ind_read_dxpl_id, TRUE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "problems closing file") + /* Since we closed the file, this should be set to TRUE */ + if(was_closed) + *was_closed = TRUE; done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F_try_close() */ @@ -2053,6 +2286,8 @@ H5F_get_file_image(H5F_t *file, void *buf_ptr, size_t buf_len, hid_t dxpl_id) /* test to see if a buffer was provided -- if not, we are done */ if(buf_ptr != NULL) { size_t space_needed; /* size of file image */ + hsize_t tmp; + size_t tmp_size; H5P_genplist_t *xfer_plist= NULL; /* Dataset transfer property list object */ /* Check for buffer too small */ @@ -2069,6 +2304,15 @@ H5F_get_file_image(H5F_t *file, void *buf_ptr, size_t buf_len, hid_t dxpl_id) /* (Note compensation for base address addition in internal routine) */ if(H5FD_read(fd_ptr, xfer_plist, H5FD_MEM_DEFAULT, 0, space_needed, buf_ptr) < 0) HGOTO_ERROR(H5E_FILE, H5E_READERROR, FAIL, "file image read request failed") + + /* Offset to "status_flags" in the superblock */ + tmp = H5F_SUPER_STATUS_FLAGS_OFF(file->shared->sblock->super_vers); + /* Size of "status_flags" depends on the superblock version */ + tmp_size = H5F_SUPER_STATUS_FLAGS_SIZE(file->shared->sblock->super_vers); + + /* Clear "status_flags" */ + HDmemset((uint8_t *)(buf_ptr) + tmp, 0, tmp_size); + } /* end if */ done: @@ -2077,6 +2321,96 @@ done: /*------------------------------------------------------------------------- + * Function: H5F_track_metadata_read_retries + * + * Purpose: To track the # of a "retries" (log10) for a metadata item. + * This routine should be used only when: + * "retries" > 0 + * f->shared->read_attempts > 1 (does not have retry when 1) + * f->shared->retries_nbins > 0 (calculated based on f->shared->read_attempts) + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Vailin Choi; October 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_track_metadata_read_retries(H5F_t *f, unsigned actype, unsigned retries) +{ + unsigned log_ind; /* Index to the array of retries based on log10 of retries */ + double tmp; /* Temporary value, to keep compiler quiet */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(f); + HDassert(f->shared->read_attempts > 1); + HDassert(f->shared->retries_nbins > 0); + HDassert(retries > 0); + HDassert(retries < f->shared->read_attempts); + HDassert(actype < H5AC_NTYPES); + + /* Allocate memory for retries */ + if(NULL == f->shared->retries[actype]) + if(NULL == (f->shared->retries[actype] = (uint32_t *)H5MM_calloc((size_t)f->shared->retries_nbins * sizeof(uint32_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + + /* Index to retries based on log10 */ + tmp = HDlog10((double)retries); + log_ind = (unsigned)tmp; + HDassert(log_ind < f->shared->retries_nbins); + + /* Increment the # of the "retries" */ + f->shared->retries[actype][log_ind]++; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5F_track_metadata_read_retries() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_set_retries + * + * Purpose: To initialize data structures for read retries: + * --zero out "retries" + * --set up "retries_nbins" based on read_attempts + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Vailin Choi; November 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_set_retries(H5F_t *f) +{ + double tmp; /* Temporary variable */ + + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity check */ + HDassert(f); + + /* Initialize the tracking for metadata read retries */ + HDmemset(f->shared->retries, 0, sizeof(f->shared->retries)); + + /* Initialize the # of bins for retries */ + f->shared->retries_nbins = 0; + if(f->shared->read_attempts > 1) { + tmp = HDlog10((double)(f->shared->read_attempts - 1)); + f->shared->retries_nbins = (unsigned)tmp + 1; + } + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5F_set_retries() */ + + +/*------------------------------------------------------------------------- * Function: H5F_object_flush_cb * * Purpose: To invoke the callback function for object flush that is set diff --git a/src/H5Fio.c b/src/H5Fio.c index b9bd354..e215666 100644 --- a/src/H5Fio.c +++ b/src/H5Fio.c @@ -264,21 +264,108 @@ H5F_evict_tagged_metadata(H5F_t * f, haddr_t tag, hid_t dxpl_id) FUNC_ENTER_NOAPI(FAIL) - /* Unpin the superblock, as this will be marked for eviction and it can't - be pinned. */ - if(H5AC_unpin_entry(f->shared->sblock) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPIN, FAIL, "unable to unpin superblock") - f->shared->sblock = NULL; - /* Evict the object's metadata */ - if(H5AC_evict_tagged_metadata(f, tag, dxpl_id)<0) + if(H5AC_evict_tagged_metadata(f, tag, TRUE, dxpl_id) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTEXPUNGE, FAIL, "unable to evict tagged metadata") - /* Re-read the superblock. */ - if(H5F__super_read(f, dxpl_id, FALSE) < 0) - HGOTO_ERROR(H5E_FILE, H5E_READERROR, FAIL, "unable to read superblock") - done: FUNC_LEAVE_NOAPI(ret_value); } /* end H5F_evict_tagged_metadata */ + +/*------------------------------------------------------------------------- + * Function: H5F__evict_cache_entries + * + * Purpose: To evict all cache entries except the pinned superblock entry + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * Dec 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F__evict_cache_entries(H5F_t *f, hid_t dxpl_id) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + HDassert(f); + HDassert(f->shared); + + /* Evict all except pinned entries in the cache */ + if(H5AC_evict(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTEXPUNGE, FAIL, "unable to evict all except pinned entries") + +#ifndef NDEBUG +{ + unsigned status = 0; + int32_t cur_num_entries; + + /* Retrieve status of the superblock */ + if(H5AC_get_entry_status(f, (haddr_t)0, &status) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "unable to get entry status") + + /* Verify status of the superblock entry in the cache */ + if(!(status & H5AC_ES__IN_CACHE) || !(status & H5AC_ES__IS_PINNED)) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "unable to get entry status") + + /* Get the number of cache entries */ + if(H5AC_get_cache_size(f->shared->cache, NULL, NULL, NULL, &cur_num_entries) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5AC_get_cache_size() failed.") + + /* Should be the only one left in the cache (the superblock) */ + if(cur_num_entries != 1) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "number of cache entries is not correct") +} +#endif /* NDEBUG */ + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5F__evict_cache_entries() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_get_checksums + * + * Purpose: Decode checksum stored in the buffer + * Calculate checksum for the data in the buffer + * + * Note: Assumes that the checksum is the last data in the buffer + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * Sept 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_get_checksums(const uint8_t *buf, size_t buf_size, uint32_t *s_chksum/*out*/, uint32_t *c_chksum/*out*/) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check arguments */ + HDassert(buf); + HDassert(buf_size); + + /* Return the stored checksum */ + if(s_chksum) { + const uint8_t *chk_p; /* Pointer into raw data buffer */ + + /* Offset to the checksum in the buffer */ + chk_p = buf + buf_size - H5_SIZEOF_CHKSUM; + + /* Decode the checksum stored in the buffer */ + UINT32DECODE(chk_p, *s_chksum); + } /* end if */ + + /* Return the computed checksum for the buffer */ + if(c_chksum) + *c_chksum = H5_checksum_metadata(buf, buf_size - H5_SIZEOF_CHKSUM, 0); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5F_get_chksums() */ + diff --git a/src/H5Fmount.c b/src/H5Fmount.c index adde071..e3d4952 100644 --- a/src/H5Fmount.c +++ b/src/H5Fmount.c @@ -69,7 +69,7 @@ H5F_close_mounts(H5F_t *f) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "can't close child group") /* Close the child file */ - if(H5F_try_close(f->shared->mtab.child[u].file) < 0) + if(H5F_try_close(f->shared->mtab.child[u].file, NULL) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "can't close child file") /* Eliminate the mount point from the table */ @@ -386,7 +386,7 @@ H5F_unmount(H5G_loc_t *loc, const char *name, hid_t dxpl_id) /* Detach child file from parent & see if it should close */ child->parent = NULL; - if(H5F_try_close(child) < 0) + if(H5F_try_close(child, NULL) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close unmounted file") done: diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index c5aed3b..1adf74b 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -56,10 +56,11 @@ /* Superblock status flags */ #define H5F_SUPER_WRITE_ACCESS 0x01 #define H5F_SUPER_FILE_OK 0x02 -#define H5F_SUPER_ALL_FLAGS (H5F_SUPER_WRITE_ACCESS | H5F_SUPER_FILE_OK) +#define H5F_SUPER_SWMR_WRITE_ACCESS 0x04 +#define H5F_SUPER_ALL_FLAGS (H5F_SUPER_WRITE_ACCESS | H5F_SUPER_FILE_OK | H5F_SUPER_SWMR_WRITE_ACCESS) /* Mask for removing private file access flags */ -#define H5F_ACC_PUBLIC_FLAGS 0x001fu +#define H5F_ACC_PUBLIC_FLAGS 0x007fu /* Free space section+aggregator merge flags */ #define H5F_FS_MERGE_METADATA 0x01 /* Section can merge with metadata aggregator */ @@ -131,12 +132,28 @@ #define H5F_SUPERBLOCK_VARLEN_SIZE(v, sizeof_addr, sizeof_size) ( \ (v == 0 ? H5F_SUPERBLOCK_VARLEN_SIZE_V0(sizeof_addr, sizeof_size) : 0) \ + (v == 1 ? H5F_SUPERBLOCK_VARLEN_SIZE_V1(sizeof_addr, sizeof_size) : 0) \ - + (v == 2 ? H5F_SUPERBLOCK_VARLEN_SIZE_V2(sizeof_addr) : 0)) + + (v >= 2 ? H5F_SUPERBLOCK_VARLEN_SIZE_V2(sizeof_addr) : 0)) /* Total size of superblock, depends on superblock version */ #define H5F_SUPERBLOCK_SIZE(s) ( H5F_SUPERBLOCK_FIXED_SIZE \ + H5F_SUPERBLOCK_VARLEN_SIZE((s)->super_vers, (s)->sizeof_addr, (s)->sizeof_size)) +/* For superblock version 0 & 1: + Offset to the file consistency flags (status_flags) in the superblock (excluding H5F_SUPERBLOCK_FIXED_SIZE) */ +#define H5F_SUPER_STATUS_OFF_V01 \ + (2 /* freespace, and root group versions */ \ + + 1 /* reserved */ \ + + 3 /* shared header vers, size of address, size of lengths */ \ + + 1 /* reserved */ \ + + 4) /* group leaf k, group internal k */ + +#define H5F_SUPER_STATUS_OFF(v) (v >= 2 ? 2 : H5F_SUPER_STATUS_OFF_V01) + +/* Offset to the file consistency flags (status_flags) in the superblock */ +#define H5F_SUPER_STATUS_FLAGS_OFF(v) (H5F_SUPERBLOCK_FIXED_SIZE + H5F_SUPER_STATUS_OFF(v)) + +/* Size of file consistency flags (status_flags) in the superblock */ +#define H5F_SUPER_STATUS_FLAGS_SIZE(v) (v >= 2 ? 1 : 4) /* Forward declaration external file cache struct used below (defined in * H5Fefc.c) */ @@ -152,6 +169,9 @@ typedef struct H5F_superblock_cache_ud_t { unsigned btree_k[H5B_NUM_BTREE_ID]; /* B-tree key values for each type */ haddr_t stored_eof; /* End-of-file in file */ hbool_t drvrinfo_removed; /* Indicate if the driver info was removed */ + unsigned super_vers; /* Superblock version obtained in get_load_size callback. + * It will be used later in verify_chksum callback + */ } H5F_superblock_cache_ud_t; /* Structure for passing 'user data' to driver info block cache callbacks */ @@ -259,8 +279,13 @@ struct H5F_file_t { /* metadata cache. This structure is */ /* fixed at creation time and should */ /* not change thereafter. */ + hbool_t use_mdc_logging; /* Set when metadata logging is desired */ + hbool_t start_mdc_log_on_access; /* set when mdc logging should */ + /* begin on file access/create */ + char *mdc_log_location; /* location of mdc log */ hid_t fcpl_id; /* File creation property list ID */ H5F_close_degree_t fc_degree; /* File close behavior degree */ + hbool_t evict_on_close; /* If the file's objects should be evicted from the metadata cache on close */ size_t rdcc_nslots; /* Size of raw data chunk cache (slots) */ size_t rdcc_nbytes; /* Size of raw data chunk cache (bytes) */ double rdcc_w0; /* Preempt read chunks first? [0.0..1.0]*/ @@ -294,6 +319,11 @@ struct H5F_file_t { /* Metadata accumulator information */ H5F_meta_accum_t accum; /* Metadata accumulator info */ + /* Metadata retry info */ + unsigned read_attempts; /* The # of reads to try when reading metadata with checksum */ + unsigned retries_nbins; /* # of bins for each retries[] */ + uint32_t *retries[H5AC_NTYPES]; /* Track # of read retries for metdata items with checksum */ + /* Object flush info */ H5F_object_flush_t object_flush; /* Information for object flush callback */ }; @@ -393,12 +423,16 @@ H5_DLL herr_t H5F_efc_try_close(H5F_t *f); H5_DLL herr_t H5F__set_eoa(const H5F_t *f, H5F_mem_t type, haddr_t addr); H5_DLL herr_t H5F__set_base_addr(const H5F_t *f, haddr_t addr); +/* Functions that flush or evict */ +H5_DLL herr_t H5F__evict_cache_entries(H5F_t *f, hid_t dxpl_id); + /* Testing functions */ #ifdef H5F_TESTING H5_DLL herr_t H5F_get_sohm_mesg_count_test(hid_t fid, unsigned type_id, size_t *mesg_count); H5_DLL herr_t H5F_check_cached_stab_test(hid_t file_id); H5_DLL herr_t H5F_get_maxaddr_test(hid_t file_id, haddr_t *maxaddr); +H5_DLL herr_t H5F_get_sbe_addr_test(hid_t file_id, haddr_t *sbe_addr); #endif /* H5F_TESTING */ #endif /* _H5Fpkg_H */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index b5c24e9..a6d1c4a 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -286,6 +286,7 @@ #define H5F_FILE_ID(F) ((F)->file_id) #define H5F_PARENT(F) ((F)->parent) #define H5F_NMOUNTS(F) ((F)->nmounts) +#define H5F_GET_READ_ATTEMPTS(F) ((F)->shared->read_attempts) #define H5F_DRIVER_ID(F) ((F)->shared->lf->driver_id) #define H5F_GET_FILENO(F,FILENUM) ((FILENUM) = (F)->shared->lf->fileno) #define H5F_HAS_FEATURE(F,FL) ((F)->shared->lf->feature_flags & (FL)) @@ -303,6 +304,7 @@ #define H5F_SET_SOHM_NINDEXES(F, N) ((F)->shared->sohm_nindexes = (N)) #define H5F_FCPL(F) ((F)->shared->fcpl_id) #define H5F_GET_FC_DEGREE(F) ((F)->shared->fc_degree) +#define H5F_EVICT_ON_CLOSE(F) ((F)->shared->evict_on_close) #define H5F_RDCC_NSLOTS(F) ((F)->shared->rdcc_nslots) #define H5F_RDCC_NBYTES(F) ((F)->shared->rdcc_nbytes) #define H5F_RDCC_W0(F) ((F)->shared->rdcc_w0) @@ -318,6 +320,9 @@ #ifdef H5_HAVE_PARALLEL #define H5F_COLL_MD_READ(F) ((F)->coll_md_read) #endif /* H5_HAVE_PARALLEL */ +#define H5F_USE_MDC_LOGGING(F) ((F)->shared->use_mdc_logging) +#define H5F_START_MDC_LOG_ON_ACCESS(F) ((F)->shared->start_mdc_log_on_access) +#define H5F_MDC_LOG_LOCATION(F) ((F)->shared->mdc_log_location) #else /* H5F_MODULE */ #define H5F_INTENT(F) (H5F_get_intent(F)) #define H5F_OPEN_NAME(F) (H5F_get_open_name(F)) @@ -331,6 +336,7 @@ #define H5F_FILE_ID(F) (H5F_get_file_id(F)) #define H5F_PARENT(F) (H5F_get_parent(F)) #define H5F_NMOUNTS(F) (H5F_get_nmounts(F)) +#define H5F_GET_READ_ATTEMPTS(F) (H5F_get_read_attempts(F)) #define H5F_DRIVER_ID(F) (H5F_get_driver_id(F)) #define H5F_GET_FILENO(F,FILENUM) (H5F_get_fileno((F), &(FILENUM))) #define H5F_HAS_FEATURE(F,FL) (H5F_has_feature(F,FL)) @@ -348,6 +354,7 @@ #define H5F_SET_SOHM_NINDEXES(F, N) (H5F_set_sohm_nindexes((F), (N))) #define H5F_FCPL(F) (H5F_get_fcpl(F)) #define H5F_GET_FC_DEGREE(F) (H5F_get_fc_degree(F)) +#define H5F_EVICT_ON_CLOSE(F) (H5F_get_evict_on_close(F)) #define H5F_RDCC_NSLOTS(F) (H5F_rdcc_nslots(F)) #define H5F_RDCC_NBYTES(F) (H5F_rdcc_nbytes(F)) #define H5F_RDCC_W0(F) (H5F_rdcc_w0(F)) @@ -363,6 +370,9 @@ #ifdef H5_HAVE_PARALLEL #define H5F_COLL_MD_READ(F) (H5F_coll_md_read(F)) #endif /* H5_HAVE_PARALLEL */ +#define H5F_USE_MDC_LOGGING(F) (H5F_use_mdc_logging(F)) +#define H5F_START_MDC_LOG_ON_ACCESS(F) (H5F_start_mdc_log_on_access(F)) +#define H5F_MDC_LOG_LOCATION(F) (H5F_mdc_log_location(F)) #endif /* H5F_MODULE */ @@ -459,11 +469,17 @@ #define H5F_ACS_MULTI_TYPE_NAME "multi_type" /* Data type in multi file driver */ #define H5F_ACS_LATEST_FORMAT_NAME "latest_format" /* 'Use latest format version' flag */ #define H5F_ACS_WANT_POSIX_FD_NAME "want_posix_fd" /* Internal: query the file descriptor from the core VFD, instead of the memory address */ +#define H5F_ACS_METADATA_READ_ATTEMPTS_NAME "metadata_read_attempts" /* # of metadata read attempts */ #define H5F_ACS_OBJECT_FLUSH_CB_NAME "object_flush_cb" /* Object flush callback */ #define H5F_ACS_EFC_SIZE_NAME "efc_size" /* Size of external file cache */ #define H5F_ACS_FILE_IMAGE_INFO_NAME "file_image_info" /* struct containing initial file image and callback info */ -#define H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME "core_write_tracking_flag" /* Whether or not core VFD backing store write tracking is enabled */ -#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME "core_write_tracking_page_size" /* The page size in kiB when core VFD write tracking is enabled */ +#define H5F_ACS_CLEAR_STATUS_FLAGS_NAME "clear_status_flags" /* Whether to clear superblock status_flags (private property only used by h5clear) */ +#define H5F_ACS_USE_MDC_LOGGING_NAME "use_mdc_logging" /* Whether to use metadata cache logging */ +#define H5F_ACS_MDC_LOG_LOCATION_NAME "mdc_log_location" /* Name of metadata cache log location */ +#define H5F_ACS_START_MDC_LOG_ON_ACCESS_NAME "start_mdc_log_on_access" /* Whether logging starts on file create/open */ +#define H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME "core_write_tracking_flag" /* Whether or not core VFD backing store write tracking is enabled */ +#define H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME "evict_on_close_flag" /* Whether or not the metadata cache will evict objects on close */ +#define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME "core_write_tracking_page_size" /* The page size in kiB when core VFD write tracking is enabled */ #define H5F_ACS_COLL_MD_WRITE_FLAG_NAME "collective_metadata_write" /* property indicating whether metadata writes are done collectively or not */ /* ======================== File Mount properties ====================*/ @@ -483,7 +499,8 @@ #define HDF5_SUPERBLOCK_VERSION_DEF 0 /* The default super block format */ #define HDF5_SUPERBLOCK_VERSION_1 1 /* Version with non-default B-tree 'K' value */ #define HDF5_SUPERBLOCK_VERSION_2 2 /* Revised version with superblock extension and checksum */ -#define HDF5_SUPERBLOCK_VERSION_LATEST HDF5_SUPERBLOCK_VERSION_2 /* The maximum super block format */ +#define HDF5_SUPERBLOCK_VERSION_3 3 /* With file locking and consistency flags (at least this version for SWMR support) */ +#define HDF5_SUPERBLOCK_VERSION_LATEST HDF5_SUPERBLOCK_VERSION_3 /* The maximum super block format */ #define HDF5_SUPERBLOCK_VERSION_V18_LATEST HDF5_SUPERBLOCK_VERSION_2 /* The latest superblock version for v18 */ #define HDF5_FREESPACE_VERSION 0 /* of the Free-Space Info */ #define HDF5_OBJECTDIR_VERSION 0 /* of the Object Directory format */ @@ -506,12 +523,20 @@ /* Default free space section threshold used by free-space managers */ #define H5F_FREE_SPACE_THRESHOLD_DEF 1 +/* Metadata read attempt values */ +#define H5F_METADATA_READ_ATTEMPTS 1 /* Default # of read attempts for non-SWMR access */ +#define H5F_SWMR_METADATA_READ_ATTEMPTS 100 /* Default # of read attempts for SWMR access */ + /* Macros to define signatures of all objects in the file */ /* Size of signature information (on disk) */ /* (all on-disk signatures should be this length) */ #define H5_SIZEOF_MAGIC 4 +/* Size of checksum information (on disk) */ +/* (all on-disk checksums should be this length) */ +#define H5_SIZEOF_CHKSUM 4 + /* v1 B-tree node signature */ #define H5B_MAGIC "TREE" @@ -625,7 +650,7 @@ typedef struct H5F_block_t { /* Private functions */ H5_DLL H5F_t *H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, hid_t dxpl_id); -H5_DLL herr_t H5F_try_close(H5F_t *f); +H5_DLL herr_t H5F_try_close(H5F_t *f, hbool_t *was_closed/*out*/); /* Functions than retrieve values from the file struct */ H5_DLL unsigned H5F_get_intent(const H5F_t *f); @@ -640,6 +665,7 @@ H5_DLL unsigned H5F_decr_nopen_objs(H5F_t *f); H5_DLL hid_t H5F_get_file_id(const H5F_t *f); H5_DLL H5F_t *H5F_get_parent(const H5F_t *f); H5_DLL unsigned H5F_get_nmounts(const H5F_t *f); +H5_DLL unsigned H5F_get_read_attempts(const H5F_t *f); H5_DLL hid_t H5F_get_access_plist(H5F_t *f, hbool_t app_ref); H5_DLL hid_t H5F_get_id(H5F_t *file, hbool_t app_ref); H5_DLL herr_t H5F_get_obj_count(const H5F_t *f, unsigned types, hbool_t app_ref, size_t *obj_id_count_ptr); @@ -660,6 +686,7 @@ H5_DLL unsigned H5F_get_sohm_nindexes(const H5F_t *f); H5_DLL herr_t H5F_set_sohm_nindexes(H5F_t *f, unsigned nindexes); H5_DLL hid_t H5F_get_fcpl(const H5F_t *f); H5_DLL H5F_close_degree_t H5F_get_fc_degree(const H5F_t *f); +H5_DLL hbool_t H5F_get_evict_on_close(const H5F_t *f); H5_DLL size_t H5F_rdcc_nbytes(const H5F_t *f); H5_DLL size_t H5F_rdcc_nslots(const H5F_t *f); H5_DLL double H5F_rdcc_w0(const H5F_t *f); @@ -676,6 +703,9 @@ H5_DLL hbool_t H5F_is_tmp_addr(const H5F_t *f, haddr_t addr); H5_DLL H5P_coll_md_read_flag_t H5F_coll_md_read(const H5F_t *f); H5_DLL void H5F_set_coll_md_read(H5F_t *f, H5P_coll_md_read_flag_t flag); #endif /* H5_HAVE_PARALLEL */ +H5_DLL hbool_t H5F_use_mdc_logging(const H5F_t *f); +H5_DLL hbool_t H5F_start_mdc_log_on_access(const H5F_t *f); +H5_DLL char *H5F_mdc_log_location(const H5F_t *f); /* Functions that retrieve values from VFD layer */ H5_DLL hid_t H5F_get_driver_id(const H5F_t *f); @@ -700,6 +730,13 @@ H5_DLL herr_t H5F_block_write(const H5F_t *f, H5FD_mem_t type, haddr_t addr, H5_DLL herr_t H5F_flush_tagged_metadata(H5F_t * f, haddr_t tag, hid_t dxpl_id); H5_DLL herr_t H5F_evict_tagged_metadata(H5F_t * f, haddr_t tag, hid_t dxpl_id); +/* Functions that verify a piece of metadata with checksum */ +H5_DLL herr_t H5F_get_checksums(const uint8_t *buf, size_t chk_size, uint32_t *s_chksum, uint32_t *c_chksum); + +/* Routine to track the # of retries */ +H5_DLL herr_t H5F_track_metadata_read_retries(H5F_t *f, unsigned actype, unsigned retries); +H5_DLL herr_t H5F_set_retries(H5F_t *f); + /* Routine to invoke callback function upon object flush */ H5_DLL herr_t H5F_object_flush_cb(H5F_t *f, hid_t obj_id); diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index fcccf07..a79da75 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -55,6 +55,23 @@ #define H5F_ACC_EXCL (H5CHECK H5OPEN 0x0004u) /*fail if file already exists*/ /* NOTE: 0x0008u was H5F_ACC_DEBUG, now deprecated */ #define H5F_ACC_CREAT (H5CHECK H5OPEN 0x0010u) /*create non-existing files */ +#define H5F_ACC_SWMR_WRITE (H5CHECK 0x0020u) /*indicate that this file is + * open for writing in a + * single-writer/multi-reader (SWMR) + * scenario. Note that the + * process(es) opening the file + * for reading must open the file + * with RDONLY access, and use + * the special "SWMR_READ" access + * flag. */ +#define H5F_ACC_SWMR_READ (H5CHECK 0x0040u) /*indicate that this file is + * open for reading in a + * single-writer/multi-reader (SWMR) + * scenario. Note that the + * process(es) opening the file + * for SWMR reading must also + * open the file with the RDONLY + * flag. */ /* Value passed to H5Pset_elink_acc_flags to cause flags to be taken from the * parent file. */ @@ -175,6 +192,14 @@ typedef enum H5F_file_space_type_t { H5F_FILE_SPACE_NTYPES /* must be last */ } H5F_file_space_type_t; +/* Data structure to report the collection of read retries for metadata items with checksum */ +/* Used by public routine H5Fget_metadata_read_retry_info() */ +#define H5F_NUM_METADATA_READ_RETRY_TYPES 21 +typedef struct H5F_retry_info_t { + unsigned nbins; + uint32_t *retries[H5F_NUM_METADATA_READ_RETRY_TYPES]; +} H5F_retry_info_t; + /* Callback for H5Pset_object_flush_cb() in a file access property list */ typedef herr_t (*H5F_flush_cb_t)(hid_t object_id, void *udata); @@ -216,9 +241,16 @@ H5_DLL herr_t H5Fget_mdc_size(hid_t file_id, H5_DLL herr_t H5Freset_mdc_hit_rate_stats(hid_t file_id); H5_DLL ssize_t H5Fget_name(hid_t obj_id, char *name, size_t size); H5_DLL herr_t H5Fget_info2(hid_t obj_id, H5F_info2_t *finfo); +H5_DLL herr_t H5Fget_metadata_read_retry_info(hid_t file_id, H5F_retry_info_t *info); +H5_DLL herr_t H5Fstart_swmr_write(hid_t file_id); H5_DLL ssize_t H5Fget_free_sections(hid_t file_id, H5F_mem_t type, size_t nsects, H5F_sect_info_t *sect_info/*out*/); H5_DLL herr_t H5Fclear_elink_file_cache(hid_t file_id); +H5_DLL herr_t H5Fstart_mdc_logging(hid_t file_id); +H5_DLL herr_t H5Fstop_mdc_logging(hid_t file_id); +H5_DLL herr_t H5Fget_mdc_logging_status(hid_t file_id, + /*OUT*/ hbool_t *is_enabled, + /*OUT*/ hbool_t *is_currently_logging); H5_DLL herr_t H5Fformat_convert(hid_t fid); #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5Fset_mpi_atomicity(hid_t file_id, hbool_t flag); diff --git a/src/H5Fquery.c b/src/H5Fquery.c index dd6e8e3..14dd655 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -322,6 +322,29 @@ H5F_get_nmounts(const H5F_t *f) /*------------------------------------------------------------------------- + * Function: H5F_get_read_attempts + * + * Purpose: Retrieve the file's 'read_attempts' value + * + * Return: '# of read attempts' on success/abort on failure (shouldn't fail) + * + * Programmer: Vaili Choi; Sept 2013 + * + *------------------------------------------------------------------------- + */ +unsigned +H5F_get_read_attempts(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + + FUNC_LEAVE_NOAPI(f->shared->read_attempts) +} /* end H5F_get_read_attempts() */ + + +/*------------------------------------------------------------------------- * Function: H5F_get_fcpl * * Purpose: Retrieve the value of a file's FCPL. @@ -835,6 +858,34 @@ H5F_get_fc_degree(const H5F_t *f) /*------------------------------------------------------------------------- + * Function: H5F_get_evict_on_close + * + * Purpose: Checks if evict-on-close is desired for objects in the + * file. + * + * Return: Success: Flag indicating whether the evict-on-close + * property was set for the file. + * Failure: (can't happen) + * + * Programmer: Dana Robinson + * Spring 2016 + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_get_evict_on_close(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->evict_on_close) +} /* end H5F_get_evict_on_close() */ + + +/*------------------------------------------------------------------------- * Function: H5F_store_msg_crt_idx * * Purpose: Retrieve the 'store message creation index' flag for the file. @@ -1100,3 +1151,87 @@ H5F_coll_md_read(const H5F_t *f) } /* end H5F_coll_md_read() */ #endif /* H5_HAVE_PARALLEL */ + +/*------------------------------------------------------------------------- + * Function: H5F_use_mdc_logging + * + * Purpose: Quick and dirty routine to determine if using MDC logging + * is enabled for this file. + * (Mainly added to stop non-file routines from poking about in the + * H5F_t data structure) + * + * Return: TRUE/FALSE on success/abort on failure (shouldn't fail) + * + * Programmer: Quincey Koziol <koziol@hdfgroup.org> + * June 5, 2016 + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_use_mdc_logging(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->use_mdc_logging) +} /* end H5F_use_mdc_logging() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_start_mdc_log_on_access + * + * Purpose: Quick and dirty routine to determine if we should start MDC + * logging on access for this file. + * (Mainly added to stop non-file routines from poking about in the + * H5F_t data structure) + * + * Return: TRUE/FALSE on success/abort on failure (shouldn't fail) + * + * Programmer: Quincey Koziol <koziol@hdfgroup.org> + * June 5, 2016 + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_start_mdc_log_on_access(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->start_mdc_log_on_access) +} /* end H5F_start_mdc_log_on_access() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_mdc_log_location + * + * Purpose: Quick and dirty routine to retrieve the MDC log location + * for this file. + * (Mainly added to stop non-file routines from poking about in the + * H5F_t data structure) + * + * Return: TRUE/FALSE on success/abort on failure (shouldn't fail) + * + * Programmer: Quincey Koziol <koziol@hdfgroup.org> + * June 5, 2016 + * + *------------------------------------------------------------------------- + */ +char * +H5F_mdc_log_location(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->mdc_log_location) +} /* end H5F_mdc_log_location() */ + diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c index 23b7f78..2a82618 100644 --- a/src/H5Fsuper.c +++ b/src/H5Fsuper.c @@ -208,7 +208,7 @@ H5F_super_ext_close(H5F_t *f, H5O_loc_t *ext_ptr, hid_t dxpl_id, /* Twiddle the number of open objects to avoid closing the file. */ f->nopen_objs++; - if(H5O_close(ext_ptr) < 0) + if(H5O_close(ext_ptr, NULL) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close superblock extension") f->nopen_objs--; @@ -250,6 +250,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) haddr_t super_addr; /* Absolute address of superblock */ haddr_t eof; /* End of file address */ unsigned rw_flags; /* Read/write permissions for file */ + hbool_t skip_eof_check = FALSE; /* Whether to skip checking the EOF value */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE_TAG(dxpl_id, H5AC__SUPERBLOCK_TAG, FAIL) @@ -317,6 +318,14 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) if(NULL == (sblock = (H5F_super_t *)H5AC_protect(f, dxpl_id, H5AC_SUPERBLOCK, (haddr_t)0, &udata, rw_flags))) HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "unable to load superblock") + if(H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) + if(sblock->super_vers < HDF5_SUPERBLOCK_VERSION_3) + HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "invalid superblock version for SWMR_WRITE") + + /* Enable all latest version support when file has v3 superblock */ + if(sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_3) + f->shared->latest_flags |= H5F_LATEST_ALL_FLAGS; + /* Pin the superblock in the cache */ if(H5AC_pin_protected_entry(sblock) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTPIN, FAIL, "unable to pin superblock") @@ -401,7 +410,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) * individually. */ /* Can skip this test when it is not the initial file open-- - * H5F_super_read() call from H5F_evict_tagged_metadata() for + * H5F__super_read() call from H5F_evict_tagged_metadata() for * refreshing object. * When flushing file buffers and fractal heap is involved, * the library will allocate actual space for tmp addresses @@ -410,7 +419,22 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) * Note: the aggregator is changed again after being reset * earlier before H5AC_flush due to allocation of tmp addresses. */ - if(initial_read) { + /* The EOF check must be skipped when the file is opened for SWMR read, + * as the file can appear truncated if only part of it has been + * been flushed to disk by the SWMR writer process. + */ + if(H5F_INTENT(f) & H5F_ACC_SWMR_READ) { + /* + * When the file is opened for SWMR read access, skip the check if: + * --the file is already marked for SWMR writing and + * --the file has version 3 superblock for SWMR support + */ + if((sblock->status_flags & H5F_SUPER_SWMR_WRITE_ACCESS) && + (sblock->status_flags & H5F_SUPER_WRITE_ACCESS) && + sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_3) + skip_eof_check = TRUE; + } /* end if */ + if(!skip_eof_check && initial_read) { if(HADDR_UNDEF == (eof = H5FD_get_eof(f->shared->lf, H5FD_MEM_DEFAULT))) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to determine file size") @@ -448,8 +472,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) * portion of the driver info block */ if(H5FD_set_eoa(f->shared->lf, H5FD_MEM_SUPER, sblock->driver_addr + H5F_DRVINFOBLOCK_HDR_SIZE) < 0) /* will extend eoa later if required */ - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, \ - "set end of space allocation request failed") + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed") /* Look up the driver info block */ if(NULL == (drvinfo = (H5O_drvinfo_t *)H5AC_protect(f, dxpl_id, H5AC_DRVRINFO, sblock->driver_addr, &drvrinfo_udata, rw_flags))) @@ -458,11 +481,8 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) /* Loading the driver info block is enough to set up the right info */ /* Check if we need to rewrite the driver info block info */ - if ( ( (rw_flags & H5AC__READ_ONLY_FLAG) == 0 ) && - ( H5F_HAS_FEATURE(f, H5FD_FEAT_DIRTY_DRVRINFO_LOAD) ) ) { - + if(((rw_flags & H5AC__READ_ONLY_FLAG) == 0) && H5F_HAS_FEATURE(f, H5FD_FEAT_DIRTY_DRVRINFO_LOAD)) drvinfo_flags |= H5AC__DIRTIED_FLAG; - } /* end if */ /* set the pin entry flag so that the driver information block * cache entry will be pinned in the cache. @@ -478,7 +498,7 @@ H5F__super_read(H5F_t *f, hid_t dxpl_id, hbool_t initial_read) } /* end if */ /* (Account for the stored EOA being absolute offset -NAF) */ - if(H5F__set_eoa(f, H5FD_MEM_SUPER, udata.stored_eof - sblock->base_addr) < 0) + if(H5F__set_eoa(f, H5FD_MEM_DEFAULT, udata.stored_eof - sblock->base_addr) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "unable to set end-of-address marker for file") /* Decode the optional superblock extension info */ diff --git a/src/H5Fsuper_cache.c b/src/H5Fsuper_cache.c index f445303..6cfd9c7 100644 --- a/src/H5Fsuper_cache.c +++ b/src/H5Fsuper_cache.c @@ -67,26 +67,26 @@ /********************/ /* Metadata cache (H5AC) callbacks */ -static herr_t H5F__cache_superblock_get_load_size(const void *udata, size_t *image_len); +static herr_t H5F__cache_superblock_get_initial_load_size(void *udata, size_t *image_len); +static herr_t H5F__cache_superblock_get_final_load_size(const void *image_ptr, + size_t image_len, void *udata, size_t *actual_len); +static htri_t H5F__cache_superblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5F__cache_superblock_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5F__cache_superblock_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5F__cache_superblock_image_len(const void *thing, size_t *image_len); static herr_t H5F__cache_superblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing, haddr_t addr, size_t len, - size_t compressed_len, haddr_t *new_addr, size_t *new_len, - size_t *new_compressed_len, unsigned *flags); + haddr_t *new_addr, size_t *new_len, unsigned *flags); static herr_t H5F__cache_superblock_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5F__cache_superblock_free_icr(void *thing); -static herr_t H5F__cache_drvrinfo_get_load_size(const void *udata, size_t *image_len); +static herr_t H5F__cache_drvrinfo_get_initial_load_size(void *udata, size_t *image_len); +static herr_t H5F__cache_drvrinfo_get_final_load_size(const void *image_ptr, + size_t image_len, void *udata, size_t *actual_len); static void *H5F__cache_drvrinfo_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5F__cache_drvrinfo_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5F__cache_drvrinfo_image_len(const void *thing, size_t *image_len); static herr_t H5F__cache_drvrinfo_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5F__cache_drvrinfo_free_icr(void *thing); @@ -102,14 +102,15 @@ const H5AC_class_t H5AC_SUPERBLOCK[1] = {{ "Superblock", /* Metadata client name (for debugging) */ H5FD_MEM_SUPER, /* File space memory type for client */ H5AC__CLASS_SPECULATIVE_LOAD_FLAG, /* Client class behavior flags */ - H5F__cache_superblock_get_load_size,/* 'get_load_size' callback */ + H5F__cache_superblock_get_initial_load_size,/* 'get_initial_load_size' callback */ + H5F__cache_superblock_get_final_load_size, /* 'get_final_load_size' callback */ + H5F__cache_superblock_verify_chksum, /* 'verify_chksum' callback */ H5F__cache_superblock_deserialize, /* 'deserialize' callback */ H5F__cache_superblock_image_len, /* 'image_len' callback */ H5F__cache_superblock_pre_serialize,/* 'pre_serialize' callback */ H5F__cache_superblock_serialize, /* 'serialize' callback */ - NULL, /* 'notify' callback */ + NULL, /* 'notify' callback */ H5F__cache_superblock_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -119,14 +120,15 @@ const H5AC_class_t H5AC_DRVRINFO[1] = {{ "Driver info block", /* Metadata client name (for debugging) */ H5FD_MEM_SUPER, /* File space memory type for client */ H5AC__CLASS_SPECULATIVE_LOAD_FLAG, /* Client class behavior flags */ - H5F__cache_drvrinfo_get_load_size, /* 'get_load_size' callback */ + H5F__cache_drvrinfo_get_initial_load_size, /* 'get_initial_load_size' callback */ + H5F__cache_drvrinfo_get_final_load_size, /* 'get_final_load_size' callback */ + NULL, /* 'verify_chksum' callback */ H5F__cache_drvrinfo_deserialize, /* 'deserialize' callback */ H5F__cache_drvrinfo_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5F__cache_drvrinfo_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5F__cache_drvrinfo_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -146,7 +148,7 @@ H5FL_EXTERN(H5F_super_t); /*------------------------------------------------------------------------- - * Function: H5F__cache_superblock_get_load_size + * Function: H5F__cache_superblock_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -159,7 +161,7 @@ H5FL_EXTERN(H5F_super_t); *------------------------------------------------------------------------- */ static herr_t -H5F__cache_superblock_get_load_size(const void H5_ATTR_UNUSED *udata, size_t *image_len) +H5F__cache_superblock_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len) { FUNC_ENTER_STATIC_NOERR @@ -168,10 +170,138 @@ H5F__cache_superblock_get_load_size(const void H5_ATTR_UNUSED *udata, size_t *im /* Set the initial image length size */ *image_len = H5F_SUPERBLOCK_FIXED_SIZE + /* Fixed size of superblock */ - H5F_SUPERBLOCK_MINIMAL_VARLEN_SIZE; + H5F_SUPERBLOCK_MINIMAL_VARLEN_SIZE; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5F__cache_superblock_get_load_size() */ +} /* end H5F__cache_superblock_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5F__cache_superblock_get_final_load_size + * + * Purpose: Compute the final size of the data structure on disk. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@lbl.gov + * November 17, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F__cache_superblock_get_final_load_size(const void *_image, size_t image_len, + void *_udata, size_t *actual_len) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5F_superblock_cache_ud_t *udata = (H5F_superblock_cache_ud_t *)_udata; /* User data */ + unsigned super_vers; /* Superblock version */ + uint8_t sizeof_addr; /* Size of offsets in the file (in bytes) */ + uint8_t sizeof_size; /* Size of lengths in the file (in bytes) */ + size_t variable_size; /* Variable size of superblock */ + htri_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check arguments */ + HDassert(image); + HDassert(udata); + HDassert(udata->f); + HDassert(actual_len); + HDassert(*actual_len == image_len); + + /* Skip over file signature */ + image += H5F_SIGNATURE_LEN; + + /* Superblock version */ + super_vers = *image++; + if(super_vers > HDF5_SUPERBLOCK_VERSION_LATEST) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad superblock version number") + + /* Save the version to be used in verify_chksum callback */ + udata->super_vers = super_vers; + + /* Sanity check */ + HDassert(((size_t)(image - (const uint8_t *)_image)) == H5F_SUPERBLOCK_FIXED_SIZE); + HDassert(image_len >= H5F_SUPERBLOCK_FIXED_SIZE + 6); + + /* Determine the size of addresses & size of offsets, for computing the + * variable-sized portion of the superblock. + */ + if(super_vers < HDF5_SUPERBLOCK_VERSION_2) { + sizeof_addr = image[4]; + sizeof_size = image[5]; + } /* end if */ + else { + sizeof_addr = image[0]; + sizeof_size = image[1]; + } /* end else */ + if(sizeof_addr != 2 && sizeof_addr != 4 && + sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad byte number in an address") + if(sizeof_size != 2 && sizeof_size != 4 && + sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad byte number for object size") + + /* Determine the size of the variable-length part of the superblock */ + variable_size = (size_t)H5F_SUPERBLOCK_VARLEN_SIZE(super_vers, sizeof_addr, sizeof_size); + HDassert(variable_size > 0); + + /* Sanity check */ + HDassert(image_len == (H5F_SUPERBLOCK_FIXED_SIZE + H5F_SUPERBLOCK_MINIMAL_VARLEN_SIZE)); + + /* Make certain we can read the variable-sized portion of the superblock */ + if(H5F__set_eoa(udata->f, H5FD_MEM_SUPER, (haddr_t)(H5F_SUPERBLOCK_FIXED_SIZE + variable_size)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed") + + /* Set the final size for the cache image */ + *actual_len = H5F_SUPERBLOCK_FIXED_SIZE + variable_size; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__cache_superblock_get_final_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5F__cache_superblock_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5F__cache_superblock_verify_chksum(const void *_image, size_t len, void *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5F_superblock_cache_ud_t *udata = (H5F_superblock_cache_ud_t *)_udata; /* User data */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + HDassert(udata); + + /* No checksum for version 0 & 1 */ + if(udata->super_vers >= HDF5_SUPERBLOCK_VERSION_2) { + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__cache_superblock_verify_chksum() */ /*------------------------------------------------------------------------- @@ -251,176 +381,161 @@ H5F__cache_superblock_deserialize(const void *_image, size_t len, void *_udata, variable_size = (size_t)H5F_SUPERBLOCK_VARLEN_SIZE(super_vers, sizeof_addr, sizeof_size); HDassert(variable_size > 0); - /* Handle metadata cache retry for variable-sized portion of the superblock */ - if(len != (H5F_SUPERBLOCK_FIXED_SIZE + variable_size)) { - /* Sanity check */ - HDassert(len == (H5F_SUPERBLOCK_FIXED_SIZE + H5F_SUPERBLOCK_MINIMAL_VARLEN_SIZE)); + HDassert(len == (H5F_SUPERBLOCK_FIXED_SIZE + variable_size)); - /* Make certain we can read the variabled-sized portion of the superblock */ - if(H5F__set_eoa(udata->f, H5FD_MEM_SUPER, (haddr_t)(H5F_SUPERBLOCK_FIXED_SIZE + variable_size)) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "set end of space allocation request failed") - } /* end if */ - else { - /* Check for older version of superblock format */ - if(super_vers < HDF5_SUPERBLOCK_VERSION_2) { - uint32_t status_flags; /* File status flags */ - unsigned sym_leaf_k; /* Symbol table leaf node's 'K' value */ - unsigned snode_btree_k; /* B-tree symbol table internal node 'K' value */ - unsigned chunk_btree_k; /* B-tree chunk internal node 'K' value */ - - /* Freespace version (hard-wired) */ - if(HDF5_FREESPACE_VERSION != *image++) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad free space version number") - - /* Root group version number (hard-wired) */ - if(HDF5_OBJECTDIR_VERSION != *image++) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad object directory version number") - - /* Skip over reserved byte */ - image++; - - /* Shared header version number (hard-wired) */ - if(HDF5_SHAREDHEADER_VERSION != *image++) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad shared-header format version number") - - /* Size of file addresses */ - sizeof_addr = *image++; - if(sizeof_addr != 2 && sizeof_addr != 4 && - sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number in an address") - sblock->sizeof_addr = sizeof_addr; - udata->f->shared->sizeof_addr = sizeof_addr; /* Keep a local copy also */ - - /* Size of file sizes */ - sizeof_size = *image++; - if(sizeof_size != 2 && sizeof_size != 4 && - sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number for object size") - sblock->sizeof_size = sizeof_size; - udata->f->shared->sizeof_size = sizeof_size; /* Keep a local copy also */ - - /* Skip over reserved byte */ - image++; - - /* Various B-tree sizes */ - UINT16DECODE(image, sym_leaf_k); - if(sym_leaf_k == 0) - HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, NULL, "bad symbol table leaf node 1/2 rank") - udata->sym_leaf_k = sym_leaf_k; /* Keep a local copy also */ - - /* Need 'get' call to set other array values */ - UINT16DECODE(image, snode_btree_k); - if(snode_btree_k == 0) - HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, NULL, "bad 1/2 rank for btree internal nodes") - udata->btree_k[H5B_SNODE_ID] = snode_btree_k; - - /* - * Delay setting the value in the property list until we've checked - * for the indexed storage B-tree internal 'K' value later. - */ - - /* File status flags (not really used yet) */ - UINT32DECODE(image, status_flags); - HDassert(status_flags <= 255); - sblock->status_flags = (uint8_t)status_flags; - if(sblock->status_flags & ~H5F_SUPER_ALL_FLAGS) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad flag value for superblock") - - /* - * If the superblock version # is greater than 0, read in the indexed - * storage B-tree internal 'K' value - */ - if(super_vers > HDF5_SUPERBLOCK_VERSION_DEF) { - UINT16DECODE(image, chunk_btree_k); - - /* Reserved bytes are present only in version 1 */ - if(super_vers == HDF5_SUPERBLOCK_VERSION_1) - image += 2; /* reserved */ - } /* end if */ - else - chunk_btree_k = HDF5_BTREE_CHUNK_IK_DEF; - udata->btree_k[H5B_CHUNK_ID] = chunk_btree_k; - - /* Remainder of "variable-sized" portion of superblock */ - H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->base_addr/*out*/); - H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->ext_addr/*out*/); - H5F_addr_decode(udata->f, (const uint8_t **)&image, &udata->stored_eof/*out*/); - H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->driver_addr/*out*/); - - /* Allocate space for the root group symbol table entry */ - HDassert(!sblock->root_ent); - if(NULL == (sblock->root_ent = (H5G_entry_t *)H5MM_calloc(sizeof(H5G_entry_t)))) - HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, NULL, "can't allocate space for root group symbol table entry") - - /* decode the root group symbol table entry */ - if(H5G_ent_decode(udata->f, (const uint8_t **)&image, sblock->root_ent) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "can't decode root group symbol table entry") - - /* Set the root group address to the correct value */ - sblock->root_addr = sblock->root_ent->header; - - /* This step is for h5repart tool only. If user wants to change file driver - * from family to sec2 while using h5repart, set the driver address to - * undefined to let the library ignore the family driver information saved - * in the superblock. - */ - if(udata->ignore_drvrinfo && H5F_addr_defined(sblock->driver_addr)) { - /* Eliminate the driver info */ - sblock->driver_addr = HADDR_UNDEF; - udata->drvrinfo_removed = TRUE; - } /* end if */ + /* Check for older version of superblock format */ + if(super_vers < HDF5_SUPERBLOCK_VERSION_2) { + uint32_t status_flags; /* File status flags */ + unsigned sym_leaf_k; /* Symbol table leaf node's 'K' value */ + unsigned snode_btree_k; /* B-tree symbol table internal node 'K' value */ + unsigned chunk_btree_k; /* B-tree chunk internal node 'K' value */ + + /* Freespace version (hard-wired) */ + if(HDF5_FREESPACE_VERSION != *image++) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad free space version number") + + /* Root group version number (hard-wired) */ + if(HDF5_OBJECTDIR_VERSION != *image++) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad object directory version number") + + /* Skip over reserved byte */ + image++; + + /* Shared header version number (hard-wired) */ + if(HDF5_SHAREDHEADER_VERSION != *image++) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad shared-header format version number") + + /* Size of file addresses */ + sizeof_addr = *image++; + if(sizeof_addr != 2 && sizeof_addr != 4 && + sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number in an address") + sblock->sizeof_addr = sizeof_addr; + udata->f->shared->sizeof_addr = sizeof_addr; /* Keep a local copy also */ + + /* Size of file sizes */ + sizeof_size = *image++; + if(sizeof_size != 2 && sizeof_size != 4 && + sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number for object size") + sblock->sizeof_size = sizeof_size; + udata->f->shared->sizeof_size = sizeof_size; /* Keep a local copy also */ + + /* Skip over reserved byte */ + image++; + + /* Various B-tree sizes */ + UINT16DECODE(image, sym_leaf_k); + if(sym_leaf_k == 0) + HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, NULL, "bad symbol table leaf node 1/2 rank") + udata->sym_leaf_k = sym_leaf_k; /* Keep a local copy also */ + + /* Need 'get' call to set other array values */ + UINT16DECODE(image, snode_btree_k); + if(snode_btree_k == 0) + HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, NULL, "bad 1/2 rank for btree internal nodes") + udata->btree_k[H5B_SNODE_ID] = snode_btree_k; + + /* + * Delay setting the value in the property list until we've checked + * for the indexed storage B-tree internal 'K' value later. + */ - /* NOTE: Driver info block is decoded separately, later */ + /* File status flags (not really used yet) */ + UINT32DECODE(image, status_flags); + HDassert(status_flags <= 255); + sblock->status_flags = (uint8_t)status_flags; + if(sblock->status_flags & ~H5F_SUPER_ALL_FLAGS) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad flag value for superblock") - } /* end if */ - else { - uint32_t computed_chksum; /* Computed checksum */ - uint32_t read_chksum; /* Checksum read from file */ - - /* Size of file addresses */ - sizeof_addr = *image++; - if(sizeof_addr != 2 && sizeof_addr != 4 && - sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number in an address") - sblock->sizeof_addr = sizeof_addr; - udata->f->shared->sizeof_addr = sizeof_addr; /* Keep a local copy also */ - - /* Size of file sizes */ - sizeof_size = *image++; - if(sizeof_size != 2 && sizeof_size != 4 && - sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number for object size") - sblock->sizeof_size = sizeof_size; - udata->f->shared->sizeof_size = sizeof_size; /* Keep a local copy also */ - - /* File status flags (not really used yet) */ - sblock->status_flags = *image++; - if(sblock->status_flags & ~H5F_SUPER_ALL_FLAGS) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad flag value for superblock") - - /* Base, superblock extension, end of file & root group object header addresses */ - H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->base_addr/*out*/); - H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->ext_addr/*out*/); - H5F_addr_decode(udata->f, (const uint8_t **)&image, &udata->stored_eof/*out*/); - H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->root_addr/*out*/); - - /* Compute checksum for superblock */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); - - /* Decode checksum */ - UINT32DECODE(image, read_chksum); - - /* Verify correct checksum */ - if(read_chksum != computed_chksum) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "bad checksum on driver information block") - - /* The Driver Information Block may not appear with the version - * 2 super block. Thus we set the driver_addr field of the in - * core representation of the super block HADDR_UNDEF to prevent - * any attempt to load the Driver Information Block. - */ - sblock->driver_addr = HADDR_UNDEF; - } /* end else */ + /* + * If the superblock version # is greater than 0, read in the indexed + * storage B-tree internal 'K' value + */ + if(super_vers > HDF5_SUPERBLOCK_VERSION_DEF) { + UINT16DECODE(image, chunk_btree_k); + + /* Reserved bytes are present only in version 1 */ + if(super_vers == HDF5_SUPERBLOCK_VERSION_1) + image += 2; /* reserved */ + } /* end if */ + else + chunk_btree_k = HDF5_BTREE_CHUNK_IK_DEF; + udata->btree_k[H5B_CHUNK_ID] = chunk_btree_k; + + /* Remainder of "variable-sized" portion of superblock */ + H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->base_addr/*out*/); + H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->ext_addr/*out*/); + H5F_addr_decode(udata->f, (const uint8_t **)&image, &udata->stored_eof/*out*/); + H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->driver_addr/*out*/); + + /* Allocate space for the root group symbol table entry */ + HDassert(!sblock->root_ent); + if(NULL == (sblock->root_ent = (H5G_entry_t *)H5MM_calloc(sizeof(H5G_entry_t)))) + HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, NULL, "can't allocate space for root group symbol table entry") + + /* decode the root group symbol table entry */ + if(H5G_ent_decode(udata->f, (const uint8_t **)&image, sblock->root_ent) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "can't decode root group symbol table entry") + + /* Set the root group address to the correct value */ + sblock->root_addr = sblock->root_ent->header; + + /* This step is for h5repart tool only. If user wants to change file driver + * from family to sec2 while using h5repart, set the driver address to + * undefined to let the library ignore the family driver information saved + * in the superblock. + */ + if(udata->ignore_drvrinfo && H5F_addr_defined(sblock->driver_addr)) { + /* Eliminate the driver info */ + sblock->driver_addr = HADDR_UNDEF; + udata->drvrinfo_removed = TRUE; + } /* end if */ + + /* NOTE: Driver info block is decoded separately, later */ + + } /* end if */ + else { + uint32_t read_chksum; /* Checksum read from file */ + + /* Size of file addresses */ + sizeof_addr = *image++; + if(sizeof_addr != 2 && sizeof_addr != 4 && + sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number in an address") + sblock->sizeof_addr = sizeof_addr; + udata->f->shared->sizeof_addr = sizeof_addr; /* Keep a local copy also */ + + /* Size of file sizes */ + sizeof_size = *image++; + if(sizeof_size != 2 && sizeof_size != 4 && + sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number for object size") + sblock->sizeof_size = sizeof_size; + udata->f->shared->sizeof_size = sizeof_size; /* Keep a local copy also */ + + /* File status flags (not really used yet) */ + sblock->status_flags = *image++; + if(sblock->status_flags & ~H5F_SUPER_ALL_FLAGS) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad flag value for superblock") + + /* Base, superblock extension, end of file & root group object header addresses */ + H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->base_addr/*out*/); + H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->ext_addr/*out*/); + H5F_addr_decode(udata->f, (const uint8_t **)&image, &udata->stored_eof/*out*/); + H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->root_addr/*out*/); + + /* checksum verification already done in verify_chksum cb */ + + /* Decode checksum */ + UINT32DECODE(image, read_chksum); + + /* The Driver Information Block may not appear with the version + * 2 super block. Thus we set the driver_addr field of the in + * core representation of the super block HADDR_UNDEF to prevent + * any attempt to load the Driver Information Block. + */ + sblock->driver_addr = HADDR_UNDEF; } /* end else */ /* Sanity check */ @@ -453,7 +568,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5F__cache_superblock_image_len(const void *_thing, size_t *image_len, hbool_t *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5F__cache_superblock_image_len(const void *_thing, size_t *image_len) { const H5F_super_t *sblock = (const H5F_super_t *)_thing; /* Pointer to the object */ @@ -464,14 +579,10 @@ H5F__cache_superblock_image_len(const void *_thing, size_t *image_len, hbool_t * HDassert(sblock->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(sblock->cache_info.type == H5AC_SUPERBLOCK); HDassert(image_len); - HDassert(compressed_ptr); /* Set the image length size */ *image_len = (size_t)H5F_SUPERBLOCK_SIZE(sblock); - /* Set *compressed_ptr to FALSE unconditionally */ - *compressed_ptr = FALSE; - FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5F__cache_superblock_image_len() */ @@ -511,8 +622,7 @@ H5F__cache_superblock_image_len(const void *_thing, size_t *image_len, hbool_t * static herr_t H5F__cache_superblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, haddr_t H5_ATTR_UNUSED addr, size_t H5_ATTR_UNUSED len, - size_t H5_ATTR_UNUSED compressed_len, haddr_t H5_ATTR_UNUSED *new_addr, - size_t H5_ATTR_UNUSED *new_len, size_t H5_ATTR_UNUSED *new_compressed_len, + haddr_t H5_ATTR_UNUSED *new_addr, size_t H5_ATTR_UNUSED *new_len, unsigned H5_ATTR_UNUSED *flags) { H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ @@ -771,9 +881,9 @@ done: /*------------------------------------------------------------------------- - * Function: H5F__cache_drvrinfo_get_load_size + * Function: H5F__cache_drvrinfo_get_initial_load_size * - * Purpose: Compute the size of the data structure on disk. + * Purpose: Compute the intiial size of the data structure on disk. * * Return: Non-negative on success/Negative on failure * @@ -784,7 +894,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5F__cache_drvrinfo_get_load_size(const void H5_ATTR_UNUSED *udata, size_t *image_len) +H5F__cache_drvrinfo_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len) { FUNC_ENTER_STATIC_NOERR @@ -795,7 +905,76 @@ H5F__cache_drvrinfo_get_load_size(const void H5_ATTR_UNUSED *udata, size_t *imag *image_len = H5F_DRVINFOBLOCK_HDR_SIZE; /* Fixed size portion of driver info block */ FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5F__cache_drvrinfo_get_load_size() */ +} /* end H5F__cache_drvrinfo_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5F__cache_drvrinfo_get_final_load_size + * + * Purpose: Compute the final size of the data structure on disk. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@lbl.gov + * November 17, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F__cache_drvrinfo_get_final_load_size(const void *_image, size_t image_len, + void *_udata, size_t *actual_len) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5F_drvrinfo_cache_ud_t *udata = (H5F_drvrinfo_cache_ud_t *)_udata; /* User data */ + unsigned drv_vers; /* Version of driver info block */ + size_t drvinfo_len; /* Length of encoded buffer */ + haddr_t eoa; /* Current EOA for the file */ + haddr_t min_eoa; /* Minimum EOA needed for reading the driver info */ + htri_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check arguments */ + HDassert(image); + HDassert(udata); + HDassert(udata->f); + HDassert(actual_len); + HDassert(*actual_len == image_len); + + /* Version number */ + drv_vers = *image++; + if(drv_vers != HDF5_DRIVERINFO_VERSION_0) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad driver information block version number") + + image += 3; /* reserved bytes */ + + /* Driver info size */ + UINT32DECODE(image, drvinfo_len); + + /* Sanity check */ + HDassert(image_len == H5F_DRVINFOBLOCK_HDR_SIZE); + + /* Extend the EOA if required so that we can read the complete driver info block */ + + /* Get current EOA... */ + if((eoa = H5FD_get_eoa(udata->f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* ... if it is too small, extend it. */ + min_eoa = udata->driver_addr + H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo_len; + + /* If it grew, set it */ + if(H5F_addr_gt(min_eoa, eoa)) + if(H5FD_set_eoa(udata->f->shared->lf, H5FD_MEM_SUPER, min_eoa) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed") + + /* Set the final size for the cache image */ + *actual_len = H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo_len; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__cache_drvrinfo_get_final_load_size() */ /*------------------------------------------------------------------------- @@ -850,35 +1029,11 @@ H5F__cache_drvrinfo_deserialize(const void *_image, size_t len, void *_udata, drv_name[8] = '\0'; image += 8; /* advance past name/version */ - /* Handle metadata cache retry for variable-sized portion of the driver info block */ - if(len != (H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo->len)) { - /* Sanity check */ - HDassert(len == H5F_DRVINFOBLOCK_HDR_SIZE); - - /* extend the eoa if required so that we can read the complete driver info block */ - { - haddr_t eoa; - haddr_t min_eoa; - - /* get current eoa... */ - if ((eoa = H5FD_get_eoa(udata->f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, NULL, "driver get_eoa request failed") - - /* ... if it is too small, extend it. */ - min_eoa = udata->driver_addr + H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo->len; + HDassert(len == (H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo->len)); - if ( H5F_addr_gt(min_eoa, eoa) ) - if(H5FD_set_eoa(udata->f->shared->lf, H5FD_MEM_SUPER, min_eoa) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, \ - "set end of space allocation request failed") - } - - } /* end if */ - else { - /* Validate and decode driver information */ - if(H5FD_sb_load(udata->f->shared->lf, drv_name, image) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "unable to decode driver information") - } /* end if */ + /* Validate and decode driver information */ + if(H5FD_sb_load(udata->f->shared->lf, drv_name, image) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "unable to decode driver information") /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) <= len); @@ -909,8 +1064,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5F__cache_drvrinfo_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5F__cache_drvrinfo_image_len(const void *_thing, size_t *image_len) { const H5O_drvinfo_t *drvinfo = (const H5O_drvinfo_t *)_thing; /* Pointer to the object */ @@ -921,11 +1075,10 @@ H5F__cache_drvrinfo_image_len(const void *_thing, size_t *image_len, HDassert(drvinfo->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(drvinfo->cache_info.type == H5AC_DRVRINFO); HDassert(image_len); - HDassert(compressed_ptr); /* Set the image length size */ *image_len = (size_t)(H5F_DRVINFOBLOCK_HDR_SIZE + /* Fixed-size portion of driver info block */ - drvinfo->len); /* Variable-size portion of driver info block */ + drvinfo->len); /* Variable-size portion of driver info block */ FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5F__cache_drvrinfo_image_len() */ diff --git a/src/H5Ftest.c b/src/H5Ftest.c index 344d7c1..e3760b7 100644 --- a/src/H5Ftest.c +++ b/src/H5Ftest.c @@ -186,3 +186,37 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F_get_maxaddr_test() */ + +/*------------------------------------------------------------------------- + * Function: H5F_get_sbe_addr_test + * + * Purpose: Retrieve the address of a superblock extension's object header + * for a file + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Quincey Koziol + * Jul 10, 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_get_sbe_addr_test(hid_t file_id, haddr_t *sbe_addr) +{ + H5F_t *file; /* File info */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check arguments */ + if(NULL == (file = (H5F_t *)H5I_object_verify(file_id, H5I_FILE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file") + + /* Retrieve maxaddr for file */ + *sbe_addr = file->shared->sblock->ext_addr; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_get_sbe_addr_test() */ + @@ -715,14 +715,14 @@ done: herr_t H5Gclose(hid_t group_id) { - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE1("e", "i", group_id); /* Check args */ if(NULL == H5I_object_verify(group_id,H5I_GROUP)) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a group") + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a group") /* * Decrement the counter on the group atom. It will be freed if the count diff --git a/src/H5Gcache.c b/src/H5Gcache.c index e3303d1..d153560 100644 --- a/src/H5Gcache.c +++ b/src/H5Gcache.c @@ -63,11 +63,10 @@ /********************/ /* Metadata cache (H5AC) callbacks */ -static herr_t H5G__cache_node_get_load_size(const void *udata, size_t *image_len); +static herr_t H5G__cache_node_get_initial_load_size(void *udata, size_t *image_len); static void *H5G__cache_node_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5G__cache_node_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5G__cache_node_image_len(const void *thing, size_t *image_len); static herr_t H5G__cache_node_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5G__cache_node_free_icr(void *thing); @@ -93,14 +92,15 @@ const H5AC_class_t H5AC_SNODE[1] = {{ "Symbol table node", /* Metadata client name (for debugging) */ H5FD_MEM_BTREE, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5G__cache_node_get_load_size, /* 'get_load_size' callback */ + H5G__cache_node_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + NULL, /* 'verify_chksum' callback */ H5G__cache_node_deserialize, /* 'deserialize' callback */ H5G__cache_node_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5G__cache_node_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5G__cache_node_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -113,17 +113,11 @@ H5FL_SEQ_EXTERN(H5G_entry_t); /*------------------------------------------------------------------------- - * Function: H5G__cache_node_get_load_size() + * Function: H5G__cache_node_get_initial_load_size() * - * Purpose: Determine the size of the on disk image of the node, and + * Purpose: Determine the size of the on-disk image of the node, and * return this value in *image_len. * - * Note that this computation requires access to the file pointer, - * which is not provided in the parameter list for this callback. - * Finesse this issue by passing in the file pointer twice to the - * H5AC_protect() call -- once as the file pointer proper, and - * again as the user data. - * * Return: Success: SUCCEED * Failure: FAIL * @@ -133,9 +127,9 @@ H5FL_SEQ_EXTERN(H5G_entry_t); *------------------------------------------------------------------------- */ static herr_t -H5G__cache_node_get_load_size(const void *_udata, size_t *image_len) +H5G__cache_node_get_initial_load_size(void *_udata, size_t *image_len) { - const H5F_t *f = (const H5F_t *)_udata; /* User data for callback */ + H5F_t *f = (H5F_t *)_udata; /* User data for callback */ FUNC_ENTER_STATIC_NOERR @@ -143,11 +137,11 @@ H5G__cache_node_get_load_size(const void *_udata, size_t *image_len) HDassert(f); HDassert(image_len); - /* report image length */ + /* Set the image length size */ *image_len = (size_t)(H5G_NODE_SIZE(f)); FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5G__cache_node_get_load_size() */ +} /* end H5G__cache_node_get_initial_load_size() */ /*------------------------------------------------------------------------- @@ -241,8 +235,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5G__cache_node_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5G__cache_node_image_len(const void *_thing, size_t *image_len) { const H5G_node_t *sym = (const H5G_node_t *)_thing; /* Pointer to object */ @@ -259,11 +252,6 @@ H5G__cache_node_image_len(const void *_thing, size_t *image_len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5G__cache_node_image_len() */ - -/*************************************/ -/* no H5G__cache_node_pre_serialize() */ -/*************************************/ - /*------------------------------------------------------------------------- * Function: H5G__cache_node_serialize @@ -323,11 +311,6 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5G__cache_node_serialize() */ - -/***************************************/ -/* no H5G__cache_node_notify() function */ -/***************************************/ - /*------------------------------------------------------------------------- * Function: H5G__cache_node_free_icr diff --git a/src/H5Gent.c b/src/H5Gent.c index 6020028..f64eaf0 100644 --- a/src/H5Gent.c +++ b/src/H5Gent.c @@ -460,7 +460,7 @@ H5G__ent_convert(H5F_t *f, hid_t dxpl_id, H5HL_t *heap, const char *name, targ_oloc.addr = lnk->u.hard.addr; /* Get the object header */ - if(NULL == (oh = H5O_protect(&targ_oloc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(&targ_oloc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_SYM, H5E_CANTPROTECT, FAIL, "unable to protect target object header") /* Check if a symbol table message exists */ diff --git a/src/H5Gint.c b/src/H5Gint.c index f5bccc3..72cc1a4 100644 --- a/src/H5Gint.c +++ b/src/H5Gint.c @@ -232,7 +232,7 @@ done: if(oloc_init) { if(H5O_dec_rc_by_loc(&(grp->oloc), dxpl_id) < 0) HDONE_ERROR(H5E_SYM, H5E_CANTDEC, NULL, "unable to decrement refcount on newly created object") - if(H5O_close(&(grp->oloc)) < 0) + if(H5O_close(&(grp->oloc), NULL) < 0) HDONE_ERROR(H5E_SYM, H5E_CLOSEERROR, NULL, "unable to release object header") if(H5O_delete(file, dxpl_id, grp->oloc.addr) < 0) HDONE_ERROR(H5E_SYM, H5E_CANTDELETE, NULL, "unable to delete object header") @@ -448,7 +448,7 @@ H5G_open_oid(H5G_t *grp, hid_t dxpl_id) done: if(ret_value < 0) { if(obj_opened) - H5O_close(&(grp->oloc)); + H5O_close(&(grp->oloc), NULL); if(grp->shared) grp->shared = H5FL_FREE(H5G_shared_t, grp->shared); } /* end if */ @@ -472,8 +472,9 @@ done: herr_t H5G_close(H5G_t *grp) { - hbool_t corked; /* Whether the group is corked or not */ - herr_t ret_value = SUCCEED; /* Return value */ + hbool_t corked; /* Whether the group is corked or not */ + hbool_t file_closed = TRUE; /* H5O_close also closed the file? */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -486,7 +487,7 @@ H5G_close(H5G_t *grp) if(0 == grp->shared->fo_count) { HDassert(grp != H5G_rootof(H5G_fileof(grp))); - /* Uncork cache entries with object address tag */ + /* Uncork cache entries with object address tag */ if(H5AC_cork(grp->oloc.file, grp->oloc.addr, H5AC__GET_CORKED, &corked) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTGET, FAIL, "unable to retrieve an object's cork status") if(corked) @@ -498,8 +499,18 @@ H5G_close(H5G_t *grp) HGOTO_ERROR(H5E_SYM, H5E_CANTRELEASE, FAIL, "can't decrement count for object") if(H5FO_delete(grp->oloc.file, H5AC_ind_read_dxpl_id, grp->oloc.addr) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTRELEASE, FAIL, "can't remove group from list of open objects") - if(H5O_close(&(grp->oloc)) < 0) + if(H5O_close(&(grp->oloc), &file_closed) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to close") + + /* Evict group metadata if evicting on close */ + if(!file_closed && H5F_SHARED(grp->oloc.file) && H5F_EVICT_ON_CLOSE(grp->oloc.file)) { + if(H5AC_flush_tagged_metadata(grp->oloc.file, grp->oloc.addr, H5AC_ind_read_dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush tagged metadata") + if(H5AC_evict_tagged_metadata(grp->oloc.file, grp->oloc.addr, FALSE, H5AC_ind_read_dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to evict tagged metadata") + } /* end if */ + + /* Free memory */ grp->shared = H5FL_FREE(H5G_shared_t, grp->shared); } else { /* Decrement the ref. count for this object in the top file */ @@ -508,7 +519,7 @@ H5G_close(H5G_t *grp) /* Check reference count for this object in the top file */ if(H5FO_top_count(grp->oloc.file, grp->oloc.addr) == 0) { - if(H5O_close(&(grp->oloc)) < 0) + if(H5O_close(&(grp->oloc), NULL) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to close") } /* end if */ else @@ -521,7 +532,7 @@ H5G_close(H5G_t *grp) */ if(grp->shared->mounted && grp->shared->fo_count == 1) { /* Attempt to close down the file hierarchy */ - if(H5F_try_close(grp->oloc.file) < 0) + if(H5F_try_close(grp->oloc.file, NULL) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "problem attempting file close") } /* end if */ } /* end else */ diff --git a/src/H5Gtest.c b/src/H5Gtest.c index d69a804..0e0a897 100644 --- a/src/H5Gtest.c +++ b/src/H5Gtest.c @@ -701,7 +701,7 @@ H5G_verify_cached_stabs_test_cb(H5F_t *f, hid_t dxpl_id, targ_oloc.addr = sn->entry[i].header; /* Load target object header */ - if(NULL == (targ_oh = H5O_protect(&targ_oloc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (targ_oh = H5O_protect(&targ_oloc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_SYM, H5E_CANTPROTECT, H5_ITER_ERROR, "unable to protect target object header") /* Check if a symbol table message exists */ diff --git a/src/H5Gtraverse.c b/src/H5Gtraverse.c index 4d84e9f..d6c90e4 100644 --- a/src/H5Gtraverse.c +++ b/src/H5Gtraverse.c @@ -727,7 +727,7 @@ H5G_traverse_real(const H5G_loc_t *_loc, const char *name, unsigned target, HGOTO_ERROR(H5E_SYM, H5E_CANTDEC, FAIL, "unable to decrement refcount on newly created object") /* Close new group */ - if(H5O_close(obj_loc.oloc) < 0) + if(H5O_close(obj_loc.oloc, NULL) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to close") /* If the parent group was holding the file open, the diff --git a/src/H5HFcache.c b/src/H5HFcache.c index 31ecc62..04f1459 100644 --- a/src/H5HFcache.c +++ b/src/H5HFcache.c @@ -73,43 +73,40 @@ static herr_t H5HF__dtable_encode(H5F_t *f, uint8_t **pp, const H5HF_dtable_t *d static herr_t H5HF__dtable_decode(H5F_t *f, const uint8_t **pp, H5HF_dtable_t *dtable); /* Metadata cache (H5AC) callbacks */ -static herr_t H5HF__cache_hdr_get_load_size(const void *udata, size_t *image_len); +static herr_t H5HF__cache_hdr_get_initial_load_size(void *udata, size_t *image_len); +static herr_t H5HF__cache_hdr_get_final_load_size(const void *image_ptr, + size_t image_len, void *udata, size_t *actual_len); +static htri_t H5HF__cache_hdr_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5HF__cache_hdr_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5HF__cache_hdr_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5HF__cache_hdr_image_len(const void *thing, size_t *image_len); static herr_t H5HF__cache_hdr_pre_serialize(const H5F_t *f, hid_t dxpl_id, - void *thing, haddr_t addr, size_t len, size_t compressed_len, - haddr_t *new_addr, size_t *new_len, size_t *new_compressed_len, + void *thing, haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len, unsigned *flags); static herr_t H5HF__cache_hdr_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5HF__cache_hdr_free_icr(void *thing); -static herr_t H5HF__cache_iblock_get_load_size(const void *udata, size_t *image_len); +static herr_t H5HF__cache_iblock_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5HF__cache_iblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5HF__cache_iblock_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5HF__cache_iblock_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5HF__cache_iblock_image_len(const void *thing, size_t *image_len); static herr_t H5HF__cache_iblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, - void *thing, haddr_t addr, size_t len, size_t compressed_len, - haddr_t *new_addr, size_t *new_len, size_t *new_compressed_len, + void *thing, haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len, unsigned *flags); static herr_t H5HF__cache_iblock_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5HF__cache_iblock_notify(H5C_notify_action_t action, void *thing); static herr_t H5HF__cache_iblock_free_icr(void *thing); -static herr_t H5HF__cache_dblock_get_load_size(const void *udata, size_t *image_len); +static herr_t H5HF__cache_dblock_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5HF__cache_dblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5HF__cache_dblock_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5HF__cache_dblock_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5HF__cache_dblock_image_len(const void *thing, size_t *image_len); static herr_t H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, - void *thing, haddr_t addr, size_t len, size_t compressed_len, - haddr_t *new_addr, size_t *new_len, size_t *new_compressed_len, + void *thing, haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len, unsigned *flags); static herr_t H5HF__cache_dblock_serialize(const H5F_t *f, void *image, size_t len, void *thing); @@ -139,14 +136,15 @@ const H5AC_class_t H5AC_FHEAP_HDR[1] = {{ "fractal heap header", /* Metadata client name (for debugging) */ H5FD_MEM_FHEAP_HDR, /* File space memory type for client */ H5AC__CLASS_SPECULATIVE_LOAD_FLAG, /* Client class behavior flags */ - H5HF__cache_hdr_get_load_size, /* 'get_load_size' callback */ + H5HF__cache_hdr_get_initial_load_size, /* 'get_initial_load_size' callback */ + H5HF__cache_hdr_get_final_load_size, /* 'get_final_load_size' callback */ + H5HF__cache_hdr_verify_chksum, /* 'verify_chksum' callback */ H5HF__cache_hdr_deserialize, /* 'deserialize' callback */ H5HF__cache_hdr_image_len, /* 'image_len' callback */ H5HF__cache_hdr_pre_serialize, /* 'pre_serialize' callback */ H5HF__cache_hdr_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5HF__cache_hdr_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -156,14 +154,15 @@ const H5AC_class_t H5AC_FHEAP_IBLOCK[1] = {{ "fractal heap indirect block", /* Metadata client name (for debugging) */ H5FD_MEM_FHEAP_IBLOCK, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5HF__cache_iblock_get_load_size, /* 'get_load_size' callback */ + H5HF__cache_iblock_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5HF__cache_iblock_verify_chksum, /* 'verify_chksum' callback */ H5HF__cache_iblock_deserialize, /* 'deserialize' callback */ H5HF__cache_iblock_image_len, /* 'image_len' callback */ H5HF__cache_iblock_pre_serialize, /* 'pre_serialize' callback */ H5HF__cache_iblock_serialize, /* 'serialize' callback */ H5HF__cache_iblock_notify, /* 'notify' callback */ H5HF__cache_iblock_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -172,15 +171,16 @@ const H5AC_class_t H5AC_FHEAP_DBLOCK[1] = {{ H5AC_FHEAP_DBLOCK_ID, /* Metadata client ID */ "fractal heap direct block", /* Metadata client name (for debugging) */ H5FD_MEM_FHEAP_DBLOCK, /* File space memory type for client */ - H5C__CLASS_COMPRESSED_FLAG, /* Client class behavior flags */ - H5HF__cache_dblock_get_load_size, /* 'get_load_size' callback */ + H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ + H5HF__cache_dblock_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5HF__cache_dblock_verify_chksum, /* 'verify_chksum' callback */ H5HF__cache_dblock_deserialize, /* 'deserialize' callback */ H5HF__cache_dblock_image_len, /* 'image_len' callback */ H5HF__cache_dblock_pre_serialize, /* 'pre_serialize' callback */ H5HF__cache_dblock_serialize, /* 'serialize' callback */ H5HF__cache_dblock_notify, /* 'notify' callback */ H5HF__cache_dblock_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -298,21 +298,13 @@ H5HF__dtable_encode(H5F_t *f, uint8_t **pp, const H5HF_dtable_t *dtable) FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HF__dtable_encode() */ -/**************************************************/ -/* metadata cache callback definitions for header */ -/**************************************************/ - /*------------------------------------------------------------------------- - * Function: H5HF__cache_hdr_get_load_size() + * Function: H5HF__cache_hdr_get_initial_load_size() * * Purpose: Determine the size of the fractal heap header on disk, * and set *image_len to this value. * - * This code is based on the old H5HF_cache_hdr_load() routine - * that was used with the version 2 metadata cache. Note the - * use of a dummy header to compute the on disk size of the header. - * * Note also that the value returned by this function presumes that * there is no I/O filtering data in the header. If there is, the * size reported will be too small, and H5C_load_entry() @@ -327,10 +319,10 @@ H5HF__dtable_encode(H5F_t *f, uint8_t **pp, const H5HF_dtable_t *dtable) *------------------------------------------------------------------------- */ static herr_t -H5HF__cache_hdr_get_load_size(const void *_udata, size_t *image_len) +H5HF__cache_hdr_get_initial_load_size(void *_udata, size_t *image_len) { - const H5HF_hdr_cache_ud_t *udata = (const H5HF_hdr_cache_ud_t *)_udata; /* pointer to user data */ - H5HF_hdr_t dummy_hdr; /* dummy header -- to compute size */ + H5HF_hdr_cache_ud_t *udata = (H5HF_hdr_cache_ud_t *)_udata; /* Pointer to user data */ + H5HF_hdr_t dummy_hdr; /* Dummy header -- to compute size */ FUNC_ENTER_STATIC_NOERR @@ -347,40 +339,108 @@ H5HF__cache_hdr_get_load_size(const void *_udata, size_t *image_len) *image_len = (size_t)H5HF_HEADER_SIZE(&dummy_hdr); FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5HF__cache_hdr_get_load_size() */ +} /* end H5HF__cache_hdr_get_initial_load_size() */ /*------------------------------------------------------------------------- - * Function: H5HF__cache_hdr_deserialize + * Function: H5HF__cache_hdr_get_final_load_size() * - * Purpose: Given a buffer containing an on disk image of a fractal heap - * header block, allocate an instance of H5HF_hdr_t, load the contents - * of the buffer into into the new instance of H5HF_hdr_t, and then - * return a pointer to the new instance. + * Purpose: Determine the final size of the fractal heap header on disk, + * and set *actual_len to this value. * - * Since H5HF__cache_hdr_get_load_size() reports header on disk size - * base on the assumption that the header contains no I/O filtering - * data, it is possible that the provided image will be too small. + * Return: Success: SUCCEED + * Failure: FAIL * - * In this case, we DO NOT flag an error when this is discovered. - * Instead, we make note of the correct image size, and report - * success. + * Programmer: Quincey Koziol + * November 18, 2016 * - * Since H5HF__cache_hdr_image_len() callback is defined, - * H5C_load_entry() will call H5HF__cache_hdr_image_len() and - * obtain the correct image length. + *------------------------------------------------------------------------- + */ +static herr_t +H5HF__cache_hdr_get_final_load_size(const void *_image, size_t image_len, + void *_udata, size_t *actual_len) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5HF_hdr_cache_ud_t *udata = (H5HF_hdr_cache_ud_t *)_udata; /* pointer to user data */ + unsigned filter_len; /* Size of I/O filter information (in bytes) */ + htri_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(image); + HDassert(udata); + HDassert(actual_len); + HDassert(*actual_len == image_len); + + /* Magic number */ + if(HDmemcmp(image, H5HF_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "wrong fractal heap header signature") + image += H5_SIZEOF_MAGIC; + + /* Version */ + if(*image++ != H5HF_HDR_VERSION) + HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong fractal heap header version") + + /* General heap information */ + image += 2; /* Heap ID length */ + UINT16DECODE(image, filter_len); /* I/O filters' encoded length */ + + /* Check for I/O filter info on this heap */ + if(filter_len > 0) + /* Compute the extra heap header size */ + *actual_len += (size_t)(H5F_SIZEOF_SIZE(udata->f) /* Size of size for filtered root direct block */ + + (unsigned)4 /* Size of filter mask for filtered root direct block */ + + filter_len); /* Size of encoded I/O filter info */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF__cache_hdr_get_final_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF__cache_hdr_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 * - * Since the H5AC__CLASS_SPECULATIVE_LOAD_FLAG is set, - * H5C_load_entry() will load an image of the correct size, and - * then call this function again to deserialize it. Before doing - * so, it will also call H5HF__cache_hdr_free_icr() to discard the - * result of the first deserialize call. + *------------------------------------------------------------------------- + */ +static htri_t +H5HF__cache_hdr_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF__cache_hdr_verify_chksum() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF__cache_hdr_deserialize * - * Note that the v2 B-tree and free space manager associated - * with the fractal heap (roots stored in the huge_bt2 and fspace - * fields respectively) are not loaded at this time. As best I can - * tell from reviewing the code, they are loaded or created when - * they are accessed. + * Purpose: Given a buffer containing an on disk image of a fractal heap + * header block, allocate an instance of H5HF_hdr_t, load the contents + * of the buffer into into the new instance of H5HF_hdr_t, and then + * return a pointer to the new instance. * * Return: Success: Pointer to in core representation * Failure: NULL @@ -397,9 +457,7 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, H5HF_hdr_t *hdr = NULL; /* Fractal heap info */ H5HF_hdr_cache_ud_t *udata = (H5HF_hdr_cache_ud_t *)_udata; /* User data for callback */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into into supplied image */ - size_t size; /* Header size */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ uint8_t heap_flags; /* Status flags for heap */ void * ret_value = NULL; /* Return value */ @@ -415,28 +473,6 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, if(NULL == (hdr = H5HF_hdr_alloc(udata->f))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - /* Compute the 'base' size of the fractal heap header on disk */ - size = (size_t)H5HF_HEADER_SIZE(hdr); - - /* the size we have just calculated presumes that there is no I/O - * filter information in the header. If there is no filter information, - * the deserialize operation should succeed. - * - * If there is filter information, the first attempt to deserialize - * the header will reveal this. In this case, we will be unable to - * deserialize the header as the supplied image will be too small. - * However, we will make note of the correct size and report success - * anyway. - * - * When H5C_load_entry() calls H5HF__cache_hdr_image_len(), we will report - * the correct size. Since the H5C__CLASS_SPECULATIVE_LOAD_FLAG is set, - * this will prompt H5C_load_entry() to load the correct size image, - * discard the result of the first attempt at deserialization, and - * call this routine a second time to deserialize the correct size - * buffer. - */ - HDassert(size <= len); - /* Magic number */ if(HDmemcmp(image, H5HF_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "wrong fractal heap header signature") @@ -480,32 +516,24 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, if(H5HF__dtable_decode(hdr->f, &image, &(hdr->man_dtable)) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTENCODE, NULL, "unable to encode managed obj. doubling table info") + /* Set the fractal heap header's 'base' size */ + hdr->heap_size = (size_t)H5HF_HEADER_SIZE(hdr); + /* Sanity check */ /* (allow for checksum not decoded yet) */ - HDassert((size_t)(image - (const uint8_t *)_image) == (size - H5HF_SIZEOF_CHKSUM)); + HDassert((size_t)(image - (const uint8_t *)_image) == (hdr->heap_size - H5HF_SIZEOF_CHKSUM)); /* Check for I/O filter information to decode */ if(hdr->filter_len > 0) { - size_t filter_info_size; /* Size of filter information */ H5O_pline_t *pline; /* Pipeline information from the header on disk */ - /* Compute the size of the extra filter information */ - filter_info_size = (size_t)(hdr->sizeof_size /* Size of size for filtered root direct block */ - + (unsigned)4 /* Size of filter mask for filtered root direct block */ - + hdr->filter_len); /* Size of encoded I/O filter info */ + /* Sanity check */ + HDassert(len > hdr->heap_size); /* A header with filter info is > than a standard header */ /* Compute the heap header's size */ - hdr->heap_size = size + filter_info_size; - - if(size == len) - /* we were supplied with too small a buffer -- goto done - * and let H5C_load_entry() retry with a larger buffer - */ - HGOTO_DONE((void *)hdr) - - else - if((size + filter_info_size) != len) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "bad image len") + hdr->heap_size += (size_t)(hdr->sizeof_size /* Size of size for filtered root direct block */ + + (unsigned)4 /* Size of filter mask for filtered root direct block */ + + hdr->filter_len); /* Size of encoded I/O filter info */ /* Decode the size of a filtered root direct block */ H5F_DECODE_LENGTH(udata->f, image, hdr->pline_root_direct_size); @@ -517,6 +545,7 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, if(NULL == (pline = (H5O_pline_t *)H5O_msg_decode(hdr->f, udata->dxpl_id, NULL, H5O_PLINE_ID, image))) HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, NULL, "can't decode I/O pipeline filters") + /* Advance past filter info to checksum */ image += hdr->filter_len; /* Copy the information into the header's I/O pipeline structure */ @@ -526,13 +555,6 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, /* Release the space allocated for the I/O pipeline filters */ H5O_msg_free(H5O_PLINE_ID, pline); } /* end if */ - else - /* Set the heap header's size */ - hdr->heap_size = size; - - /* Compute checksum on entire header */ - /* (including the filter information, if present) */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -540,10 +562,6 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == hdr->heap_size); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap header") - /* Finish initialization of heap header */ if(H5HF_hdr_finish_init(hdr) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, NULL, "can't finish initializing shared fractal heap header") @@ -567,7 +585,7 @@ done: * disk image. * * If the header contains filter information, this size will be - * larger than the value returned by H5HF__cache_hdr_get_load_size(). + * larger than the value returned by H5HF__cache_hdr_get_initial_load_size(). * * Return: Success: SUCCEED * Failure: FAIL @@ -578,8 +596,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF__cache_hdr_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5HF__cache_hdr_image_len(const void *_thing, size_t *image_len) { const H5HF_hdr_t *hdr = (const H5HF_hdr_t *)_thing; /* Fractal heap info */ @@ -619,9 +636,8 @@ H5HF__cache_hdr_image_len(const void *_thing, size_t *image_len, */ static herr_t H5HF__cache_hdr_pre_serialize(const H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, - void *_thing, haddr_t addr, size_t len, size_t H5_ATTR_UNUSED compressed_len, - haddr_t H5_ATTR_UNUSED *new_addr, size_t H5_ATTR_UNUSED *new_len, - size_t H5_ATTR_UNUSED *new_compressed_len, unsigned *flags) + void *_thing, haddr_t addr, size_t len, haddr_t H5_ATTR_UNUSED *new_addr, + size_t H5_ATTR_UNUSED *new_len, unsigned *flags) { H5HF_hdr_t *hdr = (H5HF_hdr_t *)_thing; /* Fractal heap info */ herr_t ret_value = SUCCEED; /* Return value */ @@ -783,10 +799,6 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5HF__cache_hdr_serialize() */ -/***************************************/ -/* no H5HF__cache_hdr_notify() function */ -/***************************************/ - /*------------------------------------------------------------------------- * Function: H5HF__cache_hdr_free_icr @@ -833,13 +845,9 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5HF__cache_hdr_free_icr() */ -/***********************************************************/ -/* metadata cache callback definitions for indirect blocks */ -/***********************************************************/ - /*------------------------------------------------------------------------- - * Function: H5HF__cache_iblock_get_load_size() + * Function: H5HF__cache_iblock_get_initial_load_size() * * Purpose: Compute the size of the on disk image of the indirect * block, and place this value in *image_len. @@ -853,24 +861,60 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF__cache_iblock_get_load_size(const void *_udata, size_t *image_len) +H5HF__cache_iblock_get_initial_load_size(void *_udata, size_t *image_len) { - const H5HF_iblock_cache_ud_t *udata = (const H5HF_iblock_cache_ud_t *)_udata; /* User data for callback */ + H5HF_iblock_cache_ud_t *udata = (H5HF_iblock_cache_ud_t *)_udata; /* User data for callback */ FUNC_ENTER_STATIC_NOERR /* Sanity checks */ HDassert(udata); + HDassert(udata->par_info); + HDassert(udata->par_info->hdr); HDassert(image_len); + /* Set the image length size */ *image_len = (size_t)H5HF_MAN_INDIRECT_SIZE(udata->par_info->hdr, *udata->nrows); FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5HF__cache_iblock_get_load_size() */ +} /* end H5HF__cache_iblock_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF__cache_iblock_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5HF__cache_iblock_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF__cache_iblock_verify_chksum() */ -/***********************************************************/ -/* metadata cache callback definitions for indirect blocks */ -/***********************************************************/ /*------------------------------------------------------------------------- @@ -903,7 +947,6 @@ H5HF__cache_iblock_deserialize(const void *_image, size_t len, void *_udata, const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ haddr_t heap_addr; /* Address of heap header in the file */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ unsigned u; /* Local index variable */ void * ret_value = NULL; /* Return value */ @@ -1030,8 +1073,7 @@ H5HF__cache_iblock_deserialize(const void *_image, size_t len, void *_udata, /* Sanity check */ HDassert(iblock->nchildren); /* indirect blocks w/no children should have been deleted */ - /* Compute checksum on indirect block */ - computed_chksum = H5_checksum_metadata((const uint8_t *)_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done by verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -1039,10 +1081,6 @@ H5HF__cache_iblock_deserialize(const void *_image, size_t len, void *_udata, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == iblock->size); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap indirect block") - /* Check if we have any indirect block children */ if(iblock->nrows > hdr->man_dtable.max_direct_rows) { unsigned indir_rows;/* Number of indirect rows in this indirect block */ @@ -1083,8 +1121,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF__cache_iblock_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5HF__cache_iblock_image_len(const void *_thing, size_t *image_len) { const H5HF_indirect_t *iblock = (const H5HF_indirect_t *)_thing; /* Indirect block info */ @@ -1123,9 +1160,8 @@ H5HF__cache_iblock_image_len(const void *_thing, size_t *image_len, */ static herr_t H5HF__cache_iblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, - haddr_t addr, size_t H5_ATTR_UNUSED len, size_t H5_ATTR_UNUSED compressed_len, - haddr_t *new_addr, size_t H5_ATTR_UNUSED *new_len, - size_t H5_ATTR_UNUSED *new_compressed_len, unsigned *flags) + haddr_t addr, size_t H5_ATTR_UNUSED len, haddr_t *new_addr, + size_t H5_ATTR_UNUSED *new_len, unsigned *flags) { H5HF_hdr_t *hdr; /* Shared fractal heap information */ H5HF_indirect_t *iblock = (H5HF_indirect_t *)_thing; /* Indirect block info */ @@ -1379,19 +1415,9 @@ H5HF__cache_iblock_notify(H5C_notify_action_t action, void *_thing) if(action == H5AC_NOTIFY_ACTION_BEFORE_EVICT) HDassert((iblock->parent == iblock->fd_parent) || ((NULL == iblock->parent) && (iblock->fd_parent))); - else - HDassert(iblock->parent == iblock->fd_parent); /* further sanity checks */ if(iblock->parent == NULL) { - /* Either this is the root iblock, or the parent pointer is */ - /* invalid. Since we save a copy of the parent pointer on */ - /* the insertion event, it doesn't matter if the parent pointer */ - /* is invalid just before eviction. However, we will not be */ - /* able to function if it is invalid on the insertion event. */ - /* Scream and die if this is the case. */ - HDassert((action == H5C_NOTIFY_ACTION_BEFORE_EVICT) || (iblock->block_off == 0)); - /* pointer from hdr to root iblock will not be set up unless */ /* the fractal heap has already pinned the hdr. Do what */ /* sanity checking we can. */ @@ -1433,6 +1459,10 @@ H5HF__cache_iblock_notify(H5C_notify_action_t action, void *_thing) break; case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: /* do nothing */ break; @@ -1500,13 +1530,9 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5HF__cache_iblock_free_icr() */ -/*********************************************************/ -/* metadata cache callback definitions for direct blocks */ -/*********************************************************/ - /*------------------------------------------------------------------------- - * Function: H5HF__cache_dblock_get_load_size() + * Function: H5HF__cache_dblock_get_initial_load_size() * * Purpose: Determine the size of the direct block on disk image, and * return it in *image_len. @@ -1520,44 +1546,158 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF__cache_dblock_get_load_size(const void *_udata, size_t *image_len) +H5HF__cache_dblock_get_initial_load_size(void *_udata, size_t *image_len) { const H5HF_dblock_cache_ud_t *udata = (const H5HF_dblock_cache_ud_t *)_udata; /* User data for callback */ - const H5HF_parent_t *par_info; /* Pointer to parent information */ - const H5HF_hdr_t *hdr; /* Shared fractal heap information */ - size_t size; + const H5HF_parent_t *par_info; /* Pointer to parent information */ + const H5HF_hdr_t *hdr; /* Shared fractal heap information */ FUNC_ENTER_STATIC_NOERR /* Sanity checks */ HDassert(udata); HDassert(image_len); + + /* Convenience variables */ par_info = (const H5HF_parent_t *)(&(udata->par_info)); HDassert(par_info); hdr = par_info->hdr; HDassert(hdr); - HDassert(hdr->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); - HDassert(hdr->cache_info.type == H5AC_FHEAP_HDR); /* Check for I/O filters on this heap */ if(hdr->filter_len > 0) { /* Check for root direct block */ if(par_info->iblock == NULL) - size = hdr->pline_root_direct_size; + /* filtered root direct block */ + *image_len = hdr->pline_root_direct_size; else - size = par_info->iblock->filt_ents[par_info->entry].size; + /* filtered direct block */ + *image_len = par_info->iblock->filt_ents[par_info->entry].size; + } /* end if */ + else + *image_len = udata->dblock_size; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5HF__cache_dblock_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF__cache_dblock_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5HF__cache_dblock_verify_chksum(const void *_image, size_t len, void *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5HF_dblock_cache_ud_t *udata = (H5HF_dblock_cache_ud_t *)_udata; /* User data for callback */ + void *read_buf = NULL; /* Pointer to buffer to read in */ + H5HF_hdr_t *hdr; /* Shared fractal heap information */ + H5HF_parent_t *par_info; /* Pointer to parent information */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + size_t chk_size; /* The size for validating checksum */ + uint8_t *chk_p; /* Pointer to the area for validating checksum */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(image); + HDassert(udata); + par_info = (H5HF_parent_t *)(&(udata->par_info)); + HDassert(par_info); + hdr = par_info->hdr; + HDassert(hdr); + + /* Reset callback context info */ + udata->decompressed = FALSE; + udata->dblk = NULL; + + /* Get out if data block is not checksummed */ + if(!(hdr->checksum_dblocks)) + HGOTO_DONE(TRUE); + + if(hdr->filter_len > 0) { + size_t nbytes; /* Number of bytes used in buffer, after applying reverse filters */ + unsigned filter_mask; /* Excluded filters for direct block */ + H5Z_cb_t filter_cb = {NULL, NULL}; /* Filter callback structure */ + + /* Allocate buffer to perform I/O filtering on and copy image into + * it. Must do this as H5Z_pipeline() may re-size the buffer + * provided to it. + */ + if(NULL == (read_buf = H5MM_malloc(len))) + HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "memory allocation failed for pipeline buffer") + + /* Set up parameters for filter pipeline */ + nbytes = len; + filter_mask = udata->filter_mask; + HDmemcpy(read_buf, image, len); + + /* Push direct block data through I/O filter pipeline */ + if(H5Z_pipeline(&(hdr->pline), H5Z_FLAG_REVERSE, &filter_mask, H5Z_ENABLE_EDC, filter_cb, &nbytes, &len, &read_buf) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTFILTER, FAIL, "output pipeline failed") + + /* Update info about direct block */ + udata->decompressed = TRUE; + len = nbytes; } /* end if */ else - size = udata->dblock_size; + read_buf = (void *)image; /* Casting away const OK - QAK */ - *image_len = size; + /* Decode checksum */ + chk_size = (size_t)(H5HF_MAN_ABS_DIRECT_OVERHEAD(hdr) - H5HF_SIZEOF_CHKSUM); + chk_p = (uint8_t *)read_buf + chk_size; - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5HF__cache_dblock_get_load_size() */ + /* Metadata checksum */ + UINT32DECODE(chk_p, stored_chksum); + + chk_p -= H5HF_SIZEOF_CHKSUM; + + /* Reset checksum field, for computing the checksum */ + /* (Casting away const OK - QAK) */ + HDmemset(chk_p, 0, (size_t)H5HF_SIZEOF_CHKSUM); + + /* Compute checksum on entire direct block */ + computed_chksum = H5_checksum_metadata(read_buf, len, 0); + + /* Restore the checksum */ + UINT32ENCODE(chk_p, stored_chksum) + + /* Verify checksum */ + if(stored_chksum != computed_chksum) + HGOTO_DONE(FALSE); + + /* Save the decompressed data to be used later in deserialize callback */ + if(hdr->filter_len > 0) { + /* Sanity check */ + HDassert(udata->decompressed); + HDassert(len == udata->dblock_size); -/*********************************************************/ -/* metadata cache callback definitions for direct blocks */ -/*********************************************************/ + /* Allocate block buffer */ + if(NULL == (udata->dblk = H5FL_BLK_MALLOC(direct_block, (size_t)len))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + + /* Copy un-filtered data into block's buffer */ + HDmemcpy(udata->dblk, read_buf, len); + } /* end if */ + +done: + /* Release the read buffer */ + if(read_buf && read_buf != image) + H5MM_xfree(read_buf); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF__cache_dblock_verify_chksum() */ /*------------------------------------------------------------------------- @@ -1588,14 +1728,15 @@ H5HF__cache_dblock_deserialize(const void *_image, size_t len, void *_udata, H5HF_dblock_cache_ud_t *udata = (H5HF_dblock_cache_ud_t *)_udata; /* User data for callback */ H5HF_parent_t *par_info; /* Pointer to parent information */ H5HF_direct_t *dblock = NULL; /* Direct block info */ - const uint8_t *image; /* Pointer into raw data buffer */ + const uint8_t *image = _image;/* Pointer into raw data buffer */ + void *read_buf = NULL; /* Pointer to buffer to decompress */ haddr_t heap_addr; /* Address of heap header in the file */ void * ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC /* Sanity checks */ - HDassert(_image); + HDassert(image); HDassert(udata); par_info = (H5HF_parent_t *)(&(udata->par_info)); HDassert(par_info); @@ -1606,7 +1747,7 @@ H5HF__cache_dblock_deserialize(const void *_image, size_t len, void *_udata, HDassert(dirty); /* Allocate space for the fractal heap direct block */ - if(NULL == (dblock = H5FL_MALLOC(H5HF_direct_t))) + if(NULL == (dblock = H5FL_CALLOC(H5HF_direct_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") HDmemset(&dblock->cache_info, 0, sizeof(H5AC_info_t)); @@ -1620,61 +1761,62 @@ H5HF__cache_dblock_deserialize(const void *_image, size_t len, void *_udata, /* Set block's internal information */ dblock->size = udata->dblock_size; - dblock->file_size = 0; - - /* initialize fields used in serialization */ - dblock->write_buf = NULL; - dblock->write_size = 0; - - /* Allocate block buffer */ -/* XXX: Change to using free-list factories */ - if(NULL == (dblock->blk = H5FL_BLK_MALLOC(direct_block, (size_t)dblock->size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") /* Check for I/O filters on this heap */ if(hdr->filter_len > 0) { - H5Z_cb_t filter_cb = {NULL, NULL}; /* Filter callback structure */ - size_t nbytes; /* Number of bytes used in buffer, after applying reverse filters */ - void *read_buf; /* Pointer to buffer to read in */ - size_t read_size; /* Size of filtered direct block to read */ - unsigned filter_mask; /* Excluded filters for direct block */ + /* Direct block is already decompressed in verify_chksum callback */ + if(udata->decompressed) { + /* Sanity check */ + HDassert(udata->dblk); - /* Check for root direct block */ - if(par_info->iblock == NULL) - /* Set up parameters to read filtered direct block */ - read_size = hdr->pline_root_direct_size; - else - /* Set up parameters to read filtered direct block */ - read_size = par_info->iblock->filt_ents[par_info->entry].size; - HDassert(len == read_size); + /* Take ownership of the decompressed direct block */ + dblock->blk = udata->dblk; + udata->dblk = NULL; + } /* end if */ + else { + H5Z_cb_t filter_cb = {NULL, NULL}; /* Filter callback structure */ + size_t nbytes; /* Number of bytes used in buffer, after applying reverse filters */ + unsigned filter_mask; /* Excluded filters for direct block */ - /* Allocate buffer to perform I/O filtering on and copy image into - * it. Must do this as H5Z_pipeline() may re-sized the buffer - * provided to it. - */ - if(NULL == (read_buf = H5MM_malloc(read_size))) - HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, NULL, "memory allocation failed for pipeline buffer") - HDmemcpy(read_buf, _image, len); + /* Sanity check */ + HDassert(udata->dblk == NULL); - /* Push direct block data through I/O filter pipeline */ - nbytes = read_size; - filter_mask = udata->filter_mask; - if(H5Z_pipeline(&(hdr->pline), H5Z_FLAG_REVERSE, &filter_mask, H5Z_ENABLE_EDC, filter_cb, &nbytes, &read_size, &read_buf) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTFILTER, NULL, "output pipeline failed") + /* Allocate buffer to perform I/O filtering on and copy image into + * it. Must do this as H5Z_pipeline() may resize the buffer + * provided to it. + */ + if(NULL == (read_buf = H5MM_malloc(len))) + HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, NULL, "memory allocation failed for pipeline buffer") - /* Sanity check */ - HDassert(nbytes == dblock->size); + /* Copy compressed image into buffer */ + HDmemcpy(read_buf, image, len); - /* Copy un-filtered data into block's buffer */ - HDmemcpy(dblock->blk, read_buf, dblock->size); + /* Push direct block data through I/O filter pipeline */ + nbytes = len; + filter_mask = udata->filter_mask; + if(H5Z_pipeline(&(hdr->pline), H5Z_FLAG_REVERSE, &filter_mask, H5Z_ENABLE_EDC, filter_cb, &nbytes, &len, &read_buf) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTFILTER, NULL, "output pipeline failed") - /* Release the read buffer */ - H5MM_xfree(read_buf); + /* Sanity check */ + HDassert(nbytes == dblock->size); + + /* Copy un-filtered data into block's buffer */ + HDmemcpy(dblock->blk, read_buf, dblock->size); + } /* end if */ } /* end if */ else { - /* copy image to dblock->blk */ + /* Sanity checks */ + HDassert(udata->dblk == NULL); + HDassert(!udata->decompressed); + + /* Allocate block buffer */ +/* XXX: Change to using free-list factories */ + if(NULL == (dblock->blk = H5FL_BLK_MALLOC(direct_block, (size_t)dblock->size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + + /* Copy image to dblock->blk */ HDassert(dblock->size == len); - HDmemcpy(dblock->blk, _image, dblock->size); + HDmemcpy(dblock->blk, image, dblock->size); } /* end else */ /* Start decoding direct block */ @@ -1709,22 +1851,12 @@ H5HF__cache_dblock_deserialize(const void *_image, size_t len, void *_udata, /* Decode checksum on direct block, if requested */ if(hdr->checksum_dblocks) { - uint32_t stored_chksum; /* Metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ + uint32_t stored_chksum; /* Metadata checksum value */ + + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); - - /* Reset checksum field, for computing the checksum */ - /* (Casting away const OK - QAK) */ - HDmemset((uint8_t *)image - H5HF_SIZEOF_CHKSUM, 0, (size_t)H5HF_SIZEOF_CHKSUM); - - /* Compute checksum on entire direct block */ - computed_chksum = H5_checksum_metadata(dblock->blk, dblock->size, 0); - - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap direct block") } /* end if */ /* Sanity check */ @@ -1734,6 +1866,11 @@ H5HF__cache_dblock_deserialize(const void *_image, size_t len, void *_udata, ret_value = (void *)dblock; done: + /* Release the read buffer */ + if(read_buf) + H5MM_xfree(read_buf); + + /* Cleanup on error */ if(!ret_value && dblock) if(H5HF_man_dblock_dest(dblock) < 0) HDONE_ERROR(H5E_HEAP, H5E_CANTFREE, NULL, "unable to destroy fractal heap direct block") @@ -1758,14 +1895,11 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF__cache_dblock_image_len(const void *_thing, size_t *image_len, hbool_t *compressed_ptr, size_t *compressed_image_len_ptr) +H5HF__cache_dblock_image_len(const void *_thing, size_t *image_len) { const H5HF_direct_t *dblock = (const H5HF_direct_t *)_thing; /* Direct block info */ - const H5HF_indirect_t *par_iblock; /* Parent iblock */ const H5HF_hdr_t *hdr; /* Shared fractal heap information */ - hbool_t compressed; size_t size; - size_t compressed_size; FUNC_ENTER_STATIC_NOERR @@ -1774,19 +1908,15 @@ H5HF__cache_dblock_image_len(const void *_thing, size_t *image_len, hbool_t *com HDassert(dblock->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(dblock->cache_info.type == H5AC_FHEAP_DBLOCK); HDassert(image_len); - HDassert(compressed_ptr); - HDassert(compressed_image_len_ptr); /* Set up convenience variables */ hdr = dblock->hdr; - par_iblock = dblock->parent; + HDassert(hdr); /* Check for I/O filters on this heap */ if(hdr->filter_len > 0) { - - /* Filters are enabled, so set compressed to TRUE, and set - * size equal to the uncompressed size of the direct block. - * If the data is available, set compressed_size to the compressed + /* + * If the data is available, set to the compressed * size of the direct block -- otherwise set it equal to the * uncompressed size. * @@ -1796,7 +1926,7 @@ H5HF__cache_dblock_image_len(const void *_thing, size_t *image_len, hbool_t *com * case, both dblock->file_size and the size stored in the * parent (either the header or the parent iblock) will all * be zero. In this case, return the uncompressed size - * stored in dblock->size as the compressed size. + * stored in dblock->size as the size. * * Second, the block may have just been serialized, in which * case, dblock->file_size should be zero, and the correct @@ -1810,42 +1940,26 @@ H5HF__cache_dblock_image_len(const void *_thing, size_t *image_len, hbool_t *com * and thus looking up the parent will likely return incorrect * data. */ - size = dblock->size; - compressed = TRUE; - compressed_size = dblock->size; /* will overwrite if compressed - * size is available. - */ - if(dblock->file_size != 0) - compressed_size = dblock->file_size; + size = dblock->file_size; else { - if(par_iblock) { - unsigned par_entry; /* Entry in parent indirect block */ + const H5HF_indirect_t *par_iblock = dblock->parent; /* Parent iblock */ - par_entry = dblock->par_entry; - compressed_size = par_iblock->filt_ents[par_entry].size; - - } /* end if */ - else { - compressed_size = hdr->pline_root_direct_size; - } - - if(compressed_size == 0) - compressed_size = dblock->size; + if(par_iblock) + size = par_iblock->filt_ents[dblock->par_entry].size; + else + size = hdr->pline_root_direct_size; + if(size == 0) + size = dblock->size; } /* end else */ } /* end if */ - else { + else size = dblock->size; - compressed = FALSE; - compressed_size = 0; /* a convenient, invalid value */ - } + /* Set the image size */ HDassert(size > 0); - - *image_len = size; - *compressed_ptr = compressed; - *compressed_image_len_ptr = compressed_size; + *image_len = size; FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HF__cache_dblock_image_len() */ @@ -1857,7 +1971,7 @@ H5HF__cache_dblock_image_len(const void *_thing, size_t *image_len, hbool_t *com * Purpose: In principle, the purpose of this function is to determine * the size and location of the disk image of the target direct * block. In this case, the uncompressed size of the block is - * fixed, but sined the H5C__CLASS_COMPRESSED_FLAG is set, + * fixed, but since the direct block could be compressed, * we may need to compute and report the compressed size. * * This is a bit sticky in the case of a direct block when I/O @@ -1869,8 +1983,8 @@ H5HF__cache_dblock_image_len(const void *_thing, size_t *image_len, hbool_t *com * * To complicate matters further, the direct block may have been * initially allocated in temporary (AKA imaginary) file space. - * In this case, we must relocate the direct block's on disk - * image to real file space regardless of whether it has changed + * In this case, we must relocate the direct block's on-disk + * image to "real" file space regardless of whether it has changed * size. * * One simplifying factor is the direct block's "blk" field, @@ -1915,8 +2029,7 @@ H5HF__cache_dblock_image_len(const void *_thing, size_t *image_len, hbool_t *com */ static herr_t H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, - haddr_t addr, size_t len, size_t compressed_len, haddr_t *new_addr, - size_t H5_ATTR_UNUSED *new_len, size_t *new_compressed_len, unsigned *flags) + haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len, unsigned *flags) { hbool_t at_tmp_addr; /* Flag to indicate direct block is */ /* at temporary address */ @@ -1942,18 +2055,14 @@ H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, HDassert(dblock->write_size == 0); HDassert(dblock->cache_info.size == len); HDassert(H5F_addr_defined(addr)); - HDassert(len == dblock->size); HDassert(new_addr); - HDassert(new_compressed_len); + HDassert(new_len); HDassert(flags); /* Set up local variables */ hdr = dblock->hdr; dblock_addr = addr; /* will update dblock_addr if we move the block */ - /* dblock->size must match dblock->cache_info.size */ - HDassert(dblock->cache_info.size == dblock->size); - /* Set the shared heap header's file context for this operation */ hdr->f = (H5F_t *)f; @@ -2041,9 +2150,10 @@ H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, /* Allocate buffer to perform I/O filtering on */ write_size = dblock->size; - if(NULL == (write_buf = H5MM_malloc(write_size))) HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "memory allocation failed for pipeline buffer") + + /* Copy the direct block's image into the buffer to compress */ HDmemcpy(write_buf, dblock->blk, write_size); /* Push direct block data through I/O filter pipeline */ @@ -2080,7 +2190,7 @@ H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, * size matches the heap's last record. This value will * likely change shortly. */ - HDassert(compressed_len == hdr->pline_root_direct_size); + HDassert(len == hdr->pline_root_direct_size); /* Check if we need to re-size the block on disk */ if(hdr->pline_root_direct_size != write_size || at_tmp_addr) { @@ -2089,26 +2199,20 @@ H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, * * (temp. file space does not need to be freed) */ - if(!at_tmp_addr) { + if(!at_tmp_addr) /* Release direct block's current disk space */ if(H5MF_xfree(f, H5FD_MEM_FHEAP_DBLOCK, dxpl_id, addr, (hsize_t)hdr->pline_root_direct_size) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTFREE, FAIL, "unable to free fractal heap direct block") - } /* end if */ /* Allocate space for the compressed direct block */ if(HADDR_UNDEF == (dblock_addr = H5MF_alloc((H5F_t *)f, H5FD_MEM_FHEAP_DBLOCK, dxpl_id, (hsize_t)write_size))) HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "file allocation failed for fractal heap direct block") - /* Let the metadata cache know, if the block moved */ - if(!H5F_addr_eq(hdr->man_dtable.table_addr, dblock_addr)) - if(H5AC_move_entry((H5F_t *)f, H5AC_FHEAP_DBLOCK, hdr->man_dtable.table_addr, dblock_addr, dxpl_id) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTMOVE, FAIL, "unable to move direct block") - /* Update information about compressed direct block's * location & size */ HDassert(hdr->man_dtable.table_addr == addr); - HDassert(hdr->pline_root_direct_size == compressed_len); + HDassert(hdr->pline_root_direct_size == len); hdr->man_dtable.table_addr = dblock_addr; hdr->pline_root_direct_size = write_size; @@ -2138,7 +2242,7 @@ H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, * size matches the heap's last record. This value will * likely change shortly. */ - HDassert(compressed_len == par_iblock->filt_ents[par_entry].size); + HDassert(len == par_iblock->filt_ents[par_entry].size); /* Check if we need to re-size the block on disk */ if(par_iblock->filt_ents[par_entry].size != write_size || at_tmp_addr) { @@ -2147,26 +2251,20 @@ H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, * * (temp. file space does not need to be freed) */ - if(!at_tmp_addr) { + if(!at_tmp_addr) /* Release direct block's current disk space */ if(H5MF_xfree(f, H5FD_MEM_FHEAP_DBLOCK, dxpl_id, addr, (hsize_t)par_iblock->filt_ents[par_entry].size) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTFREE, FAIL, "unable to free fractal heap direct block") - } /* end if */ /* Allocate space for the compressed direct block */ if(HADDR_UNDEF == (dblock_addr = H5MF_alloc((H5F_t *)f, H5FD_MEM_FHEAP_DBLOCK, dxpl_id, (hsize_t)write_size))) HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "file allocation failed for fractal heap direct block") - /* Let the metadata cache know, if the block moved */ - if(!H5F_addr_eq(par_iblock->ents[par_entry].addr, dblock_addr)) - if(H5AC_move_entry((H5F_t *)f, H5AC_FHEAP_DBLOCK, par_iblock->ents[par_entry].addr, dblock_addr, dxpl_id) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTMOVE, FAIL, "unable to move direct block") - /* Update information about compressed direct block's * location & size */ HDassert(par_iblock->ents[par_entry].addr == addr); - HDassert(par_iblock->filt_ents[par_entry].size == compressed_len); + HDassert(par_iblock->filt_ents[par_entry].size == len); par_iblock->ents[par_entry].addr = dblock_addr; par_iblock->filt_ents[par_entry].size = write_size; @@ -2198,22 +2296,16 @@ H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, * to 'normal' file space */ if(at_tmp_addr) { + /* Allocate 'normal' space for the direct block */ + if(HADDR_UNDEF == (dblock_addr = H5MF_alloc((H5F_t *)f, H5FD_MEM_FHEAP_DBLOCK, dxpl_id, (hsize_t)write_size))) + HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "file allocation failed for fractal heap direct block") + /* Check for root direct block */ if(NULL == dblock->parent) { - /* Sanity check */ + /* Sanity checks */ HDassert(H5F_addr_eq(hdr->man_dtable.table_addr, addr)); - - /* Allocate 'normal' space for the direct block */ - if(HADDR_UNDEF == (dblock_addr = H5MF_alloc((H5F_t *)f, H5FD_MEM_FHEAP_DBLOCK, dxpl_id, (hsize_t)write_size))) - HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "file allocation failed for fractal heap direct block") - - /* Sanity check */ HDassert(!H5F_addr_eq(hdr->man_dtable.table_addr, dblock_addr)); - /* Let the metadata cache know the block moved */ - if(H5AC_move_entry((H5F_t *)f, H5AC_FHEAP_DBLOCK, hdr->man_dtable.table_addr, dblock_addr, dxpl_id) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTMOVE, FAIL, "unable to move direct block") - /* Update information about direct block's location */ hdr->man_dtable.table_addr = dblock_addr; @@ -2222,22 +2314,12 @@ H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, HGOTO_ERROR(H5E_HEAP, H5E_CANTDIRTY, FAIL, "can't mark heap header as dirty") } /* end if */ else { /* the direct block's parent is an indirect block */ - /* Sanity check */ + /* Sanity checks */ HDassert(par_iblock); HDassert(par_iblock->ents); HDassert(H5F_addr_eq(par_iblock->ents[par_entry].addr, addr)); - - /* Allocate 'normal' space for the direct block */ - if(HADDR_UNDEF == (dblock_addr = H5MF_alloc((H5F_t *)f, H5FD_MEM_FHEAP_DBLOCK, dxpl_id, (hsize_t)write_size))) - HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "file allocation failed for fractal heap direct block") - - /* Sanity check */ HDassert(!H5F_addr_eq(par_iblock->ents[par_entry].addr, dblock_addr)); - /* Let the metadata cache know the block moved */ - if(H5AC_move_entry((H5F_t *)f, H5AC_FHEAP_DBLOCK, par_iblock->ents[par_entry].addr, dblock_addr, dxpl_id) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTMOVE, FAIL, "unable to move direct block") - /* Update information about direct block's location */ par_iblock->ents[par_entry].addr = dblock_addr; @@ -2267,9 +2349,9 @@ H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, *new_addr = dblock_addr; } /* end if */ - if((hdr->filter_len > 0) && (compressed_len != write_size)) { - dblock_flags |= H5C__SERIALIZE_COMPRESSED_FLAG; - *new_compressed_len = write_size; + if((hdr->filter_len > 0) && (len != write_size)) { + dblock_flags |= H5C__SERIALIZE_RESIZED_FLAG; + *new_len = write_size; } /* end if */ *flags = dblock_flags; @@ -2394,6 +2476,10 @@ H5HF__cache_dblock_notify(H5C_notify_action_t action, void *_thing) break; case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: /* do nothing */ break; diff --git a/src/H5HFiblock.c b/src/H5HFiblock.c index 341f423..bb31002 100644 --- a/src/H5HFiblock.c +++ b/src/H5HFiblock.c @@ -451,14 +451,11 @@ H5HF_man_iblock_root_create(H5HF_hdr_t *hdr, hid_t dxpl_id, size_t min_dblock_si /* destroy flush dependency between direct block and header */ if(H5AC_destroy_flush_dependency(dblock->hdr, dblock) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTUNDEPEND, FAIL, \ - "unable to destroy flush dependency") + HGOTO_ERROR(H5E_HEAP, H5E_CANTUNDEPEND, FAIL, "unable to destroy flush dependency") /* create flush dependency between direct block and new root indirect block */ if(H5AC_create_flush_dependency(dblock->parent, dblock) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTDEPEND, FAIL, \ - "unable to create flush dependency") - + HGOTO_ERROR(H5E_HEAP, H5E_CANTDEPEND, FAIL, "unable to create flush dependency") if(H5HF_man_iblock_attach(iblock, 0, hdr->man_dtable.table_addr) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTATTACH, FAIL, "can't attach root direct block to parent indirect block") diff --git a/src/H5HFpkg.h b/src/H5HFpkg.h index 6253160..6abae65 100644 --- a/src/H5HFpkg.h +++ b/src/H5HFpkg.h @@ -544,6 +544,14 @@ typedef struct H5HF_dblock_cache_ud_t { * calls to it. */ unsigned filter_mask; /* Excluded filters for direct block */ + uint8_t *dblk; /* Pointer to the buffer containing the decompressed + * direct block data obtained in verify_chksum callback. + * It will be used later in deserialize callback. + */ + htri_t decompressed; /* Indicate that the direct block has been + * decompressed in verify_chksum callback. + * It will be used later in deserialize callback. + */ } H5HF_dblock_cache_ud_t; @@ -155,7 +155,7 @@ H5HG_create(H5F_t *f, hid_t dxpl_id, size_t size) H5_CHECK_OVERFLOW(size, size_t, hsize_t); if(HADDR_UNDEF == (addr = H5MF_alloc(f, H5FD_MEM_GHEAP, dxpl_id, (hsize_t)size))) HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, HADDR_UNDEF, "unable to allocate file space for global heap") - if(NULL == (heap = H5FL_MALLOC(H5HG_heap_t))) + if(NULL == (heap = H5FL_CALLOC(H5HG_heap_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, HADDR_UNDEF, "memory allocation failed") heap->addr = addr; heap->size = size; diff --git a/src/H5HGcache.c b/src/H5HGcache.c index c1eb8d9..6bc9860 100644 --- a/src/H5HGcache.c +++ b/src/H5HGcache.c @@ -62,11 +62,12 @@ /********************/ /* Metadata cache callbacks */ -static herr_t H5HG__cache_heap_get_load_size(const void *udata, size_t *image_len); +static herr_t H5HG__cache_heap_get_initial_load_size(void *udata, size_t *image_len); +static herr_t H5HG__cache_heap_get_final_load_size(const void *_image, + size_t image_len, void *udata, size_t *actual_len); static void *H5HG__cache_heap_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5HG__cache_heap_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5HG__cache_heap_image_len(const void *thing, size_t *image_len); static herr_t H5HG__cache_heap_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5HG__cache_heap_free_icr(void *thing); @@ -82,14 +83,15 @@ const H5AC_class_t H5AC_GHEAP[1] = {{ "global heap", /* Metadata client name (for debugging) */ H5FD_MEM_GHEAP, /* File space memory type for client */ H5AC__CLASS_SPECULATIVE_LOAD_FLAG, /* Client class behavior flags */ - H5HG__cache_heap_get_load_size, /* 'get_load_size' callback */ + H5HG__cache_heap_get_initial_load_size, /* 'get_initial_load_size' callback */ + H5HG__cache_heap_get_final_load_size, /* 'get_final_load_size' callback */ + NULL, /* 'verify_chksum' callback */ H5HG__cache_heap_deserialize, /* 'deserialize' callback */ H5HG__cache_heap_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5HG__cache_heap_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5HG__cache_heap_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -106,13 +108,13 @@ const H5AC_class_t H5AC_GHEAP[1] = {{ /*------------------------------------------------------------------------- - * Function: H5HG__cache_heap_get_load_size() + * Function: H5HG__cache_heap_get_initial_load_size() * * Purpose: Return the initial speculative read size to the metadata * cache. This size will be used in the initial attempt to read * the global heap. If this read is too small, the cache will * try again with the correct value obtained from - * H5HG__cache_heap_image_len(). + * H5HG__cache_get_final_load_size(). * * Return: Success: SUCCEED * Failure: FAIL @@ -123,16 +125,74 @@ const H5AC_class_t H5AC_GHEAP[1] = {{ *------------------------------------------------------------------------- */ static herr_t -H5HG__cache_heap_get_load_size(const void H5_ATTR_UNUSED *_udata, size_t *image_len) +H5HG__cache_heap_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len) { FUNC_ENTER_STATIC_NOERR + /* Sanity check */ HDassert(image_len); + /* Set the image length size */ *image_len = (size_t)H5HG_MINSIZE; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5HG__cache_heap_get_load_size() */ +} /* end H5HG__cache_heap_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5HG__cache_heap_get_initial_load_size() + * + * Purpose: Return the final read size for a speculatively ready heap to + * the metadata cache. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * November 18, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5HG__cache_heap_get_final_load_size(const void *_image, size_t image_len, + void *_udata, size_t *actual_len) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5F_t *f = (H5F_t *)_udata; /* File pointer -- obtained from user data */ + size_t heap_size = 0; /* Total size of collection */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(image); + HDassert(f); + HDassert(actual_len); + HDassert(*actual_len == image_len); + + /* Magic number */ + if(HDmemcmp(image, H5HG_MAGIC, (size_t)H5_SIZEOF_MAGIC)) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad global heap collection signature") + image += H5_SIZEOF_MAGIC; + + /* Version */ + if(H5HG_VERSION != *image++) + HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong version number in global heap") + + /* Reserved */ + image += 3; + + /* Size */ + H5F_DECODE_LENGTH(f, image, heap_size); + HDassert(heap_size >= H5HG_MINSIZE); + HDassert(image_len == H5HG_MINSIZE); + + /* Set the final size for the cache image */ + *actual_len = heap_size; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HG__cache_heap_get_final_load_size() */ /*------------------------------------------------------------------------- @@ -142,10 +202,6 @@ H5HG__cache_heap_get_load_size(const void H5_ATTR_UNUSED *_udata, size_t *image_ * heap, deserialize it, load its contents into a newly allocated * instance of H5HG_heap_t, and return a pointer to the new instance. * - * Note that this heap client uses speculative reads. If the supplied - * buffer is too small, we simply make note of the correct size, and - * wait for the metadata cache to try again. - * * Return: Success: Pointer to in core representation * Failure: NULL * @@ -161,6 +217,8 @@ H5HG__cache_heap_deserialize(const void *_image, size_t len, void *_udata, H5F_t *f = (H5F_t *)_udata; /* File pointer -- obtained from user data */ H5HG_heap_t *heap = NULL; /* New global heap */ uint8_t *image; /* Pointer to image to decode */ + size_t max_idx = 0; /* Maximum heap object index seen */ + size_t nalloc; /* Number of objects allocated */ void *ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC @@ -201,109 +259,98 @@ H5HG__cache_heap_deserialize(const void *_image, size_t len, void *_udata, HDassert((len == H5HG_MINSIZE) /* first try */ || ((len == heap->size) && (len > H5HG_MINSIZE))); /* second try */ - if(len == heap->size) { /* proceed with the deserialize */ - size_t max_idx = 0; - size_t nalloc; - - /* Decode each object */ - image = heap->chunk + H5HG_SIZEOF_HDR(f); - nalloc = H5HG_NOBJS(f, heap->size); - - /* Calloc the obj array because the file format spec makes no guarantee - * about the order of the objects, and unused slots must be set to zero. - */ - if(NULL == (heap->obj = H5FL_SEQ_CALLOC(H5HG_obj_t, nalloc))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - heap->nalloc = nalloc; - - while(image < (heap->chunk + heap->size)) { - if((image + H5HG_SIZEOF_OBJHDR(f)) > (heap->chunk + heap->size)) { - /* - * The last bit of space is too tiny for an object header, so - * we assume that it's free space. - */ - HDassert(NULL == heap->obj[0].begin); - heap->obj[0].size = (size_t)(((const uint8_t *)heap->chunk + heap->size) - image); - heap->obj[0].begin = image; - image += heap->obj[0].size; + /* Decode each object */ + image = heap->chunk + H5HG_SIZEOF_HDR(f); + nalloc = H5HG_NOBJS(f, heap->size); + + /* Calloc the obj array because the file format spec makes no guarantee + * about the order of the objects, and unused slots must be set to zero. + */ + if(NULL == (heap->obj = H5FL_SEQ_CALLOC(H5HG_obj_t, nalloc))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + heap->nalloc = nalloc; + + while(image < (heap->chunk + heap->size)) { + if((image + H5HG_SIZEOF_OBJHDR(f)) > (heap->chunk + heap->size)) { + /* + * The last bit of space is too tiny for an object header, so + * we assume that it's free space. + */ + HDassert(NULL == heap->obj[0].begin); + heap->obj[0].size = (size_t)(((const uint8_t *)heap->chunk + heap->size) - image); + heap->obj[0].begin = image; + image += heap->obj[0].size; + } /* end if */ + else { + size_t need; + unsigned idx; + uint8_t *begin = image; + + UINT16DECODE(image, idx); + + /* Check if we need more room to store heap objects */ + if(idx >= heap->nalloc) { + size_t new_alloc; /* New allocation number */ + H5HG_obj_t *new_obj; /* New array of object descriptions */ + + /* Determine the new number of objects to index */ + new_alloc = MAX(heap->nalloc * 2, (idx + 1)); + HDassert(idx < new_alloc); + + /* Reallocate array of objects */ + if(NULL == (new_obj = H5FL_SEQ_REALLOC(H5HG_obj_t, heap->obj, new_alloc))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + + /* Clear newly allocated space */ + HDmemset(&new_obj[heap->nalloc], 0, (new_alloc - heap->nalloc) * sizeof(heap->obj[0])); + + /* Update heap information */ + heap->nalloc = new_alloc; + heap->obj = new_obj; + HDassert(heap->nalloc > heap->nused); + } /* end if */ + + UINT16DECODE(image, heap->obj[idx].nrefs); + image += 4; /*reserved*/ + H5F_DECODE_LENGTH(f, image, heap->obj[idx].size); + heap->obj[idx].begin = begin; + + /* + * The total storage size includes the size of the object + * header and is zero padded so the next object header is + * properly aligned. The entire obj array was calloc'ed, + * so no need to zero the space here. The last bit of space + * is the free space object whose size is never padded and + * already includes the object header. + */ + if(idx > 0) { + need = H5HG_SIZEOF_OBJHDR(f) + H5HG_ALIGN(heap->obj[idx].size); + if(idx > max_idx) + max_idx = idx; } /* end if */ - else { - size_t need; - unsigned idx; - uint8_t *begin = image; - - UINT16DECODE(image, idx); - - /* Check if we need more room to store heap objects */ - if(idx >= heap->nalloc) { - size_t new_alloc; /* New allocation number */ - H5HG_obj_t *new_obj; /* New array of object descriptions */ - - /* Determine the new number of objects to index */ - new_alloc = MAX(heap->nalloc * 2, (idx + 1)); - HDassert(idx < new_alloc); - - /* Reallocate array of objects */ - if(NULL == (new_obj = H5FL_SEQ_REALLOC(H5HG_obj_t, heap->obj, new_alloc))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - - /* Clear newly allocated space */ - HDmemset(&new_obj[heap->nalloc], 0, (new_alloc - heap->nalloc) * sizeof(heap->obj[0])); - - /* Update heap information */ - heap->nalloc = new_alloc; - heap->obj = new_obj; - HDassert(heap->nalloc > heap->nused); - } /* end if */ - - UINT16DECODE(image, heap->obj[idx].nrefs); - image += 4; /*reserved*/ - H5F_DECODE_LENGTH(f, image, heap->obj[idx].size); - heap->obj[idx].begin = begin; - - /* - * The total storage size includes the size of the object - * header and is zero padded so the next object header is - * properly aligned. The entire obj array was calloc'ed, - * so no need to zero the space here. The last bit of space - * is the free space object whose size is never padded and - * already includes the object header. - */ - if(idx > 0) { - need = H5HG_SIZEOF_OBJHDR(f) + H5HG_ALIGN(heap->obj[idx].size); - if(idx > max_idx) - max_idx = idx; - } /* end if */ - else - need = heap->obj[idx].size; - - image = begin + need; - } /* end else */ - } /* end while */ - - HDassert(image == heap->chunk + heap->size); - HDassert(H5HG_ISALIGNED(heap->obj[0].size)); - - /* Set the next index value to use */ - if(max_idx > 0) - heap->nused = max_idx + 1; - else - heap->nused = 1; - - HDassert(max_idx < heap->nused); - - /* Add the new heap to the CWFS list for the file */ - if(H5F_cwfs_add(f, heap) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, NULL, "unable to add global heap collection to file's CWFS") - } /* end if ( len == heap->size ) */ + else + need = heap->obj[idx].size; + + image = begin + need; + } /* end else */ + } /* end while */ + + /* Sanity checks */ + HDassert(image == heap->chunk + heap->size); + HDassert(H5HG_ISALIGNED(heap->obj[0].size)); + + /* Set the next index value to use */ + if(max_idx > 0) + heap->nused = max_idx + 1; else - /* if len is less than heap size, then the initial speculative - * read was too small. In this case we return without reporting - * failure. H5C_load_entry() will call H5HG__cache_heap_image_len() - * to get the actual read size, and then repeat the read with the - * correct size, and call this function a second time. - */ - HDassert(len < heap->size); + heap->nused = 1; + + /* Sanity check */ + HDassert(max_idx < heap->nused); + + /* Add the new heap to the CWFS list for the file */ + if(H5F_cwfs_add(f, heap) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, NULL, "unable to add global heap collection to file's CWFS") ret_value = heap; @@ -331,8 +378,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HG__cache_heap_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5HG__cache_heap_image_len(const void *_thing, size_t *image_len) { const H5HG_heap_t *heap = (const H5HG_heap_t *)_thing; @@ -350,10 +396,6 @@ H5HG__cache_heap_image_len(const void *_thing, size_t *image_len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HG__cache_heap_image_len() */ -/**************************************/ -/* no H5HG_cache_heap_pre_serialize() */ -/**************************************/ - /*------------------------------------------------------------------------- * Function: H5HG__cache_heap_serialize @@ -393,10 +435,6 @@ H5HG__cache_heap_serialize(const H5F_t *f, void *image, size_t len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HG__cache_heap_serialize() */ -/****************************************/ -/* no H5HG_cache_heap_notify() function */ -/****************************************/ - /*------------------------------------------------------------------------- * Function: H5HG__cache_heap_free_icr @@ -341,12 +341,10 @@ H5HL_protect(H5F_t *f, hid_t dxpl_id, haddr_t addr, unsigned flags)) HDassert((flags & (unsigned)(~H5AC__READ_ONLY_FLAG)) == 0); /* Construct the user data for protect callback */ - prfx_udata.made_attempt = FALSE; prfx_udata.sizeof_size = H5F_SIZEOF_SIZE(f); prfx_udata.sizeof_addr = H5F_SIZEOF_ADDR(f); prfx_udata.prfx_addr = addr; prfx_udata.sizeof_prfx = H5HL_SIZEOF_HDR(f); - prfx_udata.loaded = FALSE; /* Protect the local heap prefix */ if(NULL == (prfx = (H5HL_prfx_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_PRFX, addr, &prfx_udata, flags))) @@ -359,25 +357,14 @@ H5HL_protect(H5F_t *f, hid_t dxpl_id, haddr_t addr, unsigned flags)) /* (for re-entrant situation) */ if(heap->prots == 0) { /* Check if heap has separate data block */ - if(heap->single_cache_obj) { + if(heap->single_cache_obj) /* Set the flag for pinning the prefix when unprotecting it */ prfx_cache_flags |= H5AC__PIN_ENTRY_FLAG; - } /* end if */ else { - H5HL_cache_dblk_ud_t dblk_udata; /* User data for protecting local heap data block */ - - /* Construct the user data for protect callback */ - dblk_udata.heap = heap; - dblk_udata.loaded = FALSE; - /* Protect the local heap data block */ - if(NULL == (dblk = (H5HL_dblk_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_DBLK, heap->dblk_addr, &dblk_udata, flags))) + if(NULL == (dblk = (H5HL_dblk_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_DBLK, heap->dblk_addr, heap, flags))) H5E_THROW(H5E_CANTPROTECT, "unable to load heap data block"); - /* Pin the prefix, if the data block was loaded from file */ - if(dblk_udata.loaded) - prfx_cache_flags |= H5AC__PIN_ENTRY_FLAG; - /* Set the flag for pinning the data block when unprotecting it */ dblk_cache_flags |= H5AC__PIN_ENTRY_FLAG; } /* end if */ @@ -931,12 +918,10 @@ H5HL_delete(H5F_t *f, hid_t dxpl_id, haddr_t addr)) HDassert(H5F_addr_defined(addr)); /* Construct the user data for protect callback */ - prfx_udata.made_attempt = FALSE; prfx_udata.sizeof_size = H5F_SIZEOF_SIZE(f); prfx_udata.sizeof_addr = H5F_SIZEOF_ADDR(f); prfx_udata.prfx_addr = addr; prfx_udata.sizeof_prfx = H5HL_SIZEOF_HDR(f); - prfx_udata.loaded = FALSE; /* Protect the local heap prefix */ if(NULL == (prfx = (H5HL_prfx_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_PRFX, addr, &prfx_udata, H5AC__NO_FLAGS_SET))) @@ -946,24 +931,11 @@ H5HL_delete(H5F_t *f, hid_t dxpl_id, haddr_t addr)) heap = prfx->heap; /* Check if heap has separate data block */ - if(!heap->single_cache_obj) { - H5HL_cache_dblk_ud_t dblk_udata; /* User data for protecting local heap data block */ - - /* Construct the user data for protect callback */ - dblk_udata.heap = heap; - dblk_udata.loaded = FALSE; - + if(!heap->single_cache_obj) /* Protect the local heap data block */ - if(NULL == (dblk = (H5HL_dblk_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_DBLK, heap->dblk_addr, &dblk_udata, H5AC__NO_FLAGS_SET))) + if(NULL == (dblk = (H5HL_dblk_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_DBLK, heap->dblk_addr, heap, H5AC__NO_FLAGS_SET))) H5E_THROW(H5E_CANTPROTECT, "unable to load heap data block"); - /* Pin the prefix, if the data block was loaded from file */ - if(dblk_udata.loaded) { - if(FAIL == H5AC_pin_protected_entry(prfx)) - H5E_THROW(H5E_CANTPIN, "unable to pin local heap prefix"); - } /* end if */ - } /* end if */ - /* Set the flags for releasing the prefix and data block */ cache_flags |= H5AC__DIRTIED_FLAG | H5AC__DELETED_FLAG | H5AC__FREE_FILE_SPACE_FLAG; @@ -1005,12 +977,10 @@ H5HL_get_size(H5F_t *f, hid_t dxpl_id, haddr_t addr, size_t *size)) HDassert(size); /* Construct the user data for protect callback */ - prfx_udata.made_attempt = FALSE; prfx_udata.sizeof_size = H5F_SIZEOF_SIZE(f); prfx_udata.sizeof_addr = H5F_SIZEOF_ADDR(f); prfx_udata.prfx_addr = addr; prfx_udata.sizeof_prfx = H5HL_SIZEOF_HDR(f); - prfx_udata.loaded = FALSE; /* Protect the local heap prefix */ if(NULL == (prfx = (H5HL_prfx_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_PRFX, addr, &prfx_udata, H5AC__READ_ONLY_FLAG))) @@ -1056,12 +1026,10 @@ H5HL_heapsize(H5F_t *f, hid_t dxpl_id, haddr_t addr, hsize_t *heap_size)) HDassert(heap_size); /* Construct the user data for protect callback */ - prfx_udata.made_attempt = FALSE; prfx_udata.sizeof_size = H5F_SIZEOF_SIZE(f); prfx_udata.sizeof_addr = H5F_SIZEOF_ADDR(f); prfx_udata.prfx_addr = addr; prfx_udata.sizeof_prfx = H5HL_SIZEOF_HDR(f); - prfx_udata.loaded = FALSE; /* Protect the local heap prefix */ if(NULL == (prfx = (H5HL_prfx_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_PRFX, addr, &prfx_udata, H5AC__READ_ONLY_FLAG))) diff --git a/src/H5HLcache.c b/src/H5HLcache.c index 95ab65d..c53292a 100644 --- a/src/H5HLcache.c +++ b/src/H5HLcache.c @@ -71,26 +71,24 @@ /* Metadata cache callbacks */ /* Local heap prefix */ -static herr_t H5HL__cache_prefix_get_load_size(const void *udata, size_t *image_len); +static herr_t H5HL__cache_prefix_get_initial_load_size(void *udata, size_t *image_len); +static herr_t H5HL__cache_prefix_get_final_load_size(const void *_image, + size_t image_len, void *udata, size_t *actual_len); static void *H5HL__cache_prefix_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5HL__cache_prefix_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5HL__cache_prefix_image_len(const void *thing, size_t *image_len); static herr_t H5HL__cache_prefix_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5HL__cache_prefix_free_icr(void *thing); /* Local heap data block */ -static herr_t H5HL__cache_datablock_get_load_size(const void *udata, - size_t *image_len); +static herr_t H5HL__cache_datablock_get_initial_load_size(void *udata, size_t *image_len); static void *H5HL__cache_datablock_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5HL__cache_datablock_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5HL__cache_datablock_image_len(const void *thing, size_t *image_len); static herr_t H5HL__cache_datablock_serialize(const H5F_t *f, void *image, size_t len, void *thing); +static herr_t H5HL__cache_datablock_notify(H5C_notify_action_t action, void *_thing); static herr_t H5HL__cache_datablock_free_icr(void *thing); /* Free list de/serialization */ @@ -107,14 +105,15 @@ const H5AC_class_t H5AC_LHEAP_PRFX[1] = {{ "local heap prefix", /* Metadata client name (for debugging) */ H5FD_MEM_LHEAP, /* File space memory type for client */ H5AC__CLASS_SPECULATIVE_LOAD_FLAG, /* Client class behavior flags */ - H5HL__cache_prefix_get_load_size, /* 'get_load_size' callback */ + H5HL__cache_prefix_get_initial_load_size, /* 'get_initial_load_size' callback */ + H5HL__cache_prefix_get_final_load_size, /* 'get_final_load_size' callback */ + NULL, /* 'verify_chksum' callback */ H5HL__cache_prefix_deserialize, /* 'deserialize' callback */ H5HL__cache_prefix_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5HL__cache_prefix_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5HL__cache_prefix_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -123,14 +122,15 @@ const H5AC_class_t H5AC_LHEAP_DBLK[1] = {{ "local heap datablock", /* Metadata client name (for debugging) */ H5FD_MEM_LHEAP, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5HL__cache_datablock_get_load_size,/* 'get_load_size' callback */ + H5HL__cache_datablock_get_initial_load_size,/* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + NULL, /* 'verify_chksum' callback */ H5HL__cache_datablock_deserialize, /* 'deserialize' callback */ H5HL__cache_datablock_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5HL__cache_datablock_serialize, /* 'serialize' callback */ - NULL, /* 'notify' callback */ + H5HL__cache_datablock_notify, /* 'notify' callback */ H5HL__cache_datablock_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -260,18 +260,11 @@ H5HL__fl_serialize(const H5HL_t *heap) /*------------------------------------------------------------------------- - * Function: H5HL__cache_prefix_get_load_size() + * Function: H5HL__cache_prefix_get_initial_load_size() * - * Purpose: Return the size of the buffer the metadata cache should + * Purpose: Return the initial size of the buffer the metadata cache should * load from file and pass to the deserialize routine. * - * The version 2 metadata cache callbacks included a test to - * ensure that the read did not pass the end of file, but this - * functionality has been moved to H5C_load_entry(). Thus - * all this function does is set *image_len equal to - * H5HL_SPEC_READ_SIZE, leaving it to the metadata cache to - * reduce the size of the read if appropriate. - * * Return: Success: SUCCEED * Failure: FAIL * @@ -281,16 +274,91 @@ H5HL__fl_serialize(const H5HL_t *heap) *------------------------------------------------------------------------- */ static herr_t -H5HL__cache_prefix_get_load_size(const void H5_ATTR_UNUSED *_udata, size_t *image_len) +H5HL__cache_prefix_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len) { FUNC_ENTER_STATIC_NOERR + /* Sanity check */ HDassert(image_len); + /* Set the image length size */ *image_len = H5HL_SPEC_READ_SIZE; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5HL__cache_prefix_get_load_size() */ +} /* end H5HL__cache_prefix_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5HL__cache_prefix_get_final_load_size() + * + * Purpose: Return the final size of the buffer the metadata cache should + * load from file and pass to the deserialize routine. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * November 18, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5HL__cache_prefix_get_final_load_size(const void *_image, size_t image_len, + void *_udata, size_t *actual_len) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5HL_cache_prfx_ud_t *udata = (H5HL_cache_prfx_ud_t *)_udata; /* User data for callback */ + H5HL_t heap; /* Local heap */ + htri_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(image); + HDassert(udata); + HDassert(actual_len); + HDassert(*actual_len == image_len); + + /* Check magic number */ + if(HDmemcmp(image, H5HL_MAGIC, (size_t)H5_SIZEOF_MAGIC)) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad local heap signature") + image += H5_SIZEOF_MAGIC; + + /* Version */ + if(H5HL_VERSION != *image++) + HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong version number in local heap") + + /* Reserved */ + image += 3; + + /* Store the prefix's address & length */ + heap.prfx_addr = udata->prfx_addr; /* NEED */ + heap.prfx_size = udata->sizeof_prfx; /* NEED */ + + /* Heap data size */ + H5F_DECODE_LENGTH_LEN(image, heap.dblk_size, udata->sizeof_size); /* NEED */ + + /* Free list head */ + H5F_DECODE_LENGTH_LEN(image, heap.free_block, udata->sizeof_size); + if(heap.free_block != H5HL_FREE_NULL && heap.free_block >= heap.dblk_size) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap free list"); + + /* Heap data address */ + H5F_addr_decode_len(udata->sizeof_addr, &image, &(heap.dblk_addr)); /* NEED */ + + /* Set the final size for the cache image */ + *actual_len = heap.prfx_size; + + /* Check if heap block exists */ + if(heap.dblk_size) + /* Check if heap data block is contiguous with header */ + if(H5F_addr_eq((heap.prfx_addr + heap.prfx_size), heap.dblk_addr)) + /* Note that the heap should be a single object in the cache */ + *actual_len += heap.dblk_size; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HL__cache_prefix_get_final_load_size() */ /*------------------------------------------------------------------------- @@ -359,7 +427,6 @@ H5HL__cache_prefix_deserialize(const void *_image, size_t len, void *_udata, /* Free list head */ H5F_DECODE_LENGTH_LEN(image, heap->free_block, udata->sizeof_size); - if((heap->free_block != H5HL_FREE_NULL) && (heap->free_block >= heap->dblk_size)) HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "bad heap free list") @@ -373,53 +440,29 @@ H5HL__cache_prefix_deserialize(const void *_image, size_t len, void *_udata, /* Note that the heap should be a single object in the cache */ heap->single_cache_obj = TRUE; - /* Check if the current buffer from the speculative read - * already has the heap data - */ - if(len >= (heap->prfx_size + heap->dblk_size)) { - /* Allocate space for the heap data image */ - if(NULL == (heap->dblk_image = H5FL_BLK_MALLOC(lheap_chunk, heap->dblk_size))) - HGOTO_ERROR(H5E_HEAP, H5E_CANTALLOC, NULL, "memory allocation failed") - - /* Set image to the start of the data block. This is necessary - * because there may be a gap between the used portion of the - * prefix and the data block due to alignment constraints. */ - image = ((const uint8_t *)_image) + heap->prfx_size; - - /* Copy the heap data from the speculative read buffer */ - HDmemcpy(heap->dblk_image, image, heap->dblk_size); - - /* Build free list */ - if(H5HL__fl_deserialize(heap) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, NULL, "can't initialize free list") - } /* end if */ - else { - /* the supplied buffer is too small -- We have already made note - * of the correct size, so simply return success. H5C_load_entry() - * will notice the size discrepency, and re-try the load. - */ - - /* Make certain that this is the first try ... */ - HDassert(!udata->made_attempt); - - /* ... and mark the udata so that we know that we have used up - * our first try. - */ - udata->made_attempt = TRUE; - } /* end else */ + /* Allocate space for the heap data image */ + if(NULL == (heap->dblk_image = H5FL_BLK_MALLOC(lheap_chunk, heap->dblk_size))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTALLOC, NULL, "memory allocation failed") + + /* Set image to the start of the data block. This is necessary + * because there may be a gap between the used portion of the + * prefix and the data block due to alignment constraints. */ + image = ((const uint8_t *)_image) + heap->prfx_size; + + /* Copy the heap data from the speculative read buffer */ + HDmemcpy(heap->dblk_image, image, heap->dblk_size); + + /* Build free list */ + if(H5HL__fl_deserialize(heap) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, NULL, "can't initialize free list") } /* end if */ - else { + else /* Note that the heap should _NOT_ be a single * object in the cache */ heap->single_cache_obj = FALSE; - - } /* end else */ } /* end if */ - /* Set flag to indicate prefix from loaded from file */ - udata->loaded = TRUE; - /* Set return value */ ret_value = prfx; @@ -455,8 +498,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HL__cache_prefix_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5HL__cache_prefix_image_len(const void *_thing, size_t *image_len) { const H5HL_prfx_t *prfx = (const H5HL_prfx_t *)_thing; /* Pointer to local heap prefix to query */ @@ -480,10 +522,6 @@ H5HL__cache_prefix_image_len(const void *_thing, size_t *image_len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HL__cache_prefix_image_len() */ -/****************************************/ -/* no H5HL_cache_prefix_pre_serialize() */ -/****************************************/ - /*------------------------------------------------------------------------- * Function: H5HL__cache_prefix_serialize @@ -579,10 +617,6 @@ H5HL__cache_prefix_serialize(const H5F_t *f, void *_image, size_t len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HL__cache_prefix_serialize() */ -/******************************************/ -/* no H5HL_cache_prefix_notify() function */ -/******************************************/ - /*------------------------------------------------------------------------- * Function: H5HL__cache_prefix_free_icr @@ -630,7 +664,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5HL__cache_datablock_get_load_size() + * Function: H5HL__cache_datablock_get_initial_load_size() * * Purpose: Tell the metadata cache how large a buffer to read from * file when loading a datablock. In this case, we simply lookup @@ -645,22 +679,22 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HL__cache_datablock_get_load_size(const void *_udata, size_t *image_len) +H5HL__cache_datablock_get_initial_load_size(void *_udata, size_t *image_len) { - const H5HL_cache_dblk_ud_t *udata = (const H5HL_cache_dblk_ud_t *)_udata; /* User data for callback */ + H5HL_t *heap = (H5HL_t *)_udata; /* User data for callback */ FUNC_ENTER_STATIC_NOERR /* Check arguments */ - HDassert(udata); - HDassert(udata->heap); - HDassert(udata->heap->dblk_size > 0); + HDassert(heap); + HDassert(heap->dblk_size > 0); HDassert(image_len); - *image_len = udata->heap->dblk_size; + /* Set the image length size */ + *image_len = heap->dblk_size; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5HL__cache_datablock_get_load_size() */ +} /* end H5HL__cache_datablock_get_initial_load_size() */ /*------------------------------------------------------------------------- @@ -682,8 +716,8 @@ static void * H5HL__cache_datablock_deserialize(const void *image, size_t len, void *_udata, hbool_t H5_ATTR_UNUSED *dirty) { - H5HL_dblk_t *dblk = NULL; /* Local heap data block deserialized */ - H5HL_cache_dblk_ud_t *udata = (H5HL_cache_dblk_ud_t *)_udata; /* User data for callback */ + H5HL_dblk_t *dblk = NULL; /* Local heap data block deserialized */ + H5HL_t *heap = (H5HL_t *)_udata; /* User data for callback */ void *ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC @@ -691,34 +725,30 @@ H5HL__cache_datablock_deserialize(const void *image, size_t len, void *_udata, /* Check arguments */ HDassert(image); HDassert(len > 0); - HDassert(udata); - HDassert(udata->heap); - HDassert(udata->heap->dblk_size == len); - HDassert(!udata->heap->single_cache_obj); - HDassert(NULL == udata->heap->dblk); + HDassert(heap); + HDassert(heap->dblk_size == len); + HDassert(!heap->single_cache_obj); + HDassert(NULL == heap->dblk); HDassert(dirty); /* Allocate space in memory for the heap data block */ - if(NULL == (dblk = H5HL__dblk_new(udata->heap))) + if(NULL == (dblk = H5HL__dblk_new(heap))) HGOTO_ERROR(H5E_HEAP, H5E_CANTALLOC, NULL, "memory allocation failed"); /* Check for heap still retaining image */ - if(NULL == udata->heap->dblk_image) { + if(NULL == heap->dblk_image) { /* Allocate space for the heap data image */ - if(NULL == (udata->heap->dblk_image = H5FL_BLK_MALLOC(lheap_chunk, udata->heap->dblk_size))) + if(NULL == (heap->dblk_image = H5FL_BLK_MALLOC(lheap_chunk, heap->dblk_size))) HGOTO_ERROR(H5E_HEAP, H5E_CANTALLOC, NULL, "can't allocate data block image buffer"); /* copy the datablock from the read buffer */ - HDmemcpy(udata->heap->dblk_image, image, len); + HDmemcpy(heap->dblk_image, image, len); /* Build free list */ - if(FAIL == H5HL__fl_deserialize(udata->heap)) + if(FAIL == H5HL__fl_deserialize(heap)) HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, NULL, "can't initialize free list"); } /* end if */ - /* Set flag to indicate data block from loaded from file */ - udata->loaded = TRUE; - /* Set return value */ ret_value = dblk; @@ -746,8 +776,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HL__cache_datablock_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5HL__cache_datablock_image_len(const void *_thing, size_t *image_len) { const H5HL_dblk_t *dblk = (const H5HL_dblk_t *)_thing; /* Pointer to the local heap data block */ @@ -766,10 +795,6 @@ H5HL__cache_datablock_image_len(const void *_thing, size_t *image_len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HL__cache_datablock_image_len() */ -/*******************************************/ -/* no H5HL_cache_datablock_pre_serialize() */ -/*******************************************/ - /*------------------------------------------------------------------------- * Function: H5HL__cache_datablock_serialize @@ -817,9 +842,73 @@ H5HL__cache_datablock_serialize(const H5F_t *f, void *image, size_t len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HL__cache_datablock_serialize() */ -/*********************************************/ -/* no H5HL_cache_datablock_notify() function */ -/*********************************************/ + +/*------------------------------------------------------------------------- + * Function: H5HL__cache_datablock_notify + * + * Purpose: This function is used to create and destroy pinned + * relationships between datablocks and their prefix parent. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * November 19, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5HL__cache_datablock_notify(H5C_notify_action_t action, void *_thing) +{ + H5HL_dblk_t *dblk = (H5HL_dblk_t *)_thing; /* Pointer to the local heap data block */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(dblk); + + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_AFTER_LOAD: + /* Sanity checks */ + HDassert(dblk->heap); + HDassert(dblk->heap->prfx); + + /* Pin the heap's prefix */ + if(FAIL == H5AC_pin_protected_entry(dblk->heap->prfx)) + HGOTO_ERROR(H5E_HEAP, H5E_CANTPIN, FAIL, "unable to pin local heap prefix") + break; + + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + /* Sanity checks */ + HDassert(dblk->heap); + HDassert(dblk->heap->prfx); + + /* Unpin the local heap prefix */ + if(FAIL == H5AC_unpin_entry(dblk->heap->prfx)) + HGOTO_ERROR(H5E_HEAP, H5E_CANTUNPIN, FAIL, "unable to unpin local heap prefix") + break; + + default: + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unknown action from metadata cache") + break; + } /* end switch */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HL__cache_datablock_notify() */ /*------------------------------------------------------------------------- @@ -859,3 +948,4 @@ H5HL__cache_datablock_free_icr(void *_thing) done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5HL__cache_datablock_free_icr() */ + diff --git a/src/H5HLdblk.c b/src/H5HLdblk.c index 69e0334..f90562b 100644 --- a/src/H5HLdblk.c +++ b/src/H5HLdblk.c @@ -119,10 +119,9 @@ H5HL__dblk_new(H5HL_t *heap)) CATCH /* Ensure that the data block memory is deallocated on errors */ - if(!ret_value && dblk != NULL) { + if(!ret_value && dblk != NULL) /* H5FL_FREE always returns NULL so we can't check for errors */ dblk = H5FL_FREE(H5HL_dblk_t, dblk); - } END_FUNC(PKG) /* end H5HL__dblk_new() */ @@ -151,10 +150,6 @@ H5HL__dblk_dest(H5HL_dblk_t *dblk)) /* Unlink data block from heap */ dblk->heap->dblk = NULL; - /* Unpin the local heap prefix */ - if(FAIL == H5AC_unpin_entry(dblk->heap->prfx)) - H5E_THROW(H5E_CANTUNPIN, "can't unpin local heap prefix") - /* Decrement ref. count on heap data structure */ if(FAIL == H5HL__dec_rc(dblk->heap)) H5E_THROW(H5E_CANTDEC, "can't decrement heap ref. count") @@ -223,7 +218,6 @@ H5HL__dblk_realloc(H5F_t *f, hid_t dxpl_id, H5HL_t *heap, size_t new_heap_size)) /* Resize the heap prefix in the cache */ if(FAIL == H5AC_resize_entry(heap->prfx, (size_t)(heap->prfx_size + new_heap_size))) H5E_THROW(H5E_CANTRESIZE, "unable to resize heap in cache"); - } /* end if */ else { /* Sanity check */ @@ -233,7 +227,6 @@ H5HL__dblk_realloc(H5F_t *f, hid_t dxpl_id, H5HL_t *heap, size_t new_heap_size)) /* Resize the heap data block in the cache */ if(H5AC_resize_entry(heap->dblk, (size_t)new_heap_size) < 0) H5E_THROW(H5E_CANTRESIZE, "unable to resize heap (data block) in cache"); - } /* end else */ } /* end if */ else { @@ -280,3 +273,4 @@ CATCH } /* end if */ END_FUNC(PKG) /* end H5HL__dblk_realloc() */ + diff --git a/src/H5HLpkg.h b/src/H5HLpkg.h index fb8667b..7075b2a 100644 --- a/src/H5HLpkg.h +++ b/src/H5HLpkg.h @@ -131,28 +131,12 @@ struct H5HL_prfx_t { /* Callback information for loading local heap prefix from disk */ typedef struct H5HL_cache_prfx_ud_t { - /* Downwards */ - hbool_t made_attempt; /* Whether the deserialize routine */ - /* was already attempted */ size_t sizeof_size; /* Size of file sizes */ size_t sizeof_addr; /* Size of file addresses */ haddr_t prfx_addr; /* Address of prefix */ size_t sizeof_prfx; /* Size of heap prefix */ - - /* Upwards */ - hbool_t loaded; /* Whether prefix was loaded */ - /* from file */ } H5HL_cache_prfx_ud_t; -/* Callback information for loading local heap data block from disk */ -typedef struct H5HL_cache_dblk_ud_t { - /* Downwards */ - H5HL_t *heap; /* Local heap */ - - /* Upwards */ - hbool_t loaded; /* Whether data block was loaded from file */ -} H5HL_cache_dblk_ud_t; - /******************************/ /* Package Private Prototypes */ diff --git a/src/H5HLprfx.c b/src/H5HLprfx.c index 66c4dad..ed1c4db 100644 --- a/src/H5HLprfx.c +++ b/src/H5HLprfx.c @@ -118,10 +118,9 @@ H5HL__prfx_new(H5HL_t *heap)) CATCH /* Ensure that the prefix memory is deallocated on errors */ - if(!ret_value && prfx != NULL) { + if(!ret_value && prfx != NULL) /* H5FL_FREE always returns NULL so we can't check for errors */ prfx = H5FL_FREE(H5HL_prfx_t, prfx); - } END_FUNC(PKG) /* end H5HL__prfx_new() */ @@ -164,3 +163,4 @@ CATCH prfx = H5FL_FREE(H5HL_prfx_t, prfx); END_FUNC(PKG) /* end H5HL__prfx_dest() */ + diff --git a/src/H5Lexternal.c b/src/H5Lexternal.c index 12cce73..7f59e50 100644 --- a/src/H5Lexternal.c +++ b/src/H5Lexternal.c @@ -323,7 +323,7 @@ H5L_extern_traverse(const char H5_ATTR_UNUSED *link_name, hid_t cur_group, */ /* Simplify intent flags for open calls */ - intent = ((intent & H5F_ACC_RDWR) ? H5F_ACC_RDWR : H5F_ACC_RDONLY); + intent &= (H5F_ACC_RDWR | H5F_ACC_SWMR_WRITE | H5F_ACC_SWMR_READ); /* Copy the file name to use */ if(NULL == (temp_file_name = H5MM_strdup(file_name))) @@ -85,7 +85,7 @@ typedef struct { /* Allocator routines */ static herr_t H5MF_alloc_create(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type); -static herr_t H5MF_alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type); +static herr_t H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type); static herr_t H5MF__close_delete(H5F_t *f, hid_t dxpl_id); @@ -245,7 +245,7 @@ H5MF_alloc_open(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) H5AC_ring_t orig_ring = H5AC_RING_INV; /* Original ring value */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_NOAPI_NOINIT_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) /* * Check arguments. @@ -274,7 +274,7 @@ done: if(H5AC_reset_ring(dxpl, orig_ring) < 0) HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") - FUNC_LEAVE_NOAPI(ret_value) + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) } /* end H5MF_alloc_open() */ @@ -301,7 +301,7 @@ H5MF_alloc_create(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) herr_t ret_value = SUCCEED; /* Return value */ H5FS_create_t fs_create; /* Free space creation parameters */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_NOAPI_NOINIT_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) /* * Check arguments. @@ -329,7 +329,7 @@ H5MF_alloc_create(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) f->shared->fs_state[type] = H5F_FS_STATE_OPEN; done: - FUNC_LEAVE_NOAPI(ret_value) + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) } /* end H5MF_alloc_create() */ @@ -379,7 +379,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5MF_alloc_close + * Function: H5MF__alloc_close * * Purpose: Close an existing free space manager of TYPE for file * @@ -391,11 +391,11 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5MF_alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) +H5MF__alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) /* * Check arguments. @@ -413,8 +413,8 @@ H5MF_alloc_close(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type) f->shared->fs_state[type] = H5F_FS_STATE_CLOSED; done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5MF_alloc_close() */ + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5MF__alloc_close() */ /*------------------------------------------------------------------------- @@ -442,7 +442,7 @@ H5MF_alloc(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, hsize_t size) H5FD_mem_t fs_type; /* Free space type (mapped from allocation type) */ haddr_t ret_value = HADDR_UNDEF; /* Return value */ - FUNC_ENTER_NOAPI(HADDR_UNDEF) + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, HADDR_UNDEF) #ifdef H5MF_ALLOC_DEBUG HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_type, size); #endif /* H5MF_ALLOC_DEBUG */ @@ -543,7 +543,7 @@ HDfprintf(stderr, "%s: Leaving: ret_value = %a, size = %Hu\n", FUNC, ret_value, H5MF_sects_dump(f, dxpl_id, stderr); #endif /* H5MF_ALLOC_DEBUG_DUMP */ - FUNC_LEAVE_NOAPI(ret_value) + FUNC_LEAVE_NOAPI_TAG(ret_value, HADDR_UNDEF) } /* end H5MF_alloc() */ @@ -633,7 +633,7 @@ H5MF_xfree(const H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr, H5FD_mem_t fs_type; /* Free space type (mapped from allocation type) */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG HDfprintf(stderr, "%s: Entering - alloc_type = %u, addr = %a, size = %Hu\n", FUNC, (unsigned)alloc_type, addr, size); #endif /* H5MF_ALLOC_DEBUG */ @@ -774,7 +774,7 @@ HDfprintf(stderr, "%s: Leaving, ret_value = %d\n", FUNC, ret_value); #ifdef H5MF_ALLOC_DEBUG_DUMP H5MF_sects_dump(f, dxpl_id, stderr); #endif /* H5MF_ALLOC_DEBUG_DUMP */ - FUNC_LEAVE_NOAPI(ret_value) + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) } /* end H5MF_xfree() */ @@ -802,7 +802,7 @@ H5MF_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type, haddr_t addr, H5FD_mem_t map_type; /* Mapped type */ htri_t ret_value = FAIL; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_requested = %Hu\n", FUNC, (unsigned)alloc_type, addr, size, extra_requested); #endif /* H5MF_ALLOC_DEBUG */ @@ -861,7 +861,7 @@ HDfprintf(stderr, "%s: Leaving: ret_value = %t\n", FUNC, ret_value); H5MF_sects_dump(f, dxpl_id, stderr); #endif /* H5MF_ALLOC_DEBUG_DUMP */ - FUNC_LEAVE_NOAPI(ret_value) + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) } /* end H5MF_try_extend() */ @@ -900,7 +900,7 @@ H5MF_get_freespace(H5F_t *f, hid_t dxpl_id, hsize_t *tot_space, hsize_t *meta_si hbool_t eoa_shrank; /* Whether an EOA shrink occurs */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) /* check args */ HDassert(f); @@ -991,7 +991,7 @@ H5MF_get_freespace(H5F_t *f, hid_t dxpl_id, hsize_t *tot_space, hsize_t *meta_si /* Close the free-space managers if they were opened earlier in this routine */ for(type = H5FD_MEM_DEFAULT; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) { if(fs_started[type]) - if(H5MF_alloc_close(f, dxpl_id, type) < 0) + if(H5MF__alloc_close(f, dxpl_id, type) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't close file free space") } /* end for */ @@ -1007,7 +1007,7 @@ done: if(H5AC_reset_ring(dxpl, orig_ring) < 0) HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") - FUNC_LEAVE_NOAPI(ret_value) + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) } /* end H5MF_get_freespace() */ @@ -1108,7 +1108,7 @@ H5MF_close_shrink_eoa(H5F_t *f, hid_t dxpl_id) H5MF_sect_ud_t udata; /* User data for callback */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_NOAPI_NOINIT_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) /* check args */ HDassert(f); @@ -1143,7 +1143,7 @@ H5MF_close_shrink_eoa(H5F_t *f, hid_t dxpl_id) } while(eoa_shrank); done: - FUNC_LEAVE_NOAPI(ret_value) + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) } /* end H5MF_close_shrink_eoa() */ @@ -1166,7 +1166,7 @@ H5MF__close_delete(H5F_t *f, hid_t dxpl_id) H5FD_mem_t type; /* Memory type for iteration */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC + FUNC_ENTER_STATIC_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG HDfprintf(stderr, "%s: Entering\n", FUNC); #endif /* H5MF_ALLOC_DEBUG */ @@ -1229,7 +1229,7 @@ done: #ifdef H5MF_ALLOC_DEBUG HDfprintf(stderr, "%s: Leaving\n", FUNC); #endif /* H5MF_ALLOC_DEBUG */ - FUNC_LEAVE_NOAPI(ret_value) + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) } /* H5MF__close_delete() */ @@ -1308,7 +1308,7 @@ H5MF_close(H5F_t *f, hid_t dxpl_id) H5FD_mem_t type; /* Memory type for iteration */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG HDfprintf(stderr, "%s: Entering\n", FUNC); #endif /* H5MF_ALLOC_DEBUG */ @@ -1446,7 +1446,7 @@ done: #ifdef H5MF_ALLOC_DEBUG HDfprintf(stderr, "%s: Leaving\n", FUNC); #endif /* H5MF_ALLOC_DEBUG */ - FUNC_LEAVE_NOAPI(ret_value) + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) } /* end H5MF_close() */ @@ -1506,7 +1506,7 @@ H5MF_get_free_sections(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, size_t nsects, H5FD_mem_t ty; /* Memory type for iteration */ ssize_t ret_value = -1; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) /* check args */ HDassert(f); @@ -1567,7 +1567,7 @@ H5MF_get_free_sections(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, size_t nsects, /* Close the free space manager of this type, if we started it here */ if(fs_started) - if(H5MF_alloc_close(f, dxpl_id, ty) < 0) + if(H5MF__alloc_close(f, dxpl_id, ty) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCLOSEOBJ, FAIL, "can't close file free space") } /* end for */ @@ -1579,6 +1579,6 @@ done: if(H5AC_reset_ring(dxpl, orig_ring) < 0) HDONE_ERROR(H5E_RESOURCE, H5E_CANTSET, FAIL, "unable to set property value") - FUNC_LEAVE_NOAPI(ret_value) + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) } /* H5MF_get_free_sections() */ diff --git a/src/H5MFdbg.c b/src/H5MFdbg.c index 59da9e8..6d5d994 100644 --- a/src/H5MFdbg.c +++ b/src/H5MFdbg.c @@ -158,7 +158,7 @@ H5MF_sects_debug(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr, FILE *stream, int ind herr_t ret_value = SUCCEED; /* Return value */ H5FD_mem_t type; /* Memory type for iteration */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) /* * Check arguments. @@ -195,7 +195,7 @@ H5MF_sects_debug(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr, FILE *stream, int ind } done: - FUNC_LEAVE_NOAPI(ret_value) + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) } /* H5MF_sects_debug() */ #ifdef H5MF_ALLOC_DEBUG_DUMP @@ -226,7 +226,7 @@ H5MF_sects_dump(H5F_t *f, hid_t dxpl_id, FILE *stream) int fwidth = 50; /* Field width */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG HDfprintf(stderr, "%s: Dumping file free space sections\n", FUNC); #endif /* H5MF_ALLOC_DEBUG */ @@ -300,7 +300,7 @@ HDfprintf(stderr, "%s: sda_addr = %a, sda_size = %Hu, end of sda = %a\n", FUNC, done: HDfprintf(stderr, "%s: Done dumping file free space sections\n", FUNC); - FUNC_LEAVE_NOAPI(ret_value) + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) } /* end H5MF_sects_dump() */ #endif /* H5MF_ALLOC_DEBUG_DUMP */ @@ -1051,7 +1051,7 @@ done: * the same effect as calling H5Gclose, H5Dclose, or H5Tclose. * * Return: Success: Non-negative - * Failure: Negative + * Failure: Negative * * Programmer: James Laird * July 14 2006 @@ -1281,6 +1281,7 @@ H5O_create(H5F_t *f, hid_t dxpl_id, size_t size_hint, size_t initial_rc, oh->version = H5O_VERSION_1; oh->sizeof_size = H5F_SIZEOF_SIZE(f); oh->sizeof_addr = H5F_SIZEOF_ADDR(f); + oh->swmr_write = !!(H5F_INTENT(f) & H5F_ACC_SWMR_WRITE); #ifdef H5O_ENABLE_BAD_MESG_COUNT /* Check whether the "bad message count" property is set */ if(H5P_exist_plist(oc_plist, H5O_BAD_MESG_COUNT_NAME) > 0) { @@ -1290,6 +1291,15 @@ H5O_create(H5F_t *f, hid_t dxpl_id, size_t size_hint, size_t initial_rc, } /* end if */ #endif /* H5O_ENABLE_BAD_MESG_COUNT */ + /* Create object header proxy if doing SWMR writes */ + if(oh->swmr_write) { + /* Create virtual entry, for use as proxy */ + if(NULL == (oh->proxy = H5AC_proxy_entry_create())) + HGOTO_ERROR(H5E_OHDR, H5E_CANTCREATE, FAIL, "can't create object header proxy") + } /* end if */ + else + oh->proxy = NULL; + /* Set initial status flags */ oh->flags = oh_flags; @@ -1356,6 +1366,7 @@ H5O_create(H5F_t *f, hid_t dxpl_id, size_t size_hint, size_t initial_rc, /* (including space for serializing the object header prefix */ if(NULL == (oh->chunk[0].image = H5FL_BLK_CALLOC(chunk_image, oh_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + oh->chunk[0].chunk_proxy = NULL; /* Put magic # for object header in first chunk */ if(oh->version > H5O_VERSION_1) @@ -1388,6 +1399,8 @@ H5O_create(H5F_t *f, hid_t dxpl_id, size_t size_hint, size_t initial_rc, /* Cache object header */ if(H5AC_insert_entry(f, dxpl_id, H5AC_OHDR, oh_addr, oh, insert_flags) < 0) HGOTO_ERROR_TAG(H5E_OHDR, H5E_CANTINSERT, FAIL, "unable to cache object header") + + /* Reset object header pointer, now that it's been inserted into the cache */ oh = NULL; /* Reset metadata tag in dxpl_id */ @@ -1403,7 +1416,7 @@ H5O_create(H5F_t *f, hid_t dxpl_id, size_t size_hint, size_t initial_rc, done: if(ret_value < 0 && oh) - if(H5O_free(oh) < 0) + if(H5O__free(oh) < 0) HDONE_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to destroy object header data") FUNC_LEAVE_NOAPI(ret_value) @@ -1556,7 +1569,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5O_close(H5O_loc_t *loc) +H5O_close(H5O_loc_t *loc, hbool_t *file_closed /*out*/) { herr_t ret_value = SUCCEED; /* Return value */ @@ -1567,6 +1580,15 @@ H5O_close(H5O_loc_t *loc) HDassert(loc->file); HDassert(H5F_NOPEN_OBJS(loc->file) > 0); + /* Set the file_closed flag to the default value. + * This flag lets downstream code know if the file struct is + * still accessible and/or likely to contain useful data. + * It's needed by the evict-on-close code. Clients can ignore + * this value by passing in NULL. + */ + if(file_closed) + *file_closed = FALSE; + /* Decrement open-lock counters */ H5F_DECR_NOPEN_OBJS(loc->file); @@ -1587,7 +1609,7 @@ H5O_close(H5O_loc_t *loc) */ if(H5F_NOPEN_OBJS(loc->file) == H5F_NMOUNTS(loc->file)) /* Attempt to close down the file hierarchy */ - if(H5F_try_close(loc->file) < 0) + if(H5F_try_close(loc->file, file_closed) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTCLOSEFILE, FAIL, "problem attempting file close") /* Release location information */ @@ -1780,7 +1802,8 @@ done: *------------------------------------------------------------------------- */ H5O_t * -H5O_protect(const H5O_loc_t *loc, hid_t dxpl_id, unsigned prot_flags) +H5O_protect(const H5O_loc_t *loc, hid_t dxpl_id, unsigned prot_flags, + hbool_t pin_all_chunks) { H5O_t *oh = NULL; /* Object header protected */ H5O_cache_ud_t udata; /* User data for protecting object header */ @@ -1809,11 +1832,12 @@ H5O_protect(const H5O_loc_t *loc, hid_t dxpl_id, unsigned prot_flags) /* Construct the user data for protect callback */ udata.made_attempt = FALSE; udata.v1_pfx_nmesgs = 0; + udata.chunk0_size = 0; + udata.oh = NULL; udata.common.f = loc->file; udata.common.dxpl_id = dxpl_id; udata.common.file_intent = file_intent; udata.common.merged_null_msgs = 0; - udata.common.mesgs_modified = FALSE; HDmemset(&cont_msg_info, 0, sizeof(cont_msg_info)); udata.common.cont_msg_info = &cont_msg_info; udata.common.addr = loc->addr; @@ -1841,10 +1865,12 @@ H5O_protect(const H5O_loc_t *loc, hid_t dxpl_id, unsigned prot_flags) chk_udata.common.dxpl_id = dxpl_id; chk_udata.common.file_intent = file_intent; chk_udata.common.merged_null_msgs = udata.common.merged_null_msgs; - chk_udata.common.mesgs_modified = udata.common.mesgs_modified; chk_udata.common.cont_msg_info = &cont_msg_info; /* Read in continuation messages, until there are no more */ + /* (Note that loading chunks could increase the # of continuation + * messages if new ones are found - QAK, 19/11/2016) + */ curr_msg = 0; while(curr_msg < cont_msg_info.nmsgs) { H5O_chunk_proxy_t *chk_proxy; /* Proxy for chunk, to bring it into memory */ @@ -1877,16 +1903,12 @@ H5O_protect(const H5O_loc_t *loc, hid_t dxpl_id, unsigned prot_flags) /* Pass back out some of the chunk's user data */ udata.common.merged_null_msgs = chk_udata.common.merged_null_msgs; - udata.common.mesgs_modified = chk_udata.common.mesgs_modified; } /* end if */ /* Check for incorrect # of object header messages, if we've just loaded * this object header from the file */ if(udata.made_attempt) { - /* Check for incorrect # of messages in v1 object header */ - if(oh->version == H5O_VERSION_1 && - (oh->nmesgs + udata.common.merged_null_msgs) != udata.v1_pfx_nmesgs) { /* Don't enforce the error on an incorrect # of object header messages bug * unless strict format checking is enabled. This allows for older * files, created with a version of the library that had a bug in tracking @@ -1894,91 +1916,56 @@ H5O_protect(const H5O_loc_t *loc, hid_t dxpl_id, unsigned prot_flags) * erroring out here. -QAK */ #ifdef H5_STRICT_FORMAT_CHECKS + /* Check for incorrect # of messages in v1 object header */ + if(oh->version == H5O_VERSION_1 && + (oh->nmesgs + udata.common.merged_null_msgs) != udata.v1_pfx_nmesgs) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "corrupt object header - incorrect # of messages") -#else /* H5_STRICT_FORMAT_CHECKS */ - /* Mark object header prefix dirty later if we don't have write access */ - /* (object header will have been marked dirty during protect, if we - * have write access -QAK) - */ - if((prot_flags & H5AC__READ_ONLY_FLAG) != 0) - oh->prefix_modified = TRUE; -#ifndef NDEBUG - else { - unsigned oh_status = 0; /* Object header entry cache status */ +#endif /* H5_STRICT_FORMAT_CHECKS */ + } /* end if */ - /* Check the object header's status in the metadata cache */ - if(H5AC_get_entry_status(loc->file, loc->addr, &oh_status) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, NULL, "unable to check metadata cache status for object header") +#ifdef H5O_DEBUG +H5O_assert(oh); +#endif /* H5O_DEBUG */ - /* Make certain that object header is not dirty */ - HDassert(!(oh_status & H5AC_ES__IS_DIRTY)); - } /* end else */ -#endif /* NDEBUG */ -#endif /* H5_STRICT_FORMAT_CHECKS */ - } /* end if */ + /* Pin the other chunks also when requested, so that the object header + * proxy can be set up. + */ + if(pin_all_chunks && oh->nchunks > 1) { + unsigned u; /* Local index variable */ - /* Check for any messages that were modified while being read in */ - if(udata.common.mesgs_modified && (0 == (prot_flags & H5AC__READ_ONLY_FLAG))) - oh->mesgs_modified = TRUE; + /* Sanity check */ + HDassert(oh->swmr_write); - /* Reset the field that contained chunk 0's size during speculative load */ - oh->chunk0_size = 0; - } /* end if */ + /* Iterate over chunks > 0 */ + for(u = 1; u < oh->nchunks; u++) { + H5O_chunk_proxy_t *chk_proxy; /* Chunk proxy */ - /* Take care of loose ends for modifications made while bringing in the - * object header & chunks. - */ - if(0 == (prot_flags & H5AC__READ_ONLY_FLAG)) { - /* Check for the object header prefix being modified somehow */ - /* (usually through updating the # of object header messages) */ - if(oh->prefix_modified) { - /* Mark the header as dirty now */ - if(H5AC_mark_entry_dirty(oh) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTMARKDIRTY, NULL, "unable to mark object header as dirty") + /* Protect chunk */ + if(NULL == (chk_proxy = H5O_chunk_protect(loc->file, dxpl_id, oh, u))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, NULL, "unable to protect object header chunk") - /* Reset flag */ - oh->prefix_modified = FALSE; - } /* end if */ + /* Pin chunk proxy*/ + if(H5AC_pin_protected_entry(chk_proxy) < 0 ) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPIN, NULL, "unable to pin object header chunk") - /* Check for deferred dirty messages */ - if(oh->mesgs_modified) { - unsigned u; /* Local index variable */ - - /* Loop through all messages, marking their chunks as dirty */ - /* (slightly inefficient, since we don't know exactly which messages - * were modified when the object header & chunks were brought in - * from the file, but this only can happen once per load -QAK) - */ - for(u = 0; u < oh->nmesgs; u++) { - /* Mark each chunk with a dirty message as dirty also */ - if(oh->mesg[u].dirty) { - H5O_chunk_proxy_t *chk_proxy; /* Chunk that message is in */ - - /* Protect chunk */ - if(NULL == (chk_proxy = H5O_chunk_protect(loc->file, dxpl_id, oh, oh->mesg[u].chunkno))) - HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, NULL, "unable to load object header chunk") - - /* Unprotect chunk, marking it dirty */ - if(H5O_chunk_unprotect(loc->file, dxpl_id, chk_proxy, TRUE) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, NULL, "unable to unprotect object header chunk") - } /* end if */ - } /* end for */ + /* Unprotect chunk */ + if(H5O_chunk_unprotect(loc->file, dxpl_id, chk_proxy, FALSE) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, NULL, "unable to unprotect object header chunk") - /* Reset flag */ - oh->mesgs_modified = FALSE; - } /* end if */ - } /* end if */ + /* Preserve chunk proxy pointer for later */ + oh->chunk[u].chunk_proxy = chk_proxy; + } /* end for */ -#ifdef H5O_DEBUG -H5O_assert(oh); -#endif /* H5O_DEBUG */ + /* Set the flag for the unprotect */ + oh->chunks_pinned = TRUE; + } /* end if */ /* Set return value */ ret_value = oh; done: if(ret_value == NULL && oh) - if(H5AC_unprotect(loc->file, dxpl_id, H5AC_OHDR, loc->addr, oh, H5AC__NO_FLAGS_SET) < 0) + if(H5O_unprotect(loc, dxpl_id, oh, H5AC__NO_FLAGS_SET) < 0) HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, NULL, "unable to release object header") FUNC_LEAVE_NOAPI_TAG(ret_value, NULL) @@ -2014,7 +2001,7 @@ H5O_pin(const H5O_loc_t *loc, hid_t dxpl_id) HDassert(loc); /* Get header */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__NO_FLAGS_SET))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__NO_FLAGS_SET, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, NULL, "unable to protect object header") /* Increment the reference count on the object header */ @@ -2096,6 +2083,27 @@ H5O_unprotect(const H5O_loc_t *loc, hid_t dxpl_id, H5O_t *oh, unsigned oh_flags) HDassert(loc); HDassert(oh); + /* Unpin the other chunks */ + if(oh->chunks_pinned && oh->nchunks > 1) { + unsigned u; /* Local index variable */ + + /* Sanity check */ + HDassert(oh->swmr_write); + + /* Iterate over chunks > 0 */ + for(u = 1; u < oh->nchunks; u++) { + if(NULL != oh->chunk[u].chunk_proxy) { + /* Release chunk proxy */ + if(H5AC_unpin_entry(oh->chunk[u].chunk_proxy) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPIN, FAIL, "unable to unpin object header chunk") + oh->chunk[u].chunk_proxy = NULL; + } /* end if */ + } /* end for */ + + /* Reet the flag from the unprotect */ + oh->chunks_pinned = FALSE; + } /* end if */ + /* Unprotect the object header */ if(H5AC_unprotect(loc->file, dxpl_id, H5AC_OHDR, oh->chunk[0].addr, oh, oh_flags) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to release object header") @@ -2227,7 +2235,7 @@ H5O_touch(const H5O_loc_t *loc, hbool_t force, hid_t dxpl_id) HDassert(loc); /* Get the object header */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__NO_FLAGS_SET))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__NO_FLAGS_SET, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header") /* Create/Update the modification time message */ @@ -2353,7 +2361,7 @@ H5O_delete(H5F_t *f, hid_t dxpl_id, haddr_t addr) loc.holding_file = FALSE; /* Get the object header information */ - if(NULL == (oh = H5O_protect(&loc, dxpl_id, H5AC__NO_FLAGS_SET))) + if(NULL == (oh = H5O_protect(&loc, dxpl_id, H5AC__NO_FLAGS_SET, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header") /* Delete object */ @@ -2444,7 +2452,7 @@ H5O_obj_type(const H5O_loc_t *loc, H5O_type_t *obj_type, hid_t dxpl_id) FUNC_ENTER_NOAPI_TAG(dxpl_id, loc->addr, FAIL) /* Load the object header */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header") /* Retrieve the type of the object */ @@ -2522,7 +2530,7 @@ H5O_obj_class(const H5O_loc_t *loc, hid_t dxpl_id) FUNC_ENTER_NOAPI_NOINIT_TAG(dxpl_id, loc->addr, NULL) /* Load the object header */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, NULL, "unable to load object header") /* Test whether entry qualifies as a particular type of object */ @@ -2780,7 +2788,7 @@ H5O_loc_free(H5O_loc_t *loc) H5F_DECR_NOPEN_OBJS(loc->file); loc->holding_file = FALSE; if(H5F_NOPEN_OBJS(loc->file) <= 0) { - if(H5F_try_close(loc->file) < 0) + if(H5F_try_close(loc->file, NULL) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "can't close file") } /* end if */ } /* end if */ @@ -2819,7 +2827,7 @@ H5O_get_hdr_info(const H5O_loc_t *loc, hid_t dxpl_id, H5O_hdr_info_t *hdr) HDmemset(hdr, 0, sizeof(*hdr)); /* Get the object header */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, FAIL, "unable to load object header") /* Get the information for the object header */ @@ -2943,7 +2951,7 @@ H5O_get_info(const H5O_loc_t *loc, hid_t dxpl_id, hbool_t want_ih_info, HDassert(oinfo); /* Get the object header */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header") /* Reset the object info structure */ @@ -3064,7 +3072,7 @@ H5O_get_create_plist(const H5O_loc_t *loc, hid_t dxpl_id, H5P_genplist_t *oc_pli HDassert(oc_plist); /* Get the object header */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header") /* Set property values, if they were used for the object */ @@ -3119,7 +3127,7 @@ H5O_get_nlinks(const H5O_loc_t *loc, hid_t dxpl_id, hsize_t *nlinks) HDassert(nlinks); /* Get the object header */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header") /* Retrieve the # of link messages seen when the object header was loaded */ @@ -3236,7 +3244,7 @@ H5O_get_rc_and_type(const H5O_loc_t *loc, hid_t dxpl_id, unsigned *rc, H5O_type_ HDassert(loc); /* Get the object header */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header") /* Set the object's reference count */ @@ -3609,7 +3617,7 @@ H5O_dec_rc_by_loc(const H5O_loc_t *loc, hid_t dxpl_id) HDassert(loc); /* Get header */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to protect object header") /* Decrement the reference count on the object header */ @@ -3627,7 +3635,31 @@ done: /*------------------------------------------------------------------------- - * Function: H5O_free + * Function: H5O_get_proxy + * + * Purpose: Retrieve the proxy for the object header. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * July 24 2016 + * + *------------------------------------------------------------------------- + */ +H5AC_proxy_entry_t * +H5O_get_proxy(const H5O_t *oh) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check args */ + HDassert(oh); + + FUNC_LEAVE_NOAPI(oh->proxy) +} /* end H5O_get_proxy() */ + + +/*------------------------------------------------------------------------- + * Function: H5O__free * * Purpose: Destroys an object header. * @@ -3640,14 +3672,16 @@ done: *------------------------------------------------------------------------- */ herr_t -H5O_free(H5O_t *oh) +H5O__free(H5O_t *oh) { unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_PACKAGE /* check args */ HDassert(oh); + HDassert(0 == oh->rc); /* Destroy chunks */ if(oh->chunk) { @@ -3678,9 +3712,15 @@ H5O_free(H5O_t *oh) oh->mesg = (H5O_mesg_t *)H5FL_SEQ_FREE(H5O_mesg_t, oh->mesg); } /* end if */ + /* Destroy the proxy */ + if(oh->proxy) + if(H5AC_proxy_entry_dest(oh->proxy) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to destroy virtual entry used for proxy") + /* destroy object header */ oh = H5FL_FREE(H5O_t, oh); - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5O_free() */ +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O__free() */ diff --git a/src/H5Oalloc.c b/src/H5Oalloc.c index 69b4b11..4f98cfa 100644 --- a/src/H5Oalloc.c +++ b/src/H5Oalloc.c @@ -67,8 +67,11 @@ static herr_t H5O_alloc_null(H5F_t *f, hid_t dxpl_id, H5O_t *oh, size_t null_idx const H5O_msg_class_t *new_type, void *new_native, size_t new_size); static htri_t H5O_alloc_extend_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned chunkno, size_t size, size_t *msg_idx); +static herr_t H5O__alloc_find_best_nonnull(const H5F_t *f, const H5O_t *oh, size_t *size, + H5O_msg_alloc_info_t *found_msg); static herr_t H5O_alloc_new_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh, size_t size, size_t *new_idx); +static herr_t H5O__alloc_find_best_null(const H5O_t *oh, size_t size, size_t *mesg_idx); static htri_t H5O_move_cont(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned cont_u); static htri_t H5O_move_msgs_forward(H5F_t *f, hid_t dxpl_id, H5O_t *oh); static htri_t H5O_merge_null(H5F_t *f, hid_t dxpl_id, H5O_t *oh); @@ -644,9 +647,9 @@ H5O_alloc_extend_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned chunkno, old_size = oh->chunk[chunkno].size; oh->chunk[chunkno].size += delta + extra_prfx_size; oh->chunk[chunkno].image = H5FL_BLK_REALLOC(chunk_image, old_image, oh->chunk[chunkno].size); - oh->chunk[chunkno].gap = 0; if(NULL == oh->chunk[chunkno].image) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "can't reallocate extended object header chunk") + oh->chunk[chunkno].gap = 0; /* Wipe new space for chunk */ HDmemset(oh->chunk[chunkno].image + old_size, 0, oh->chunk[chunkno].size - old_size); @@ -664,8 +667,8 @@ H5O_alloc_extend_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned chunkno, oh->mesg[u].raw = oh->chunk[chunkno].image + extra_prfx_size + (oh->mesg[u].raw - old_image); /* Find continuation message which points to this chunk and adjust chunk's size */ - /* (Chunk 0 doesn't have a continuation message that points to it and - * it's size is directly encoded in the object header) */ + /* (Chunk 0 doesn't have a continuation message that points to it, + * its size is directly encoded in the object header) */ if(chunkno > 0 && (H5O_CONT_ID == oh->mesg[u].type->id) && (((H5O_cont_t *)(oh->mesg[u].native))->chunkno == chunkno)) { H5O_chunk_proxy_t *chk_proxy2 = NULL; /* Chunk that continuation message is in */ @@ -707,17 +710,10 @@ done: /*------------------------------------------------------------------------- - * Function: H5O_alloc_new_chunk - * - * Purpose: Allocates a new chunk for the object header, including - * file space. + * Function: H5O__alloc_find_best_nonnull * - * One of the other chunks will get an object continuation - * message. If there isn't room in any other chunk for the - * object continuation message, then some message from - * another chunk is moved into this chunk to make room. - * - * SIZE need not be aligned. + * Purpose: Find the best fit non-null message for a given size of message + * to allocate. * * Note: The algorithm for finding a message to replace with a * continuation message is still fairly limited. It's possible @@ -735,50 +731,33 @@ done: * * Failure: Negative * - * Programmer: Robb Matzke - * matzke@llnl.gov - * Aug 7 1997 + * Programmer: Quincey Koziol + * koziol@lbl.gov + * Oct 21 2016 * *------------------------------------------------------------------------- */ static herr_t -H5O_alloc_new_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh, size_t size, size_t *new_idx) +H5O__alloc_find_best_nonnull(const H5F_t *f, const H5O_t *oh, size_t *size, + H5O_msg_alloc_info_t *found_msg) { - /* Struct for storing information about "best" messages to allocate from */ - typedef struct { - int msgno; /* Index in message array */ - size_t gap_size; /* Size of any "gap" in the chunk immediately after message */ - size_t null_size; /* Size of any null message in the chunk immediately after message */ - size_t total_size; /* Total size of "available" space around message */ - unsigned null_msgno; /* Message index of null message immediately after message */ - } alloc_info; - H5O_mesg_t *curr_msg; /* Pointer to current message to operate on */ - H5O_chunk_proxy_t *chk_proxy; /* Chunk that message is in */ - size_t cont_size; /*continuation message size */ - size_t multi_size = 0; /* Size of all the messages in the last chunk */ - int found_null = (-1); /* Best fit null message */ - alloc_info found_attr = {-1, 0, 0, 0, 0}; /* Best fit attribute message */ - alloc_info found_other = {-1, 0, 0, 0, 0}; /* Best fit other message */ - size_t idx; /* Message number */ - uint8_t *p = NULL; /*ptr into new chunk */ - H5O_cont_t *cont = NULL; /*native continuation message */ - unsigned chunkno; /* Chunk allocated */ - haddr_t new_chunk_addr; + size_t cont_size; /* Continuation message size */ + size_t multi_size; /* Size of all the messages in the last chunk */ unsigned u; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC_NOERR - /* check args */ + /* Check args */ + HDassert(f); HDassert(oh); - HDassert(size > 0); - size = H5O_ALIGN_OH(oh, size); + HDassert(size); + HDassert(*size > 0); + HDassert(found_msg); /* - * Find the smallest null message that will hold an object - * continuation message. Failing that, find the smallest message - * that could be moved to make room for the continuation message. + * Find the smallest message that could be moved to make room for the + * continuation message. * * Don't ever move continuation message from one chunk to another. * @@ -787,23 +766,10 @@ H5O_alloc_new_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh, size_t size, size_t *new * */ cont_size = H5O_ALIGN_OH(oh, (size_t)(H5F_SIZEOF_ADDR(f) + H5F_SIZEOF_SIZE(f))); + multi_size = 0; for(u = 0, curr_msg = &oh->mesg[0]; u < oh->nmesgs; u++, curr_msg++) { - if(curr_msg->type->id == H5O_NULL_ID) { - if(cont_size == curr_msg->raw_size) { - found_null = (int)u; - break; - } /* end if */ - else if(curr_msg->raw_size > cont_size && - (found_null < 0 || curr_msg->raw_size < oh->mesg[found_null].raw_size)) - found_null = (int)u; - } /* end if */ - else if(curr_msg->type->id == H5O_CONT_ID) { - /* Don't consider continuation messages (for now) */ - } /* end if */ - else if(curr_msg->locked) { - /* Don't consider locked messages */ - } /* end if */ - else { + /* Don't consider continuation messages (for now) */ + if(H5O_CONT_ID != curr_msg->type->id) { unsigned msg_chunkno = curr_msg->chunkno; /* Chunk that the message is in */ uint8_t *end_chunk_data = (oh->chunk[msg_chunkno].image + oh->chunk[msg_chunkno].size) - (H5O_SIZEOF_CHKSUM_OH(oh) + oh->chunk[msg_chunkno].gap); /* End of message data in chunk */ uint8_t *end_msg = curr_msg->raw + curr_msg->raw_size; /* End of current message */ @@ -837,34 +803,51 @@ H5O_alloc_new_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh, size_t size, size_t *new /* Check if message is large enough to hold continuation info */ if(total_size >= cont_size) { - if(curr_msg->type->id == H5O_ATTR_ID) { - if(found_attr.msgno < 0 || total_size < found_attr.total_size) { - found_attr.msgno = (int)u; - found_attr.gap_size = gap_size; - found_attr.null_size = null_size; - found_attr.total_size = total_size; - found_attr.null_msgno = null_msgno; - } /* end if */ - } /* end if */ + hbool_t better = FALSE; /* Whether the current message is better than a previous one */ + + /* Check for first message that can be moved */ + if(found_msg->msgno < 0) + better = TRUE; else { - if(found_other.msgno < 0 || total_size < found_other.total_size) { - found_other.msgno = (int)u; - found_other.gap_size = gap_size; - found_other.null_size = null_size; - found_other.total_size = total_size; - found_other.null_msgno = null_msgno; - } /* end if */ + /* Prioritize moving non-attributes above attributes */ + /* (Even attributes with an otherwise better fit */ + if(found_msg->id == H5O_ATTR_ID && curr_msg->type->id != H5O_ATTR_ID) + better = TRUE; + /* Either two attributes, or two non-attributes */ + else { + /* Take a smaller one */ + if(total_size < found_msg->total_size) + better = TRUE; + /* If they are the same size, choose the earliest one + * in the chunk array */ + /* (Could also bias toward message earlier / later + * chunk in, but shouldn't be a big deal - QAK, 2016/10/21) + */ + else if(total_size == found_msg->total_size) { + if(msg_chunkno < found_msg->chunkno) + better = TRUE; + } /* end else-if */ + } /* end else */ } /* end else */ + + /* If we found a better message, keep its info */ + if(better) { + found_msg->msgno = (int)u; + found_msg->id = curr_msg->type->id; + found_msg->chunkno = msg_chunkno; + found_msg->gap_size = gap_size; + found_msg->null_size = null_size; + found_msg->total_size = total_size; + found_msg->null_msgno = null_msgno; + } /* end if */ } /* end if */ - else if(found_null < 0 && found_attr.msgno < 0 && found_other.msgno < 0 && msg_chunkno == oh->nchunks - 1) + else if(found_msg->msgno < 0 && msg_chunkno == oh->nchunks - 1) /* Keep track of the total size of smaller messages in the last * chunk, in case we need to move more than 1 message. */ multi_size += curr_msg->raw_size + (size_t)H5O_SIZEOF_MSGHDR_OH(oh); - } /* end else */ + } /* end if */ } /* end for */ - if(found_null >= 0 || found_attr.msgno >= 0 || found_other.msgno >= 0) - multi_size = 0; /* * If we must move some other message to make room for the null @@ -876,22 +859,60 @@ H5O_alloc_new_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh, size_t size, size_t *new * If all else fails, move every message in the last chunk. * */ - if(multi_size == 0) { - if(found_null < 0) { - if(found_other.msgno < 0) - found_other = found_attr; - - HDassert(found_other.msgno >= 0); - size += (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + oh->mesg[found_other.msgno].raw_size; - } /* end if */ - } /* end if */ + if(found_msg->msgno < 0) + *size += multi_size; else - size += multi_size; + *size += (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + oh->mesg[found_msg->msgno].raw_size; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5O__alloc_find_best_nonnull() */ + + +/*------------------------------------------------------------------------- + * Function: H5O__alloc_chunk + * + * Purpose: Allocates and initializes a new chunk for the object header, + * including file space. + * + * Return: Success: SUCCEED, with chunk number for the + * new chunk and a pointer to the location in its + * image where the first message should be placed. + * + * Failure: Negative + * + * Programmer: Quincey Koziol + * koziol@lbl.gov + * Oct 21 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5O__alloc_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh, size_t size, + size_t found_null, const H5O_msg_alloc_info_t *found_msg, size_t *new_idx) +{ + H5O_mesg_t *curr_msg; /* Pointer to current message to operate on */ + H5O_chunk_proxy_t *chk_proxy; /* Chunk that message is in */ + size_t cont_size; /*continuation message size */ + size_t idx; /* Message number */ + uint8_t *p = NULL; /* Pointer into new chunk image */ + H5O_cont_t *cont = NULL; /*native continuation message */ + unsigned chunkno; /* Chunk allocated */ + haddr_t new_chunk_addr; /* Address of new chunk in file */ + unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* check args */ + HDassert(f); + HDassert(oh); + HDassert(found_msg); + HDassert(new_idx); /* * The total chunk size must include the requested space plus enough * for the message header. This must be at least some minimum and - * aligned propertly. + * aligned properly. */ size = MAX(H5O_MIN_SIZE, size + (size_t)H5O_SIZEOF_MSGHDR_OH(oh)); HDassert(size == H5O_ALIGN_OH(oh, size)); @@ -903,20 +924,18 @@ H5O_alloc_new_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh, size_t size, size_t *new */ size += H5O_SIZEOF_CHKHDR_OH(oh); - /* allocate space in file to hold the new chunk */ + /* Allocate space in file to hold the new chunk */ new_chunk_addr = H5MF_alloc(f, H5FD_MEM_OHDR, dxpl_id, (hsize_t)size); - if(HADDR_UNDEF == new_chunk_addr) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate space for new chunk") + if(!H5F_addr_defined(new_chunk_addr)) + HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "unable to allocate space for new chunk") - /* - * Create the new chunk giving it a file address. - */ + /* Create the new chunk giving it a file address. */ if(oh->nchunks >= oh->alloc_nchunks) { size_t na = MAX(H5O_NCHUNKS, oh->alloc_nchunks * 2); /* Double # of chunks allocated */ H5O_chunk_t *x; if(NULL == (x = H5FL_SEQ_REALLOC(H5O_chunk_t, oh->chunk, na))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "can't allocate larger chunk array, na = %zu", na) oh->alloc_nchunks = na; oh->chunk = x; } /* end if */ @@ -927,7 +946,8 @@ H5O_alloc_new_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh, size_t size, size_t *new oh->chunk[chunkno].size = size; oh->chunk[chunkno].gap = 0; if(NULL == (oh->chunk[chunkno].image = p = H5FL_BLK_CALLOC(chunk_image, size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "can't allocate image for chunk, size = %zu", size) + oh->chunk[chunkno].chunk_proxy = NULL; /* If this is a later version of the object header format, put the magic * # at the beginning of the chunk image. @@ -946,133 +966,136 @@ H5O_alloc_new_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh, size_t size, size_t *new HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate more space for messages") /* Check if we need to move multiple messages, in order to make room for the new message */ - if(multi_size > 0) { - /* Move all non-null messages in the last chunk to the new chunk. This - * should be extremely rare so we don't care too much about minimizing - * the space used. - */ - H5O_mesg_t *null_msg; /* Pointer to new null message */ - - /* Protect last chunk */ - if(NULL == (chk_proxy = H5O_chunk_protect(f, dxpl_id, oh, chunkno - 1))) - HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header chunk") - - /* Copy each message to the new location */ - for(u = 0, curr_msg = &oh->mesg[0]; u < oh->nmesgs; u++, curr_msg++) - if(curr_msg->chunkno == chunkno - 1) { - if(curr_msg->type->id == H5O_NULL_ID) { - /* Delete the null message */ - if(u < oh->nmesgs - 1) - HDmemmove(curr_msg, curr_msg + 1, ((oh->nmesgs - 1) - u) * sizeof(H5O_mesg_t)); - oh->nmesgs--; + cont_size = H5O_ALIGN_OH(oh, (size_t)(H5F_SIZEOF_ADDR(f) + H5F_SIZEOF_SIZE(f))); + if(found_null >= oh->nmesgs) { + if(found_msg->msgno < 0) { + /* Move all non-null messages in the last chunk to the new chunk. This + * should be extremely rare so we don't care too much about minimizing + * the space used. + */ + H5O_mesg_t *null_msg; /* Pointer to new null message */ + + /* Protect last chunk */ + if(NULL == (chk_proxy = H5O_chunk_protect(f, dxpl_id, oh, chunkno - 1))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header chunk") + + /* Copy each message to the new location */ + for(u = 0, curr_msg = &oh->mesg[0]; u < oh->nmesgs; u++, curr_msg++) + if(curr_msg->chunkno == chunkno - 1) { + if(curr_msg->type->id == H5O_NULL_ID) { + /* Delete the null message */ + if(u < oh->nmesgs - 1) + HDmemmove(curr_msg, curr_msg + 1, ((oh->nmesgs - 1) - u) * sizeof(H5O_mesg_t)); + oh->nmesgs--; + } /* end if */ + else { + HDassert(curr_msg->type->id != H5O_CONT_ID); + + /* Copy the raw data */ + HDmemcpy(p, curr_msg->raw - (size_t)H5O_SIZEOF_MSGHDR_OH(oh), + curr_msg->raw_size + (size_t)H5O_SIZEOF_MSGHDR_OH(oh)); + + /* Update the message info */ + curr_msg->chunkno = chunkno; + curr_msg->raw = p + H5O_SIZEOF_MSGHDR_OH(oh); + + /* Account for copied message in new chunk */ + p += (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + curr_msg->raw_size; + size -= (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + curr_msg->raw_size; + } /* end else */ } /* end if */ - else { - HDassert(curr_msg->type->id != H5O_CONT_ID); - /* Copy the raw data */ - HDmemcpy(p, curr_msg->raw - (size_t)H5O_SIZEOF_MSGHDR_OH(oh), - curr_msg->raw_size + (size_t)H5O_SIZEOF_MSGHDR_OH(oh)); + /* Create a null message spanning the entire last chunk */ + found_null = oh->nmesgs++; + null_msg = &(oh->mesg[found_null]); + null_msg->type = H5O_MSG_NULL; + null_msg->dirty = TRUE; + null_msg->native = NULL; + null_msg->raw = oh->chunk[chunkno - 1].image + + ((chunkno == 1) ? H5O_SIZEOF_HDR(oh) : H5O_SIZEOF_CHKHDR_OH(oh)) + - H5O_SIZEOF_CHKSUM_OH(oh) + H5O_SIZEOF_MSGHDR_OH(oh); + null_msg->raw_size = oh->chunk[chunkno - 1].size + - ((chunkno == 1) ? (size_t)H5O_SIZEOF_HDR(oh) : (size_t)H5O_SIZEOF_CHKHDR_OH(oh)) + - (size_t)H5O_SIZEOF_MSGHDR_OH(oh); + null_msg->chunkno = chunkno - 1; - /* Update the message info */ - curr_msg->chunkno = chunkno; - curr_msg->raw = p + H5O_SIZEOF_MSGHDR_OH(oh); + HDassert(null_msg->raw_size >= cont_size); - /* Account for copied message in new chunk */ - p += (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + curr_msg->raw_size; - size -= (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + curr_msg->raw_size; - } /* end else */ - } /* end if */ + /* Remove any gap in the chunk */ + oh->chunk[chunkno - 1].gap = 0; - /* Create a null message spanning the entire last chunk */ - found_null = (int)oh->nmesgs++; - null_msg = &(oh->mesg[found_null]); - null_msg->type = H5O_MSG_NULL; - null_msg->dirty = TRUE; - null_msg->native = NULL; - null_msg->raw = oh->chunk[chunkno - 1].image - + ((chunkno == 1) ? H5O_SIZEOF_HDR(oh) : H5O_SIZEOF_CHKHDR_OH(oh)) - - H5O_SIZEOF_CHKSUM_OH(oh) + H5O_SIZEOF_MSGHDR_OH(oh); - null_msg->raw_size = oh->chunk[chunkno - 1].size - - ((chunkno == 1) ? (size_t)H5O_SIZEOF_HDR(oh) : (size_t)H5O_SIZEOF_CHKHDR_OH(oh)) - - (size_t)H5O_SIZEOF_MSGHDR_OH(oh); - null_msg->chunkno = chunkno - 1; - - HDassert(null_msg->raw_size >= cont_size); - - /* Remove any gap in the chunk */ - oh->chunk[chunkno - 1].gap = 0; - - /* Release chunk, marking it dirty */ - if(H5O_chunk_unprotect(f, dxpl_id, chk_proxy, TRUE) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk") - } else if(found_null < 0) { - /* Move message (that will be replaced with continuation message) - * to new chunk, if necessary. - */ - H5O_mesg_t *null_msg; /* Pointer to new null message */ - - /* Protect chunk */ - if(NULL == (chk_proxy = H5O_chunk_protect(f, dxpl_id, oh, oh->mesg[found_other.msgno].chunkno))) - HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header chunk") - - /* Create null message for space that message to copy currently occupies */ - found_null = (int)oh->nmesgs++; - null_msg = &(oh->mesg[found_null]); - null_msg->type = H5O_MSG_NULL; - null_msg->native = NULL; - null_msg->raw = oh->mesg[found_other.msgno].raw; - null_msg->raw_size = oh->mesg[found_other.msgno].raw_size; - null_msg->chunkno = oh->mesg[found_other.msgno].chunkno; - - /* Copy the message to move (& its prefix) to its new location */ - HDmemcpy(p, oh->mesg[found_other.msgno].raw - H5O_SIZEOF_MSGHDR_OH(oh), - oh->mesg[found_other.msgno].raw_size + (size_t)H5O_SIZEOF_MSGHDR_OH(oh)); - - /* Switch moved message to point to new location */ - oh->mesg[found_other.msgno].raw = p + H5O_SIZEOF_MSGHDR_OH(oh); - oh->mesg[found_other.msgno].chunkno = chunkno; - - /* Account for copied message in new chunk */ - p += (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + oh->mesg[found_other.msgno].raw_size; - size -= (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + oh->mesg[found_other.msgno].raw_size; - - /* Add any available space after the message to move to the new null message */ - if(found_other.gap_size > 0) { - /* Absorb a gap after the moved message */ - HDassert(oh->chunk[null_msg->chunkno].gap == found_other.gap_size); - null_msg->raw_size += found_other.gap_size; - oh->chunk[null_msg->chunkno].gap = 0; + /* Release chunk, marking it dirty */ + if(H5O_chunk_unprotect(f, dxpl_id, chk_proxy, TRUE) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk") } /* end if */ - else if(found_other.null_size > 0) { - H5O_mesg_t *old_null_msg = &oh->mesg[found_other.null_msgno]; /* Pointer to NULL message to eliminate */ + else { + /* Move message (that will be replaced with continuation message) + * to new chunk, if necessary. + */ + H5O_mesg_t *null_msg; /* Pointer to new null message */ - /* Absorb a null message after the moved message */ - HDassert((null_msg->raw + null_msg->raw_size) == (old_null_msg->raw - H5O_SIZEOF_MSGHDR_OH(oh))); - null_msg->raw_size += found_other.null_size; + /* Protect chunk */ + if(NULL == (chk_proxy = H5O_chunk_protect(f, dxpl_id, oh, oh->mesg[found_msg->msgno].chunkno))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header chunk") - /* Release any information/memory for message */ - H5O_msg_free_mesg(old_null_msg); + /* Create null message for space that message to copy currently occupies */ + found_null = oh->nmesgs++; + null_msg = &(oh->mesg[found_null]); + null_msg->type = H5O_MSG_NULL; + null_msg->native = NULL; + null_msg->raw = oh->mesg[found_msg->msgno].raw; + null_msg->raw_size = oh->mesg[found_msg->msgno].raw_size; + null_msg->chunkno = oh->mesg[found_msg->msgno].chunkno; + + /* Copy the message to move (& its prefix) to its new location */ + HDmemcpy(p, oh->mesg[found_msg->msgno].raw - H5O_SIZEOF_MSGHDR_OH(oh), + oh->mesg[found_msg->msgno].raw_size + (size_t)H5O_SIZEOF_MSGHDR_OH(oh)); + + /* Switch moved message to point to new location */ + oh->mesg[found_msg->msgno].raw = p + H5O_SIZEOF_MSGHDR_OH(oh); + oh->mesg[found_msg->msgno].chunkno = chunkno; + + /* Account for copied message in new chunk */ + p += (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + oh->mesg[found_msg->msgno].raw_size; + size -= (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + oh->mesg[found_msg->msgno].raw_size; + + /* Add any available space after the message to move to the new null message */ + if(found_msg->gap_size > 0) { + /* Absorb a gap after the moved message */ + HDassert(oh->chunk[null_msg->chunkno].gap == found_msg->gap_size); + null_msg->raw_size += found_msg->gap_size; + oh->chunk[null_msg->chunkno].gap = 0; + } /* end if */ + else if(found_msg->null_size > 0) { + H5O_mesg_t *old_null_msg = &oh->mesg[found_msg->null_msgno]; /* Pointer to NULL message to eliminate */ - /* Remove null message from list of messages */ - if(found_other.null_msgno < (oh->nmesgs - 1)) - HDmemmove(old_null_msg, old_null_msg + 1, ((oh->nmesgs - 1) - found_other.null_msgno) * sizeof(H5O_mesg_t)); + /* Absorb a null message after the moved message */ + HDassert((null_msg->raw + null_msg->raw_size) == (old_null_msg->raw - H5O_SIZEOF_MSGHDR_OH(oh))); + null_msg->raw_size += found_msg->null_size; - /* Decrement # of messages */ - /* (Don't bother reducing size of message array for now -QAK) */ - oh->nmesgs--; + /* Release any information/memory for message */ + H5O_msg_free_mesg(old_null_msg); - /* Adjust message index for new NULL message */ - found_null--; - } /* end if */ + /* Remove null message from list of messages */ + if(found_msg->null_msgno < (oh->nmesgs - 1)) + HDmemmove(old_null_msg, old_null_msg + 1, ((oh->nmesgs - 1) - found_msg->null_msgno) * sizeof(H5O_mesg_t)); - /* Mark the new null message as dirty */ - null_msg->dirty = TRUE; + /* Decrement # of messages */ + /* (Don't bother reducing size of message array for now -QAK) */ + oh->nmesgs--; + + /* Adjust message index for new NULL message */ + found_null--; + } /* end if */ + + /* Mark the new null message as dirty */ + null_msg->dirty = TRUE; - /* Release chunk, marking it dirty */ - if(H5O_chunk_unprotect(f, dxpl_id, chk_proxy, TRUE) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk") + /* Release chunk, marking it dirty */ + if(H5O_chunk_unprotect(f, dxpl_id, chk_proxy, TRUE) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk") + } /* end if */ } /* end if */ - HDassert(found_null >= 0); /* Create null message for [rest of] space in new chunk */ /* (account for chunk's magic # & checksum) */ @@ -1085,7 +1108,7 @@ H5O_alloc_new_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh, size_t size, size_t *new oh->mesg[idx].chunkno = chunkno; /* Insert the new chunk into the cache */ - if(H5O_chunk_add(f, dxpl_id, oh, chunkno) < 0) + if(H5O_chunk_add(f, dxpl_id, oh, chunkno, oh->mesg[found_null].chunkno) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINSERT, FAIL, "can't add new chunk to cache") /* Initialize the continuation information */ @@ -1096,7 +1119,7 @@ H5O_alloc_new_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh, size_t size, size_t *new cont->chunkno = chunkno; /* Split the null message and point at continuation message */ - if(H5O_alloc_null(f, dxpl_id, oh, (size_t)found_null, H5O_MSG_CONT, cont, cont_size) < 0) + if(H5O_alloc_null(f, dxpl_id, oh, found_null, H5O_MSG_CONT, cont, cont_size) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINSERT, FAIL, "can't split null message") /* Set new message index value */ @@ -1104,16 +1127,163 @@ H5O_alloc_new_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh, size_t size, size_t *new done: FUNC_LEAVE_NOAPI(ret_value) +} /* H5O__alloc_chunk() */ + + +/*------------------------------------------------------------------------- + * Function: H5O_alloc_new_chunk + * + * Purpose: Allocates a new chunk for the object header, including + * file space. + * + * One of the other chunks will get an object continuation + * message. If there isn't room in any other chunk for the + * object continuation message, then some message from + * another chunk is moved into this chunk to make room. + * + * SIZE need not be aligned. + * + * Note: The algorithm for finding a message to replace with a + * continuation message is still fairly limited. It's possible + * that two (or more) messages smaller than a continuation message + * might occupy a chunk and need to be moved in order to make + * room for the continuation message. + * + * Also, we aren't checking for NULL messages in front of another + * message right now... + * + * Return: Success: Index number of the null message for the + * new chunk. The null message will be at + * least SIZE bytes not counting the message + * ID or size fields. + * + * Failure: Negative + * + * Programmer: Robb Matzke + * matzke@llnl.gov + * Aug 7 1997 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O_alloc_new_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh, size_t size, size_t *new_idx) +{ + size_t cont_size; /*continuation message size */ + size_t idx; /* Message number */ + H5O_msg_alloc_info_t found_msg; /* Best fit non-null message */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* check args */ + HDassert(oh); + HDassert(size > 0); + size = H5O_ALIGN_OH(oh, size); + + /* Find the smallest null message that could hold a continuation message */ + idx = oh->nmesgs; + cont_size = H5O_ALIGN_OH(oh, (size_t)(H5F_SIZEOF_ADDR(f) + H5F_SIZEOF_SIZE(f))); + if(H5O__alloc_find_best_null(oh, cont_size, &idx) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_NOTFOUND, FAIL, "error while locating best null header message") + + /* If we couldn't find a null message, locate the best message to move to new chunk */ + if(idx >= oh->nmesgs) { + found_msg.msgno = -1; + if(H5O__alloc_find_best_nonnull(f, oh, &size, &found_msg) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_NOTFOUND, FAIL, "error while locating best non-null header message") + } /* end if */ + + /* Allocate and initialize new chunk in the file */ + if(H5O__alloc_chunk(f, dxpl_id, oh, size, idx, &found_msg, new_idx) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "can't allocate new object header chunk") + +done: + FUNC_LEAVE_NOAPI(ret_value) } /* H5O_alloc_new_chunk() */ /*------------------------------------------------------------------------- + * Function: H5O__alloc_find_best_null + * + * Purpose: Find the best fit null message for a given size of message + * to allocate. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@lbl.gov + * Oct 21 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O__alloc_find_best_null(const H5O_t *oh, size_t size, size_t *mesg_idx) +{ + size_t idx; /* Index of message which fits allocation */ + ssize_t found_null; /* Best fit null message */ + + FUNC_ENTER_STATIC_NOERR + + /* check args */ + HDassert(oh); + HDassert(size > 0); + HDassert(mesg_idx); + + /* Find the smallest null message that could hold the new object header message */ + found_null = -1; + for(idx = 0; idx < oh->nmesgs; idx++) { + if(H5O_NULL_ID == oh->mesg[idx].type->id) { + /* If we found an exact fit, use it */ + if(oh->mesg[idx].raw_size == size) { + /* Keep first exact fit */ + if(found_null < 0) + found_null = (ssize_t)idx; + else + /* If we've got more than one exact fit, choose the one in the earliest chunk */ + if(oh->mesg[idx].chunkno < oh->mesg[found_null].chunkno) { + found_null = (ssize_t)idx; + + /* If we found an exact fit in object header chunk #0, we can get out */ + /* (Could extend this to look for earliest message in + * chunk #0 - QAK, 2016/10/21) + */ + if(0 == oh->mesg[idx].chunkno) + break; + } /* end if */ + } /* end if */ + /* Look for null message that's larger than needed */ + else if(oh->mesg[idx].raw_size > size) { + /* Keep first one found */ + if(found_null < 0) + found_null = (ssize_t)idx; + else + /* Check for better fit */ + if(oh->mesg[idx].raw_size < oh->mesg[found_null].raw_size) + found_null = (ssize_t)idx; + else { + /* If they are the same size, choose the one in the earliest chunk */ + if(oh->mesg[idx].raw_size == oh->mesg[found_null].raw_size) { + if(oh->mesg[idx].chunkno < oh->mesg[found_null].chunkno) + found_null = (ssize_t)idx; + } /* end if */ + } /* end else */ + } /* end else-if */ + /* else: Ignore too-small null messages */ + } /* end if */ + } /* end for */ + if(found_null >= 0) + *mesg_idx = (size_t)found_null; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5O__alloc_find_best_null() */ + + +/*------------------------------------------------------------------------- * Function: H5O_alloc * * Purpose: Allocate enough space in the object header for this message. * - * Return: Success: Index of message - * Failure: Negative + * Return: Non-negative on success/Negative on failure * * Programmer: Robb Matzke * matzke@llnl.gov @@ -1146,10 +1316,10 @@ H5O_alloc(H5F_t *f, hid_t dxpl_id, H5O_t *oh, const H5O_msg_class_t *type, HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, FAIL, "object header message is too large") aligned_size = H5O_ALIGN_OH(oh, raw_size); - /* look for a null message which is large enough */ - for(idx = 0; idx < oh->nmesgs; idx++) - if(H5O_NULL_ID == oh->mesg[idx].type->id && oh->mesg[idx].raw_size >= aligned_size) - break; + /* Find the smallest null message that could hold the new object header message */ + idx = oh->nmesgs; + if(H5O__alloc_find_best_null(oh, aligned_size, &idx) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_NOTFOUND, FAIL, "error while locating best null header message") /* if we didn't find one, then allocate more header space */ if(idx >= oh->nmesgs) { @@ -1165,7 +1335,7 @@ H5O_alloc(H5F_t *f, hid_t dxpl_id, H5O_t *oh, const H5O_msg_class_t *type, htri_t tri_result; /* Status from attempting to extend chunk */ if((tri_result = H5O_alloc_extend_chunk(f, dxpl_id, oh, chunkno, raw_size, &idx)) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTEXTEND, FAIL, "H5O_alloc_extend_chunk failed unexpectedly") + HGOTO_ERROR(H5E_OHDR, H5E_CANTEXTEND, FAIL, "can't extend existing chunk") if(tri_result == TRUE) break; } /* end for */ @@ -1310,12 +1480,6 @@ H5O_move_cont(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned cont_u) nonnull_size = 0; for(v = 0, curr_msg = &oh->mesg[0]; v < oh->nmesgs; v++, curr_msg++) { if(curr_msg->chunkno == deleted_chunkno) { - /* If there's a locked message, we can't move all messages out of - * chunk to delete, so get out now. - */ - if(curr_msg->locked) - HGOTO_DONE(FALSE) - /* Find size of all non-null messages in the chunk pointed to by the continuation message */ if(curr_msg->type->id != H5O_NULL_ID) { HDassert(curr_msg->type->id != H5O_CONT_ID); @@ -1458,6 +1622,7 @@ H5O_move_msgs_forward(H5F_t *f, hid_t dxpl_id, H5O_t *oh) { H5O_chunk_proxy_t *null_chk_proxy = NULL; /* Chunk that null message is in */ H5O_chunk_proxy_t *curr_chk_proxy = NULL; /* Chunk that message is in */ + H5O_chunk_proxy_t *cont_targ_chk_proxy = NULL; /* Chunk that continuation message points to */ hbool_t null_chk_dirtied = FALSE; /* Flags for unprotecting null chunk */ hbool_t curr_chk_dirtied = FALSE; /* Flags for unprotecting curr chunk */ hbool_t packed_msg; /* Flag to indicate that messages were packed */ @@ -1555,152 +1720,203 @@ H5O_move_msgs_forward(H5F_t *f, hid_t dxpl_id, H5O_t *oh) } /* end else-if */ } /* end if */ - /* Don't let locked messages be moved into earlier chunk */ - if(!curr_msg->locked) { - /* Loop over messages again, looking for large enough null message in earlier chunk */ - for(v = 0, null_msg = &oh->mesg[0]; v < oh->nmesgs; v++, null_msg++) { - if(H5O_NULL_ID == null_msg->type->id && curr_msg->chunkno > null_msg->chunkno - && curr_msg->raw_size <= null_msg->raw_size) { - unsigned old_chunkno; /* Old message information */ - uint8_t *old_raw; - - /* Keep old information about non-null message */ - old_chunkno = curr_msg->chunkno; - old_raw = curr_msg->raw; - - /* Protect chunks */ - if(NULL == (null_chk_proxy = H5O_chunk_protect(f, dxpl_id, oh, null_msg->chunkno))) - HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header chunk") - if(NULL == (curr_chk_proxy = H5O_chunk_protect(f, dxpl_id, oh, curr_msg->chunkno))) + /* Loop over messages again, looking for large enough null message in earlier chunk */ + for(v = 0, null_msg = &oh->mesg[0]; v < oh->nmesgs; v++, null_msg++) { + if(H5O_NULL_ID == null_msg->type->id && curr_msg->chunkno > null_msg->chunkno + && curr_msg->raw_size <= null_msg->raw_size) { + unsigned old_chunkno; /* Old message information */ + uint8_t *old_raw; + + /* Keep old information about non-null message */ + old_chunkno = curr_msg->chunkno; + old_raw = curr_msg->raw; + + /* Protect chunks */ + if(NULL == (null_chk_proxy = H5O_chunk_protect(f, dxpl_id, oh, null_msg->chunkno))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header chunk") + if(NULL == (curr_chk_proxy = H5O_chunk_protect(f, dxpl_id, oh, curr_msg->chunkno))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header chunk") + + /* If the message being moved is a continuation + * message and we are doing SWMR writes, we must + * update the flush dependencies */ + if(oh->swmr_write && (H5O_CONT_ID == curr_msg->type->id)) { + void *null_chk_mdc_obj; /* The metadata cache object for the null_msg chunk */ + + /* Point to the metadata cache object for the + * null message chunk, oh if in chunk 0 or the + * proxy otherwise */ + null_chk_mdc_obj = (null_msg->chunkno == 0 + ? (void *)oh + : (void *)null_chk_proxy); + + /* The other chunks involved should never be + * chunk 0 */ + HDassert(curr_msg->chunkno > 0); + HDassert(((H5O_cont_t *)(curr_msg->native))->chunkno > 0); + + /* Protect continuation message target chunk */ + if(NULL == (cont_targ_chk_proxy = H5O_chunk_protect(f, dxpl_id, oh, ((H5O_cont_t *)(curr_msg->native))->chunkno))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header chunk") - /* Copy raw data for non-null message to new chunk */ - HDmemcpy(null_msg->raw - H5O_SIZEOF_MSGHDR_OH(oh), curr_msg->raw - H5O_SIZEOF_MSGHDR_OH(oh), curr_msg->raw_size + (size_t)H5O_SIZEOF_MSGHDR_OH(oh)); + /* Remove flush dependency on old continuation + * message chunk */ + HDassert(cont_targ_chk_proxy); + HDassert(cont_targ_chk_proxy->parent); + HDassert(curr_chk_proxy); + HDassert((void *)curr_chk_proxy == cont_targ_chk_proxy->parent); - /* Point non-null message at null message's space */ - curr_msg->chunkno = null_msg->chunkno; - curr_msg->raw = null_msg->raw; - curr_chk_dirtied = TRUE; + if(H5AC_destroy_flush_dependency(curr_chk_proxy, cont_targ_chk_proxy) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTUNDEPEND, FAIL, "unable to destroy flush dependency") + + cont_targ_chk_proxy->parent = NULL; + + /* Create flush dependency on new continuation + * message chunk */ + if(H5AC_create_flush_dependency(null_chk_mdc_obj, cont_targ_chk_proxy) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTDEPEND, FAIL, "unable to create flush dependency") + + HDassert(null_chk_mdc_obj); + HDassert(((H5C_cache_entry_t *)null_chk_mdc_obj)->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + HDassert(((H5C_cache_entry_t *)null_chk_mdc_obj)->type); + HDassert((((H5C_cache_entry_t *)null_chk_mdc_obj)->type->id == H5AC_OHDR_ID) || + (((H5C_cache_entry_t *)null_chk_mdc_obj)->type->id == H5AC_OHDR_CHK_ID)); + + cont_targ_chk_proxy->parent = null_chk_mdc_obj; + + /* Unprotect continuation message target chunk + */ + if(H5O_chunk_unprotect(f, dxpl_id, cont_targ_chk_proxy, FALSE) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk") + cont_targ_chk_proxy = NULL; + } /* end if */ + + /* Copy raw data for non-null message to new chunk */ + HDmemcpy(null_msg->raw - H5O_SIZEOF_MSGHDR_OH(oh), curr_msg->raw - H5O_SIZEOF_MSGHDR_OH(oh), curr_msg->raw_size + (size_t)H5O_SIZEOF_MSGHDR_OH(oh)); + + /* Point non-null message at null message's space */ + curr_msg->chunkno = null_msg->chunkno; + curr_msg->raw = null_msg->raw; + curr_chk_dirtied = TRUE; + + /* Change information for null message */ + if(curr_msg->raw_size == null_msg->raw_size) { + /* Point null message at old non-null space */ + /* (Instead of freeing it and allocating new message) */ + null_msg->chunkno = old_chunkno; + null_msg->raw = old_raw; + + /* Mark null message dirty */ + null_msg->dirty = TRUE; + null_chk_dirtied = TRUE; + + /* Release current chunk, marking it dirty */ + if(H5O_chunk_unprotect(f, dxpl_id, curr_chk_proxy, curr_chk_dirtied) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk") + curr_chk_proxy = NULL; + curr_chk_dirtied = FALSE; + + /* Check for gap in null message's chunk */ + if(oh->chunk[old_chunkno].gap > 0) { + /* Eliminate the gap in the chunk */ + if(H5O_eliminate_gap(oh, &null_chk_dirtied, null_msg, + ((oh->chunk[old_chunkno].image + oh->chunk[old_chunkno].size) - (H5O_SIZEOF_CHKSUM_OH(oh) + oh->chunk[old_chunkno].gap)), + oh->chunk[old_chunkno].gap) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTREMOVE, FAIL, "can't eliminate gap in chunk") + } /* end if */ + + /* Release null chunk, marking it dirty */ + if(H5O_chunk_unprotect(f, dxpl_id, null_chk_proxy, null_chk_dirtied) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk") + null_chk_proxy = NULL; + null_chk_dirtied = FALSE; + } /* end if */ + else { + size_t new_null_msg; /* Message index for new null message */ - /* Change information for null message */ - if(curr_msg->raw_size == null_msg->raw_size) { - /* Point null message at old non-null space */ - /* (Instead of freeing it and allocating new message) */ - null_msg->chunkno = old_chunkno; - null_msg->raw = old_raw; + /* Check if null message is large enough to still exist */ + if((null_msg->raw_size - curr_msg->raw_size) < (size_t)H5O_SIZEOF_MSGHDR_OH(oh)) { + size_t gap_size = null_msg->raw_size - curr_msg->raw_size; /* Size of gap produced */ + + /* Adjust the size of the null message being eliminated */ + null_msg->raw_size = curr_msg->raw_size; /* Mark null message dirty */ null_msg->dirty = TRUE; null_chk_dirtied = TRUE; - /* Release current chunk, marking it dirty */ - if(H5O_chunk_unprotect(f, dxpl_id, curr_chk_proxy, curr_chk_dirtied) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk") - curr_chk_proxy = NULL; - curr_chk_dirtied = FALSE; - - /* Check for gap in null message's chunk */ - if(oh->chunk[old_chunkno].gap > 0) { - /* Eliminate the gap in the chunk */ - if(H5O_eliminate_gap(oh, &null_chk_dirtied, null_msg, - ((oh->chunk[old_chunkno].image + oh->chunk[old_chunkno].size) - (H5O_SIZEOF_CHKSUM_OH(oh) + oh->chunk[old_chunkno].gap)), - oh->chunk[old_chunkno].gap) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTREMOVE, FAIL, "can't eliminate gap in chunk") - } /* end if */ + /* Add the gap to the chunk */ + if(H5O_add_gap(f, oh, null_msg->chunkno, &null_chk_dirtied, v, null_msg->raw + null_msg->raw_size, gap_size) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTINSERT, FAIL, "can't insert gap in chunk") - /* Release null chunk, marking it dirty */ - if(H5O_chunk_unprotect(f, dxpl_id, null_chk_proxy, null_chk_dirtied) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk") - null_chk_proxy = NULL; - null_chk_dirtied = FALSE; + /* Re-use message # for new null message taking place of non-null message */ + new_null_msg = v; } /* end if */ else { - size_t new_null_msg; /* Message index for new null message */ + /* Adjust null message's size & offset */ + null_msg->raw += curr_msg->raw_size + (size_t)H5O_SIZEOF_MSGHDR_OH(oh); + null_msg->raw_size -= curr_msg->raw_size + (size_t)H5O_SIZEOF_MSGHDR_OH(oh); - /* Check if null message is large enough to still exist */ - if((null_msg->raw_size - curr_msg->raw_size) < (size_t)H5O_SIZEOF_MSGHDR_OH(oh)) { - size_t gap_size = null_msg->raw_size - curr_msg->raw_size; /* Size of gap produced */ - - /* Adjust the size of the null message being eliminated */ - null_msg->raw_size = curr_msg->raw_size; - - /* Mark null message dirty */ - null_msg->dirty = TRUE; - null_chk_dirtied = TRUE; + /* Mark null message dirty */ + null_msg->dirty = TRUE; + null_chk_dirtied = TRUE; - /* Add the gap to the chunk */ - if(H5O_add_gap(f, oh, null_msg->chunkno, &null_chk_dirtied, v, null_msg->raw + null_msg->raw_size, gap_size) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTINSERT, FAIL, "can't insert gap in chunk") + /* Create new null message for previous location of non-null message */ + if(oh->nmesgs >= oh->alloc_nmesgs) { + if(H5O_alloc_msgs(oh, (size_t)1) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate more space for messages") - /* Re-use message # for new null message taking place of non-null message */ - new_null_msg = v; - } /* end if */ - else { - /* Adjust null message's size & offset */ - null_msg->raw += curr_msg->raw_size + (size_t)H5O_SIZEOF_MSGHDR_OH(oh); - null_msg->raw_size -= curr_msg->raw_size + (size_t)H5O_SIZEOF_MSGHDR_OH(oh); - - /* Mark null message dirty */ - null_msg->dirty = TRUE; - null_chk_dirtied = TRUE; - - /* Create new null message for previous location of non-null message */ - if(oh->nmesgs >= oh->alloc_nmesgs) { - if(H5O_alloc_msgs(oh, (size_t)1) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate more space for messages") - - /* "Retarget" 'curr_msg' pointer into newly re-allocated array of messages */ - curr_msg = &oh->mesg[u]; - } /* end if */ - - /* Get message # for new null message */ - new_null_msg = oh->nmesgs++; - } /* end else */ - - /* Release null message's chunk, marking it dirty */ - if(H5O_chunk_unprotect(f, dxpl_id, null_chk_proxy, null_chk_dirtied) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk") - null_chk_proxy = NULL; - null_chk_dirtied = FALSE; - - /* Initialize new null message to take over non-null message's location */ - oh->mesg[new_null_msg].type = H5O_MSG_NULL; - oh->mesg[new_null_msg].native = NULL; - oh->mesg[new_null_msg].raw = old_raw; - oh->mesg[new_null_msg].raw_size = curr_msg->raw_size; - oh->mesg[new_null_msg].chunkno = old_chunkno; - - /* Mark new null message dirty */ - oh->mesg[new_null_msg].dirty = TRUE; - curr_chk_dirtied = TRUE; - - /* Check for gap in new null message's chunk */ - if(oh->chunk[old_chunkno].gap > 0) { - /* Eliminate the gap in the chunk */ - if(H5O_eliminate_gap(oh, &curr_chk_dirtied, &oh->mesg[new_null_msg], - ((oh->chunk[old_chunkno].image + oh->chunk[old_chunkno].size) - (H5O_SIZEOF_CHKSUM_OH(oh) + oh->chunk[old_chunkno].gap)), - oh->chunk[old_chunkno].gap) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTREMOVE, FAIL, "can't eliminate gap in chunk") + /* "Retarget" 'curr_msg' pointer into newly re-allocated array of messages */ + curr_msg = &oh->mesg[u]; } /* end if */ - /* Release new null message's chunk, marking it dirty */ - if(H5O_chunk_unprotect(f, dxpl_id, curr_chk_proxy, curr_chk_dirtied) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk") - curr_chk_proxy = NULL; - curr_chk_dirtied = FALSE; + /* Get message # for new null message */ + new_null_msg = oh->nmesgs++; } /* end else */ - /* Indicate that we packed messages */ - packed_msg = TRUE; + /* Release null message's chunk, marking it dirty */ + if(H5O_chunk_unprotect(f, dxpl_id, null_chk_proxy, null_chk_dirtied) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk") + null_chk_proxy = NULL; + null_chk_dirtied = FALSE; + + /* Initialize new null message to take over non-null message's location */ + oh->mesg[new_null_msg].type = H5O_MSG_NULL; + oh->mesg[new_null_msg].native = NULL; + oh->mesg[new_null_msg].raw = old_raw; + oh->mesg[new_null_msg].raw_size = curr_msg->raw_size; + oh->mesg[new_null_msg].chunkno = old_chunkno; + + /* Mark new null message dirty */ + oh->mesg[new_null_msg].dirty = TRUE; + curr_chk_dirtied = TRUE; - /* Break out of loop */ - /* (If it's possible to move message to even earlier chunk - * we'll get it on the next pass - QAK) - */ - break; - } /* end if */ - } /* end for */ - } /* end if */ + /* Check for gap in new null message's chunk */ + if(oh->chunk[old_chunkno].gap > 0) { + /* Eliminate the gap in the chunk */ + if(H5O_eliminate_gap(oh, &curr_chk_dirtied, &oh->mesg[new_null_msg], + ((oh->chunk[old_chunkno].image + oh->chunk[old_chunkno].size) - (H5O_SIZEOF_CHKSUM_OH(oh) + oh->chunk[old_chunkno].gap)), + oh->chunk[old_chunkno].gap) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTREMOVE, FAIL, "can't eliminate gap in chunk") + } /* end if */ + + /* Release new null message's chunk, marking it dirty */ + if(H5O_chunk_unprotect(f, dxpl_id, curr_chk_proxy, curr_chk_dirtied) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk") + curr_chk_proxy = NULL; + curr_chk_dirtied = FALSE; + } /* end else */ + + /* Indicate that we packed messages */ + packed_msg = TRUE; + + /* Break out of loop */ + /* (If it's possible to move message to even earlier chunk + * we'll get it on the next pass - QAK) + */ + break; + } /* end if */ + } /* end for */ /* If we packed messages, get out of loop and start over */ /* (Don't know if this has any benefit one way or the other -QAK) */ @@ -1718,10 +1934,16 @@ H5O_move_msgs_forward(H5F_t *f, hid_t dxpl_id, H5O_t *oh) ret_value = (htri_t)did_packing; done: - if(null_chk_proxy && H5O_chunk_unprotect(f, dxpl_id, null_chk_proxy, null_chk_dirtied) < 0) - HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect null object header chunk") - if(curr_chk_proxy && H5O_chunk_unprotect(f, dxpl_id, curr_chk_proxy, curr_chk_dirtied) < 0) - HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect current object header chunk") + if(ret_value < 0) { + if(null_chk_proxy && H5O_chunk_unprotect(f, dxpl_id, null_chk_proxy, null_chk_dirtied) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect null object header chunk") + if(curr_chk_proxy && H5O_chunk_unprotect(f, dxpl_id, curr_chk_proxy, curr_chk_dirtied) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect current object header chunk") + if(cont_targ_chk_proxy && H5O_chunk_unprotect(f, dxpl_id, cont_targ_chk_proxy, FALSE) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect continuation message target object header chunk") + } /* end if */ + else + HDassert(!null_chk_proxy && !curr_chk_proxy && !cont_targ_chk_proxy); FUNC_LEAVE_NOAPI(ret_value) } /* H5O_move_msgs_forward() */ diff --git a/src/H5Oattribute.c b/src/H5Oattribute.c index 9b494a1..71f54eb 100644 --- a/src/H5Oattribute.c +++ b/src/H5Oattribute.c @@ -484,7 +484,7 @@ H5O_attr_open_by_name(const H5O_loc_t *loc, const char *name, hid_t dxpl_id) HDassert(name); /* Protect the object header to iterate over */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_ATTR, H5E_CANTPROTECT, NULL, "unable to load object header") /* Check for attribute info stored */ @@ -633,7 +633,7 @@ H5O_attr_open_by_idx(const H5O_loc_t *loc, H5_index_t idx_type, HGOTO_ERROR(H5E_ATTR, H5E_BADITER, NULL, "can't locate attribute") /* Protect the object header to iterate over */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_ATTR, H5E_CANTPROTECT, NULL, "unable to load object header") /* Find out whether it has already been opened. If it has, close the object @@ -1284,7 +1284,7 @@ H5O_attr_iterate_real(hid_t loc_id, const H5O_loc_t *loc, hid_t dxpl_id, HDassert(attr_op); /* Protect the object header to iterate over */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_ATTR, H5E_CANTPROTECT, FAIL, "unable to load object header") /* Check for attribute info stored */ @@ -1847,7 +1847,7 @@ H5O_attr_exists(const H5O_loc_t *loc, const char *name, hid_t dxpl_id) HDassert(name); /* Protect the object header to iterate over */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_ATTR, H5E_CANTPROTECT, FAIL, "unable to load object header") /* Check for attribute info stored */ @@ -2001,7 +2001,7 @@ H5O_attr_count(const H5O_loc_t *loc, hid_t dxpl_id) HDassert(loc); /* Protect the object header to iterate over */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_ATTR, H5E_CANTPROTECT, FAIL, "unable to load object header") /* Retrieve # of attributes on object */ diff --git a/src/H5Ocache.c b/src/H5Ocache.c index 3803978..d398e41 100644 --- a/src/H5Ocache.c +++ b/src/H5Ocache.c @@ -46,12 +46,6 @@ /* Local Macros */ /****************/ -/* Set the object header size to speculatively read in */ -/* (needs to be more than the object header prefix size to work at all and - * should be larger than the largest object type's default object header - * size to save the extra I/O operations) */ -#define H5O_SPEC_READ_SIZE 512 - /******************/ /* Local Typedefs */ @@ -68,28 +62,27 @@ /********************/ /* Metadata cache callbacks */ -static herr_t H5O__cache_get_load_size(const void *udata, size_t *image_len); +static herr_t H5O__cache_get_initial_load_size(void *udata, size_t *image_len); +static herr_t H5O__cache_get_final_load_size(const void *image_ptr, size_t image_len, + void *udata, size_t *actual_len); +static htri_t H5O__cache_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5O__cache_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5O__cache_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5O__cache_image_len(const void *thing, size_t *image_len); static herr_t H5O__cache_serialize(const H5F_t *f, void *image, size_t len, void *thing); +static herr_t H5O__cache_notify(H5AC_notify_action_t action, void *_thing); static herr_t H5O__cache_free_icr(void *thing); -static herr_t H5O__cache_clear(const H5F_t *f, void *thing, hbool_t about_to_destroy); -static herr_t H5O__cache_chk_get_load_size(const void *udata, size_t *image_len); +static herr_t H5O__cache_chk_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5O__cache_chk_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5O__cache_chk_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5O__cache_chk_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5O__cache_chk_image_len(const void *thing, size_t *image_len); static herr_t H5O__cache_chk_serialize(const H5F_t *f, void *image, size_t len, void *thing); +static herr_t H5O__cache_chk_notify(H5AC_notify_action_t action, void *_thing); static herr_t H5O__cache_chk_free_icr(void *thing); -static herr_t H5O__cache_chk_clear(const H5F_t *f, void *thing, hbool_t about_to_destroy); - -/* Chunk proxy routines */ -static herr_t H5O__chunk_proxy_dest(H5O_chunk_proxy_t *chunk_proxy); /* Chunk routines */ static herr_t H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, @@ -111,14 +104,15 @@ const H5AC_class_t H5AC_OHDR[1] = {{ "object header", /* Metadata client name (for debugging) */ H5FD_MEM_OHDR, /* File space memory type for client */ H5AC__CLASS_SPECULATIVE_LOAD_FLAG, /* Client class behavior flags */ - H5O__cache_get_load_size, /* 'get_load_size' callback */ + H5O__cache_get_initial_load_size, /* 'get_initial_load_size' callback */ + H5O__cache_get_final_load_size, /* 'get_final_load_size' callback */ + H5O__cache_verify_chksum, /* 'verify_chksum' callback */ H5O__cache_deserialize, /* 'deserialize' callback */ H5O__cache_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5O__cache_serialize, /* 'serialize' callback */ - NULL, /* 'notify' callback */ + H5O__cache_notify, /* 'notify' callback */ H5O__cache_free_icr, /* 'free_icr' callback */ - H5O__cache_clear, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -128,14 +122,15 @@ const H5AC_class_t H5AC_OHDR_CHK[1] = {{ "object header continuation chunk", /* Metadata client name (for debugging) */ H5FD_MEM_OHDR, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5O__cache_chk_get_load_size, /* 'get_load_size' callback */ + H5O__cache_chk_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5O__cache_chk_verify_chksum, /* 'verify_chksum' callback */ H5O__cache_chk_deserialize, /* 'deserialize' callback */ H5O__cache_chk_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5O__cache_chk_serialize, /* 'serialize' callback */ - NULL, /* 'notify' callback */ + H5O__cache_chk_notify, /* 'notify' callback */ H5O__cache_chk_free_icr, /* 'free_icr' callback */ - H5O__cache_chk_clear, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -158,14 +153,13 @@ H5FL_SEQ_DEFINE(H5O_cont_t); /* Local Variables */ /*******************/ + /*------------------------------------------------------------------------- - * Function: H5O__cache_get_load_size() + * Function: H5O__cache_get_initial_load_size() * * Purpose: Tell the metadata cache how much data to read from file in - * the first speculative read for the object header. Note that we do - * not have to be concerned about reading past the end of file, as the - * cache will clamp the read to avoid this if needed. + * the first speculative read for the object header. * * Return: Success: SUCCEED * Failure: FAIL @@ -176,63 +170,53 @@ H5FL_SEQ_DEFINE(H5O_cont_t); *------------------------------------------------------------------------- */ static herr_t -H5O__cache_get_load_size(const void H5_ATTR_UNUSED *_udata, size_t *image_len) +H5O__cache_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len) { FUNC_ENTER_STATIC_NOERR /* Check arguments */ HDassert(image_len); + /* Set the image length size */ *image_len = H5O_SPEC_READ_SIZE; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5O__cache_get_load_size() */ +} /* end H5O__cache_get_initial_load_size() */ /*------------------------------------------------------------------------- - * Function: H5O__cache_deserialize - * - * Purpose: Attempt to deserialize the object header contained in the - * supplied buffer, load the data into an instance of H5O_t, and - * return a pointer to the new instance. + * Function: H5O__cache_get_final_load_size() * - * Note that the object header is read with with a speculative read. - * If the initial read is too small, make note of this fact and return - * without error. H5C_load_entry() will note the size discrepency - * and retry the deserialize operation with the correct size read. + * Purpose: Tell the metadata cache the final size of an object header. * - * Return: Success: Pointer to in core representation - * Failure: NULL + * Return: Success: SUCCEED + * Failure: FAIL * - * Programmer: John Mainzer - * 7/28/14 + * Programmer: Quincey Koziol + * November 18, 2016 * *------------------------------------------------------------------------- */ -static void * -H5O__cache_deserialize(const void *_image, size_t len, void *_udata, - hbool_t *dirty) +static herr_t +H5O__cache_get_final_load_size(const void *_image, size_t image_len, + void *_udata, size_t *actual_len) { - H5O_t *oh = NULL; /* Object header read in */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ - const uint8_t *image = (const uint8_t *)_image; /* Pointer into buffer to decode */ - size_t prefix_size; /* Size of object header prefix */ - size_t buf_size; /* Size of prefix+chunk #0 buffer */ - void * ret_value = NULL; /* Return value */ + H5O_t *oh = NULL; /* Object header read in */ + htri_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC /* Check arguments */ HDassert(image); - HDassert(len > 0); HDassert(udata); - HDassert(udata->common.f); - HDassert(udata->common.cont_msg_info); - HDassert(dirty); + HDassert(actual_len); + HDassert(*actual_len == image_len); - /* Allocate space for the object header data structure */ + /* Allocate space for the new object header data structure */ if(NULL == (oh = H5FL_CALLOC(H5O_t))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "memory allocation failed") /* File-specific, non-stored information */ oh->sizeof_size = H5F_SIZEOF_SIZE(udata->common.f); @@ -246,13 +230,13 @@ H5O__cache_deserialize(const void *_image, size_t len, void *_udata, /* Version */ oh->version = *image++; - if(H5O_VERSION_2 != oh->version) - HGOTO_ERROR(H5E_OHDR, H5E_VERSION, NULL, "bad object header version number") + if(H5O_VERSION_2 != oh->version) + HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "bad object header version number") /* Flags */ oh->flags = *image++; - if(oh->flags & ~H5O_HDR_ALL_FLAGS) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "unknown object header status flag(s)") + if(oh->flags & ~H5O_HDR_ALL_FLAGS) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "unknown object header status flag(s)") /* Number of links to object (unless overridden by refcount message) */ oh->nlink = 1; @@ -277,9 +261,8 @@ H5O__cache_deserialize(const void *_image, size_t len, void *_udata, if(oh->flags & H5O_HDR_ATTR_STORE_PHASE_CHANGE) { UINT16DECODE(image, oh->max_compact); UINT16DECODE(image, oh->min_dense); - if(oh->max_compact < oh->min_dense) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "bad object header attribute phase change values") + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header attribute phase change values") } /* end if */ else { oh->max_compact = H5O_CRT_ATTR_MAX_COMPACT_DEF; @@ -289,32 +272,32 @@ H5O__cache_deserialize(const void *_image, size_t len, void *_udata, /* First chunk size */ switch(oh->flags & H5O_HDR_CHUNK0_SIZE) { case 0: /* 1 byte size */ - oh->chunk0_size = *image++; + udata->chunk0_size = *image++; break; case 1: /* 2 byte size */ - UINT16DECODE(image, oh->chunk0_size); + UINT16DECODE(image, udata->chunk0_size); break; case 2: /* 4 byte size */ - UINT32DECODE(image, oh->chunk0_size); + UINT32DECODE(image, udata->chunk0_size); break; case 3: /* 8 byte size */ - UINT64DECODE(image, oh->chunk0_size); + UINT64DECODE(image, udata->chunk0_size); break; default: - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "bad size for chunk 0") + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad size for chunk 0") } /* end switch */ - if(oh->chunk0_size > 0 && oh->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "bad object header chunk size") + if(udata->chunk0_size > 0 && udata->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header chunk size") } /* end if */ else { /* Version */ oh->version = *image++; if(H5O_VERSION_1 != oh->version) - HGOTO_ERROR(H5E_OHDR, H5E_VERSION, NULL, "bad object header version number") + HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "bad object header version number") /* Flags */ oh->flags = H5O_CRT_OHDR_FLAGS_DEF; @@ -336,36 +319,137 @@ H5O__cache_deserialize(const void *_image, size_t len, void *_udata, oh->min_dense = 0; /* First chunk size */ - UINT32DECODE(image, oh->chunk0_size); - - if((udata->v1_pfx_nmesgs > 0 && - oh->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) || - (udata->v1_pfx_nmesgs == 0 && oh->chunk0_size > 0)) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "bad object header chunk size") + UINT32DECODE(image, udata->chunk0_size); + if((udata->v1_pfx_nmesgs > 0 && udata->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) || + (udata->v1_pfx_nmesgs == 0 && udata->chunk0_size > 0)) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header chunk size") /* Reserved, in version 1 (for 8-byte alignment padding) */ image += 4; } /* end else */ /* Determine object header prefix length */ - prefix_size = (size_t)(image - (const uint8_t *)_image); - HDassert((size_t)prefix_size == (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh))); + HDassert((size_t)((const uint8_t *)image - (const uint8_t *)_image) == (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh))); - /* Compute the size of the buffer used */ - buf_size = oh->chunk0_size + (size_t)H5O_SIZEOF_HDR(oh); + /* Set the final size for the cache image */ + *actual_len = udata->chunk0_size + (size_t)H5O_SIZEOF_HDR(oh); - /* Check to see if the buffer provided is large enough to contain both - * the prefix and the first chunk. If it isn't, make note of the desired - * size, but otherwise do nothing. H5C_load_entry() will notice the - * discrepency, load the correct size buffer, and retry the deserialize. - */ - if(len >= buf_size) { - /* Parse the first chunk */ - if(H5O__chunk_deserialize(oh, udata->common.addr, oh->chunk0_size, (const uint8_t *)_image, &(udata->common), dirty) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't deserialize first object header chunk") + /* Save the object header for later use in 'deserialize' callback */ + udata->oh = oh; + oh = NULL; + +done: + /* Release the [possibly partially initialized] object header on errors */ + if(ret_value < 0 && oh) + if(H5O__free(oh) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTRELEASE, FAIL, "unable to destroy object header data") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O__cache_get_final_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5O__cache_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi + * Aug 2015 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5O__cache_verify_chksum(const void *_image, size_t len, void *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + HDassert(udata); + HDassert(udata->oh); + + /* There is no checksum for version 1 */ + if(udata->oh->version != H5O_VERSION_1) { + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O__cache_verify_chksum() */ + + +/*------------------------------------------------------------------------- + * Function: H5O__cache_deserialize + * + * Purpose: Attempt to deserialize the object header contained in the + * supplied buffer, load the data into an instance of H5O_t, and + * return a pointer to the new instance. + * + * Note that the object header is read with with a speculative read. + * If the initial read is too small, make note of this fact and return + * without error. H5C_load_entry() will note the size discrepency + * and retry the deserialize operation with the correct size read. + * + * Return: Success: Pointer to in core representation + * Failure: NULL + * + * Programmer: John Mainzer + * 7/28/14 + * + *------------------------------------------------------------------------- + */ +static void * +H5O__cache_deserialize(const void *_image, size_t len, void *_udata, + hbool_t *dirty) +{ + H5O_t *oh; /* Object header read in */ + H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into buffer to decode */ + void * ret_value = NULL; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check arguments */ + HDassert(image); + HDassert(len > 0); + HDassert(udata); + HDassert(udata->oh); + HDassert(udata->common.f); + HDassert(udata->common.cont_msg_info); + HDassert(dirty); + + /* Retrieve partially deserialized object header from user data */ + oh = udata->oh; + + /* Set SWMR flag, if appropriate */ + oh->swmr_write = !!(H5F_INTENT(udata->common.f) & H5F_ACC_SWMR_WRITE); + + /* Create object header proxy if doing SWMR writes */ + if(oh->swmr_write) { + /* Create virtual entry, for use as proxy */ + if(NULL == (oh->proxy = H5AC_proxy_entry_create())) + HGOTO_ERROR(H5E_OHDR, H5E_CANTCREATE, NULL, "can't create object header proxy") } /* end if */ else - HDassert(!udata->made_attempt); + oh->proxy = NULL; + + /* Parse the first chunk */ + if(H5O__chunk_deserialize(oh, udata->common.addr, udata->chunk0_size, (const uint8_t *)_image, &(udata->common), dirty) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't deserialize first object header chunk") /* Note that we've loaded the object header from the file */ udata->made_attempt = TRUE; @@ -376,7 +460,7 @@ H5O__cache_deserialize(const void *_image, size_t len, void *_udata, done: /* Release the [possibly partially initialized] object header on errors */ if(!ret_value && oh) - if(H5O_free(oh) < 0) + if(H5O__free(oh) < 0) HDONE_ERROR(H5E_OHDR, H5E_CANTRELEASE, NULL, "unable to destroy object header data") FUNC_LEAVE_NOAPI(ret_value) @@ -399,8 +483,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5O__cache_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5O__cache_image_len(const void *_thing, size_t *image_len) { const H5O_t *oh = (const H5O_t *)_thing; /* Object header to query */ @@ -413,18 +496,11 @@ H5O__cache_image_len(const void *_thing, size_t *image_len, HDassert(image_len); /* Report the object header's prefix+first chunk length */ - if(oh->chunk0_size) - *image_len = (size_t)H5O_SIZEOF_HDR(oh) + oh->chunk0_size; - else - *image_len = oh->chunk[0].size; + *image_len = oh->chunk[0].size; FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5O__cache_image_len() */ -/********************************/ -/* no H5O_cache_pre_serialize() */ -/********************************/ - /*------------------------------------------------------------------------- * Function: H5O__cache_serialize @@ -568,9 +644,89 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__cache_serialize() */ -/**********************************/ -/* no H5O_cache_notify() function */ -/**********************************/ + +/*------------------------------------------------------------------------- + * Function: H5O__cache_notify + * + * Purpose: Handle cache action notifications + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Jul 23 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O__cache_notify(H5AC_notify_action_t action, void *_thing) +{ + H5O_t *oh = (H5O_t *)_thing; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* + * Check arguments. + */ + HDassert(oh); + + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + case H5AC_NOTIFY_ACTION_AFTER_LOAD: + if(oh->swmr_write) { + /* Sanity check */ + HDassert(oh->proxy); + + /* Register the object header as a parent of the virtual entry */ + if(H5AC_proxy_entry_add_parent(oh->proxy, oh) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "can't add object header as parent of proxy") + } /* end if */ + break; + + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + { + unsigned u; /* Local index variable */ + + /* Mark messages stored with the object header (i.e. messages in chunk 0) as clean */ + for(u = 0; u < oh->nmesgs; u++) + if(oh->mesg[u].chunkno == 0) + oh->mesg[u].dirty = FALSE; +#ifndef NDEBUG + /* Reset the number of messages dirtied by decoding */ + oh->ndecode_dirtied = 0; +#endif /* NDEBUG */ + } + break; + + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + if(oh->swmr_write) { + /* Unregister the object header as a parent of the virtual entry */ + if(H5AC_proxy_entry_remove_parent(oh->proxy, oh) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "can't remove object header as parent of proxy") + } /* end if */ + break; + + default: +#ifdef NDEBUG + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "unknown action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Unknown action?!?"); +#endif /* NDEBUG */ + } /* end switch */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O__cache_notify() */ /*------------------------------------------------------------------------- @@ -604,7 +760,7 @@ H5O__cache_free_icr(void *_thing) HDassert(oh->cache_info.type == H5AC_OHDR); /* Destroy object header */ - if(H5O_free(oh) < 0) + if(H5O__free(oh) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTRELEASE, FAIL, "can't destroy object header") done: @@ -613,112 +769,79 @@ done: /*------------------------------------------------------------------------- - * Function: H5O__cache_clear - * - * Purpose: Clear all dirty bits associated with this cache entry. + * Function: H5O__cache_chk_get_initial_load_size() * - * This is ncessary as the object header cache client maintains - * its own dirty bits on individual messages. These dirty bits - * used to be cleared by the old V2 metadata cache flush callback, - * but now the metadata cache must clear them explicitly, as - * the serialize callback does not imply that the data has been - * written to disk. - * - * This callback is also necessary for the parallel case. + * Purpose: Tell the metadata cache how large the on disk image of the + * chunk proxy is, so it can load the image into a buffer for the + * deserialize call. * * Return: Success: SUCCEED * Failure: FAIL * * Programmer: John Mainzer - * 9/22/14 + * 7/28/14 * *------------------------------------------------------------------------- */ static herr_t -#ifdef H5_HAVE_PARALLEL -H5O__cache_clear(const H5F_t *f, void *_thing, hbool_t H5_ATTR_UNUSED about_to_destroy) -#else -H5O__cache_clear(const H5F_t H5_ATTR_UNUSED *f, void *_thing, hbool_t H5_ATTR_UNUSED about_to_destroy) -#endif /* H5_HAVE_PARALLEL */ -{ - H5O_t *oh = (H5O_t *)_thing; /* Object header to reset */ - unsigned u; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ +H5O__cache_chk_get_initial_load_size(void *_udata, size_t *image_len) +{ + const H5O_chk_cache_ud_t *udata = (const H5O_chk_cache_ud_t *)_udata; /* User data for callback */ -#ifdef H5_HAVE_PARALLEL - FUNC_ENTER_STATIC -#else FUNC_ENTER_STATIC_NOERR -#endif /* H5_HAVE_PARALLEL */ /* Check arguments */ - HDassert(oh); - HDassert(oh->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); - HDassert(oh->cache_info.type == H5AC_OHDR); - -#ifdef H5_HAVE_PARALLEL - if((oh->nchunks > 0) && (!about_to_destroy)) { - /* Scan through chunk 0 (the chunk stored contiguously with this - * object header) and cause it to update its image of all entries - * currently marked dirty. Must do this in the parallel case, as - * it is possible that this processor may clear this object header - * several times before flushing it -- thus causing undefined - * sections of the image to be written to disk overwriting valid data. - */ - if(H5O__chunk_serialize(f, oh, 0) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTSERIALIZE, FAIL, "unable to serialize object header chunk") - } /* end if */ -#endif /* H5_HAVE_PARALLEL */ - - /* Mark messages stored with the object header (i.e. messages in chunk 0) as clean */ - for(u = 0; u < oh->nmesgs; u++) - if(oh->mesg[u].chunkno == 0) - oh->mesg[u].dirty = FALSE; + HDassert(udata); + HDassert(udata->oh); + HDassert(image_len); -#ifndef NDEBUG - /* Reset the number of messages dirtied by decoding */ - oh->ndecode_dirtied = 0; -#endif /* NDEBUG */ + /* Set the image length size */ + *image_len = udata->size; -#ifdef H5_HAVE_PARALLEL -done: -#endif /* H5_HAVE_PARALLEL */ - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5O__cache_clear() */ + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5O__cache_chk_get_initial_load_size() */ /*------------------------------------------------------------------------- - * Function: H5O__cache_chk_get_load_size() + * Function: H5B2__cache_chk_verify_chksum * - * Purpose: Tell the metadata cache how large the on disk image of the - * chunk proxy is, so it can load the image into a buffer for the - * deserialize call. In this case, we simply look up the size in - * the user data, and return it in *image_len, + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. * - * Return: Success: SUCCEED - * Failure: FAIL + * Return: Success: TRUE/FALSE + * Failure: Negative * - * Programmer: John Mainzer - * 7/28/14 + * Programmer: Vailin Choi + * Aug 2015 * *------------------------------------------------------------------------- */ -static herr_t -H5O__cache_chk_get_load_size(const void *_udata, size_t *image_len) +static htri_t +H5O__cache_chk_verify_chksum(const void *_image, size_t len, void *_udata) { - const H5O_chk_cache_ud_t *udata = (const H5O_chk_cache_ud_t *)_udata; /* User data for callback */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5O_chk_cache_ud_t *udata = (H5O_chk_cache_ud_t *)_udata; /* User data for callback */ + htri_t ret_value = TRUE; /* Return value */ FUNC_ENTER_STATIC_NOERR /* Check arguments */ - HDassert(udata); - HDassert(udata->oh); - HDassert(image_len); + HDassert(image); - *image_len = udata->size; + /* There is no checksum for version 1 */ + if(udata->oh->version != H5O_VERSION_1) { + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5O__cache_chk_get_load_size() */ + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O__cache_chk_verify_chksum() */ /*------------------------------------------------------------------------- @@ -768,32 +891,35 @@ H5O__cache_chk_deserialize(const void *image, size_t len, void *_udata, if(H5O__chunk_deserialize(udata->oh, udata->common.addr, udata->size, (const uint8_t *)image, &(udata->common), dirty) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't deserialize object header chunk") - /* Set the fields for the chunk proxy */ - chk_proxy->oh = udata->oh; + /* Set the chunk number for the chunk proxy */ H5_CHECKED_ASSIGN(chk_proxy->chunkno, unsigned, udata->oh->nchunks - 1, size_t); } /* end if */ else { /* Sanity check */ HDassert(udata->chunkno < udata->oh->nchunks); - /* Set the fields for the chunk proxy */ - chk_proxy->oh = udata->oh; + /* Set the chunk number for the chunk proxy */ chk_proxy->chunkno = udata->chunkno; /* Sanity check that the chunk representation we have in memory is * the same as the one being brought in from disk. */ - HDassert(0 == HDmemcmp(image, chk_proxy->oh->chunk[chk_proxy->chunkno].image, chk_proxy->oh->chunk[chk_proxy->chunkno].size)); + HDassert(0 == HDmemcmp(image, udata->oh->chunk[chk_proxy->chunkno].image, udata->oh->chunk[chk_proxy->chunkno].size)); } /* end else */ /* Increment reference count of object header */ if(H5O_inc_rc(udata->oh) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINC, NULL, "can't increment reference count on object header") + chk_proxy->oh = udata->oh; /* Set return value */ ret_value = chk_proxy; done: + if(NULL == ret_value) + if(chk_proxy && H5O__chunk_dest(chk_proxy) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTRELEASE, NULL, "unable to destroy object header chunk") + FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__cache_chk_deserialize() */ @@ -813,8 +939,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5O__cache_chk_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5O__cache_chk_image_len(const void *_thing, size_t *image_len) { const H5O_chunk_proxy_t * chk_proxy = (const H5O_chunk_proxy_t *)_thing; /* Chunk proxy to query */ @@ -832,10 +957,6 @@ H5O__cache_chk_image_len(const void *_thing, size_t *image_len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5O__cache_chk_image_len() */ -/************************************/ -/* no H5O_cache_chk_pre_serialize() */ -/************************************/ - /*------------------------------------------------------------------------- * Function: H5O__cache_chk_serialize @@ -883,116 +1004,178 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__cache_chk_serialize() */ -/**************************************/ -/* no H5O_cache_chk_notify() function */ -/**************************************/ - /*------------------------------------------------------------------------- - * Function: H5O__cache_chk_free_icr + * Function: H5O__cache_chk_notify * - * Purpose: Free the in core memory associated with the supplied object - * header continuation chunk. + * Purpose: Handle cache action notifications * - * Note: The metadata cache sets the object's cache_info.magic to - * H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC before calling a free_icr - * callback (checked in assert). + * Return: Non-negative on success/Negative on failure * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: John Mainzer - * 7/28/14 + * Programmer: Neil Fortner + * Mar 20 2012 * *------------------------------------------------------------------------- */ static herr_t -H5O__cache_chk_free_icr(void *_thing) +H5O__cache_chk_notify(H5AC_notify_action_t action, void *_thing) { - H5O_chunk_proxy_t * chk_proxy = (H5O_chunk_proxy_t *)_thing; /* Object header chunk proxy to release */ - herr_t ret_value = SUCCEED; /* Return value */ + H5O_chunk_proxy_t *chk_proxy = (H5O_chunk_proxy_t *)_thing; + H5O_chunk_proxy_t *cont_chk_proxy = NULL; /* Proxy for chunk containing continuation message that points to this chunk, if not chunk 0 */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC - /* Check arguments */ + /* + * Check arguments. + */ HDassert(chk_proxy); - HDassert(chk_proxy->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC); - HDassert(chk_proxy->cache_info.type == H5AC_OHDR_CHK); + HDassert(chk_proxy->oh); - /* Destroy object header chunk proxy */ - if(H5O__chunk_proxy_dest(chk_proxy) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTRELEASE, FAIL, "unable to destroy object header chunk proxy") + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + case H5AC_NOTIFY_ACTION_AFTER_LOAD: + if(chk_proxy->oh->swmr_write) { + /* Add flush dependency on chunk parent */ + { + void *parent; /* Chunk containing continuation message that points to this chunk */ + + /* Determine the parent of the chunk */ + if(chk_proxy->cont_chunkno == 0) + parent = chk_proxy->oh; + else { + if(NULL == (cont_chk_proxy = H5O_chunk_protect(chk_proxy->f, H5AC_ind_read_dxpl_id, chk_proxy->oh, chk_proxy->cont_chunkno))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header chunk") + parent = cont_chk_proxy; + } /* end else */ + + /* Sanity checks */ + HDassert(parent); + HDassert(((H5C_cache_entry_t *)parent)->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + HDassert(((H5C_cache_entry_t *)parent)->type); + HDassert((((H5C_cache_entry_t *)(parent))->type->id == H5AC_OHDR_ID) + || (((H5C_cache_entry_t *)(parent))->type->id == H5AC_OHDR_CHK_ID)); + + /* Add flush dependency from chunk containing the continuation message + * that points to this chunk (either oh or another chunk proxy object) + */ + if(H5AC_create_flush_dependency(parent, chk_proxy) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTDEPEND, FAIL, "unable to create flush dependency") + + /* Make note of the address and pointer of the flush dependency + * parent so we can take the dependency down on eviction. + */ + chk_proxy->parent = parent; + } + + /* Add flush dependency on object header proxy, if proxy exists */ + { + /* Sanity check */ + HDassert(chk_proxy->oh->proxy); + + /* Register the object header chunk as a parent of the virtual entry */ + if(H5AC_proxy_entry_add_parent(chk_proxy->oh->proxy, chk_proxy) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "can't add object header chunk as parent of proxy") + } + } /* end if */ + break; + + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + { + unsigned u; /* Local index variable */ + + /* Mark messages in chunk as clean */ + for(u = 0; u < chk_proxy->oh->nmesgs; u++) + if(chk_proxy->oh->mesg[u].chunkno == chk_proxy->chunkno) + chk_proxy->oh->mesg[u].dirty = FALSE; + } + break; + + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + if(chk_proxy->oh->swmr_write) { + /* Remove flush dependency on parent object header chunk */ + { + /* Sanity checks */ + HDassert(chk_proxy->parent != NULL); + HDassert(((H5C_cache_entry_t *)(chk_proxy->parent))->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + HDassert(((H5C_cache_entry_t *)(chk_proxy->parent))->type); + HDassert((((H5C_cache_entry_t *)(chk_proxy->parent))->type->id == H5AC_OHDR_ID) || (((H5C_cache_entry_t *)(chk_proxy->parent))->type->id == H5AC_OHDR_CHK_ID)); + + if(H5AC_destroy_flush_dependency(chk_proxy->parent, chk_proxy) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTUNDEPEND, FAIL, "unable to destroy flush dependency") + } + + /* Unregister the object header chunk as a parent of the virtual entry */ + if(H5AC_proxy_entry_remove_parent(chk_proxy->oh->proxy, chk_proxy) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "can't remove object header chunk as parent of proxy") + } /* end if */ + break; + + default: +#ifdef NDEBUG + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "unknown action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Unknown action?!?"); +#endif /* NDEBUG */ + } /* end switch */ done: + if(cont_chk_proxy) + if(H5O_chunk_unprotect(chk_proxy->f, H5AC_ind_read_dxpl_id, cont_chk_proxy, FALSE) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk") + FUNC_LEAVE_NOAPI(ret_value) -} /* end H5O__cache_chk_free_icr() */ +} /* end H5O__cache_chk_notify() */ /*------------------------------------------------------------------------- - * Function: H5O__cache_chk_clear - * - * Purpose: Clear all dirty bits associated with this cache entry. + * Function: H5O__cache_chk_free_icr * - * This is ncessary as the object header cache client maintains - * its own dirty bits on individual messages. These dirty bits - * used to be cleared by the old V2 metadata cache flush callback, - * but now the metadata cache must clear them explicitly, as - * the serialize callback does not imply that the data has been - * written to disk. + * Purpose: Free the in core memory associated with the supplied object + * header continuation chunk. * - * This callback is also necessary for the parallel case. + * Note: The metadata cache sets the object's cache_info.magic to + * H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC before calling a free_icr + * callback (checked in assert). * * Return: Success: SUCCEED * Failure: FAIL * * Programmer: John Mainzer - * 9/22/14 + * 7/28/14 * *------------------------------------------------------------------------- */ static herr_t -#ifdef H5_HAVE_PARALLEL -H5O__cache_chk_clear(const H5F_t *f, void *_thing, hbool_t about_to_destroy) -#else -H5O__cache_chk_clear(const H5F_t H5_ATTR_UNUSED *f, void *_thing, hbool_t H5_ATTR_UNUSED about_to_destroy) -#endif /* H5_HAVE_PARALLEL */ -{ - H5O_chunk_proxy_t *chk_proxy = (H5O_chunk_proxy_t *)_thing; /* Object header chunk to reset */ - H5O_t *oh; /* Object header for chunk */ - unsigned u; /* Local index variable */ +H5O__cache_chk_free_icr(void *_thing) +{ + H5O_chunk_proxy_t * chk_proxy = (H5O_chunk_proxy_t *)_thing; /* Object header chunk proxy to release */ herr_t ret_value = SUCCEED; /* Return value */ -#ifdef H5_HAVE_PARALLEL FUNC_ENTER_STATIC -#else - FUNC_ENTER_STATIC_NOERR -#endif /* H5_HAVE_PARALLEL */ /* Check arguments */ HDassert(chk_proxy); - HDassert(chk_proxy->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + HDassert(chk_proxy->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC); HDassert(chk_proxy->cache_info.type == H5AC_OHDR_CHK); - oh = chk_proxy->oh; - HDassert(oh); - HDassert(oh->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); - HDassert(oh->cache_info.type == H5AC_OHDR); -#ifdef H5_HAVE_PARALLEL - if((chk_proxy->oh->cache_info.is_dirty) && (!about_to_destroy)) - if(H5O__chunk_serialize(f, chk_proxy->oh, chk_proxy->chunkno) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTSERIALIZE, FAIL, "unable to serialize object header chunk") -#endif /* H5_HAVE_PARALLEL */ - - /* Mark messages in chunk as clean */ - for(u = 0; u < chk_proxy->oh->nmesgs; u++) - if(chk_proxy->oh->mesg[u].chunkno == chk_proxy->chunkno) - chk_proxy->oh->mesg[u].dirty = FALSE; + /* Destroy object header chunk proxy */ + if(H5O__chunk_dest(chk_proxy) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTRELEASE, FAIL, "unable to destroy object header chunk proxy") -#ifdef H5_HAVE_PARALLEL done: -#endif /* H5_HAVE_PARALLEL */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5O__cache_chk_clear() */ +} /* end H5O__cache_chk_free_icr() */ /*------------------------------------------------------------------------- @@ -1064,12 +1247,12 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image { const uint8_t *chunk_image; /* Pointer into buffer to decode */ uint8_t *eom_ptr; /* Pointer to end of messages for a chunk */ - size_t curmesg; /* Current message being decoded in object header */ unsigned merged_null_msgs = 0; /* Number of null messages merged together */ unsigned chunkno; /* Current chunk's index */ #ifndef NDEBUG unsigned nullcnt; /* Count of null messages (for sanity checking gaps in chunks) */ #endif /* NDEBUG */ + hbool_t mesgs_modified = FALSE; /* Whether any messages were modified when the object header was deserialized */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -1095,17 +1278,15 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image /* Init the chunk data info */ chunkno = (unsigned)oh->nchunks++; oh->chunk[chunkno].gap = 0; - if(chunkno == 0) { + oh->chunk[chunkno].addr = addr; + if(chunkno == 0) /* First chunk's 'image' includes room for the object header prefix */ - oh->chunk[0].addr = addr; oh->chunk[0].size = len + (size_t)H5O_SIZEOF_HDR(oh); - } /* end if */ - else { - oh->chunk[chunkno].addr = addr; + else oh->chunk[chunkno].size = len; - } /* end else */ if(NULL == (oh->chunk[chunkno].image = H5FL_BLK_MALLOC(chunk_image, oh->chunk[chunkno].size))) HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "memory allocation failed") + oh->chunk[chunkno].chunk_proxy = NULL; /* Copy disk image into chunk's image */ HDmemcpy(oh->chunk[chunkno].image, image, oh->chunk[chunkno].size); @@ -1125,16 +1306,12 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image chunk_image += H5_SIZEOF_MAGIC; } /* end if */ - /* Save # of messages already inspected */ - curmesg = oh->nmesgs; - /* Decode messages from this chunk */ eom_ptr = oh->chunk[chunkno].image + (oh->chunk[chunkno].size - H5O_SIZEOF_CHKSUM_OH(oh)); #ifndef NDEBUG nullcnt = 0; #endif /* NDEBUG */ while(chunk_image < eom_ptr) { - size_t mesgno; /* Current message to operate on */ size_t mesg_size; /* Size of message read in */ unsigned id; /* ID (type) of current message */ uint8_t flags; /* Flags for current message */ @@ -1193,39 +1370,45 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image H5O_NULL_ID == id && oh->nmesgs > 0 && H5O_NULL_ID == oh->mesg[oh->nmesgs - 1].type->id && oh->mesg[oh->nmesgs - 1].chunkno == chunkno) { + size_t mesgno; /* Current message to operate on */ /* Combine adjacent null messages */ mesgno = oh->nmesgs - 1; oh->mesg[mesgno].raw_size += (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + mesg_size; oh->mesg[mesgno].dirty = TRUE; merged_null_msgs++; - udata->merged_null_msgs++; } /* end if */ else { + H5O_mesg_t *mesg; /* Pointer to new message */ + unsigned ioflags = 0; /* Flags for decode routine */ + /* Check if we need to extend message table to hold the new message */ if(oh->nmesgs >= oh->alloc_nmesgs) if(H5O_alloc_msgs(oh, (size_t)1) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "can't allocate more space for messages") - /* Get index for message */ - mesgno = oh->nmesgs++; + /* Get pointer to message to set up */ + mesg = &oh->mesg[oh->nmesgs]; + + /* Increment # of messages */ + oh->nmesgs++; /* Initialize information about message */ - oh->mesg[mesgno].dirty = FALSE; - oh->mesg[mesgno].flags = flags; - oh->mesg[mesgno].crt_idx = crt_idx; - oh->mesg[mesgno].native = NULL; - oh->mesg[mesgno].raw = (uint8_t *)chunk_image; /* Casting away const OK - QAK */ - oh->mesg[mesgno].raw_size = mesg_size; - oh->mesg[mesgno].chunkno = chunkno; + mesg->dirty = FALSE; + mesg->flags = flags; + mesg->crt_idx = crt_idx; + mesg->native = NULL; + mesg->raw = (uint8_t *)chunk_image; /* Casting away const OK - QAK */ + mesg->raw_size = mesg_size; + mesg->chunkno = chunkno; /* Point unknown messages at 'unknown' message class */ /* (Usually from future versions of the library) */ if(id >= H5O_UNKNOWN_ID || #ifdef H5O_ENABLE_BOGUS - id == H5O_BOGUS_VALID_ID || + id == H5O_BOGUS_VALID_ID || #endif - NULL == H5O_msg_class_g[id]) { + NULL == H5O_msg_class_g[id]) { H5O_unknown_t *unknown; /* Pointer to "unknown" message info */ @@ -1237,10 +1420,10 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image *unknown = id; /* Save 'native' form of unknown message */ - oh->mesg[mesgno].native = unknown; + mesg->native = unknown; /* Set message to "unknown" class */ - oh->mesg[mesgno].type = H5O_msg_class_g[H5O_UNKNOWN_ID]; + mesg->type = H5O_msg_class_g[H5O_UNKNOWN_ID]; /* Check for "fail if unknown" message flags */ if(((udata->file_intent & H5F_ACC_RDWR) && @@ -1263,17 +1446,66 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image * the metadata cache in some other "weird" way, like * using H5Ocopy() - QAK */ - oh->mesg[mesgno].flags |= H5O_MSG_FLAG_WAS_UNKNOWN; + mesg->flags |= H5O_MSG_FLAG_WAS_UNKNOWN; /* Mark the message and chunk as dirty */ - oh->mesg[mesgno].dirty = TRUE; - udata->mesgs_modified = TRUE; - *dirty = TRUE; + mesg->dirty = TRUE; + mesgs_modified = TRUE; } /* end if */ } /* end if */ else /* Set message class for "known" messages */ - oh->mesg[mesgno].type = H5O_msg_class_g[id]; + mesg->type = H5O_msg_class_g[id]; + + /* Do some inspection/interpretation of new messages from this chunk */ + /* (detect continuation messages, ref. count messages, etc.) */ + + /* Check if message is a continuation message */ + if(H5O_CONT_ID == id) { + H5O_cont_t *cont; + + /* Decode continuation message */ + cont = (H5O_cont_t *)(H5O_MSG_CONT->decode)(udata->f, udata->dxpl_id, NULL, 0, &ioflags, mesg->raw); + H5_CHECKED_ASSIGN(cont->chunkno, unsigned, udata->cont_msg_info->nmsgs + 1, size_t); /* the next continuation message/chunk */ + + /* Save 'native' form of continuation message */ + mesg->native = cont; + + /* Add to continuation messages left to interpret */ + if(H5O__add_cont_msg(udata->cont_msg_info, cont) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "can't add continuation message") + } /* end if */ + /* Check if message is a ref. count message */ + else if(H5O_REFCOUNT_ID == id) { + H5O_refcount_t *refcount; + + /* Decode ref. count message */ + HDassert(oh->version > H5O_VERSION_1); + refcount = (H5O_refcount_t *)(H5O_MSG_REFCOUNT->decode)(udata->f, udata->dxpl_id, NULL, 0, &ioflags, mesg->raw); + + /* Save 'native' form of ref. count message */ + mesg->native = refcount; + + /* Set object header values */ + oh->has_refcount_msg = TRUE; + oh->nlink = *refcount; + } /* end if */ + /* Check if message is a link message */ + else if(H5O_LINK_ID == id) { + /* Increment the count of link messages */ + oh->link_msgs_seen++; + } /* end if */ + /* Check if message is an attribute message */ + else if(H5O_ATTR_ID == id) { + /* Increment the count of attribute messages */ + oh->attr_msgs_seen++; + } /* end if */ + + /* Mark the message & chunk as dirty if the message was changed by decoding */ + if((ioflags & H5O_DECODEIO_DIRTY) && (udata->file_intent & H5F_ACC_RDWR)) { + mesg->dirty = TRUE; + mesgs_modified = TRUE; + } /* end if */ } /* end else */ /* Advance decode pointer past message */ @@ -1298,89 +1530,23 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image /* Check for correct checksum on chunks, in later versions of the format */ if(oh->version > H5O_VERSION_1) { uint32_t stored_chksum; /* Checksum from file */ - uint32_t computed_chksum; /* Checksum computed in memory */ + + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(chunk_image, stored_chksum); - - /* Compute checksum on chunk */ - computed_chksum = H5_checksum_metadata(oh->chunk[chunkno].image, (oh->chunk[chunkno].size - H5O_SIZEOF_CHKSUM), 0); - - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "incorrect metadata checksum for object header chunk") } /* end if */ /* Sanity check */ HDassert(chunk_image == oh->chunk[chunkno].image + oh->chunk[chunkno].size); - /* Do some inspection/interpretation of new messages from this chunk */ - /* (detect continuation messages, ref. count messages, etc.) */ - while(curmesg < oh->nmesgs) { - /* Check if next message to examine is a continuation message */ - if(H5O_CONT_ID == oh->mesg[curmesg].type->id) { - H5O_cont_t *cont; - unsigned ioflags = 0; /* Flags for decode routine */ - - /* Decode continuation message */ - cont = (H5O_cont_t *)(H5O_MSG_CONT->decode)(udata->f, udata->dxpl_id, NULL, 0, &ioflags, oh->mesg[curmesg].raw); - H5_CHECKED_ASSIGN(cont->chunkno, unsigned, udata->cont_msg_info->nmsgs + 1, size_t); /* the next continuation message/chunk */ - - /* Save 'native' form of continuation message */ - oh->mesg[curmesg].native = cont; - - /* Add to continuation messages left to interpret */ - if(H5O__add_cont_msg(udata->cont_msg_info, cont) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "can't add continuation message") - - /* Mark the message & chunk as dirty if the message was changed by decoding */ - if((ioflags & H5O_DECODEIO_DIRTY) && (udata->file_intent & H5F_ACC_RDWR)) { - oh->mesg[curmesg].dirty = TRUE; - udata->mesgs_modified = TRUE; - *dirty = TRUE; - } /* end if */ - } /* end if */ - /* Check if next message to examine is a ref. count message */ - else if(H5O_REFCOUNT_ID == oh->mesg[curmesg].type->id) { - H5O_refcount_t *refcount; - unsigned ioflags = 0; /* Flags for decode routine */ - - /* Decode ref. count message */ - HDassert(oh->version > H5O_VERSION_1); - refcount = (H5O_refcount_t *)(H5O_MSG_REFCOUNT->decode)(udata->f, udata->dxpl_id, NULL, 0, &ioflags, oh->mesg[curmesg].raw); - - /* Save 'native' form of ref. count message */ - oh->mesg[curmesg].native = refcount; - - /* Set object header values */ - oh->has_refcount_msg = TRUE; - oh->nlink = *refcount; - - /* Mark the message & chunk as dirty if the message was changed by decoding */ - if((ioflags & H5O_DECODEIO_DIRTY) && (udata->file_intent & H5F_ACC_RDWR)) { - oh->mesg[curmesg].dirty = TRUE; - udata->mesgs_modified = TRUE; - *dirty = TRUE; - } /* end if */ - } /* end if */ - /* Check if next message to examine is a link message */ - else if(H5O_LINK_ID == oh->mesg[curmesg].type->id) { - /* Increment the count of link messages */ - oh->link_msgs_seen++; - } /* end if */ - /* Check if next message to examine is an attribute message */ - else if(H5O_ATTR_ID == oh->mesg[curmesg].type->id) { - /* Increment the count of attribute messages */ - oh->attr_msgs_seen++; - } /* end if */ - - /* Advance to next message */ - curmesg++; - } /* end while */ + /* Mark the chunk dirty if we've modified messages */ + if(mesgs_modified) + *dirty = TRUE; /* Mark the chunk dirty if we've merged null messages */ - if(merged_null_msgs) { - udata->mesgs_modified = TRUE; + if(merged_null_msgs > 0) { + udata->merged_null_msgs += merged_null_msgs; *dirty = TRUE; } /* end if */ @@ -1453,39 +1619,3 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5O__chunk_serialize() */ - -/*------------------------------------------------------------------------- - * Function: H5O__chunk_proxy_dest - * - * Purpose: Destroy a chunk proxy object - * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: Quincey Koziol - * koziol@hdfgroup.org - * July 13, 2008 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5O__chunk_proxy_dest(H5O_chunk_proxy_t *chk_proxy) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Check arguments */ - HDassert(chk_proxy); - - /* Decrement reference count of object header */ - if(chk_proxy->oh && H5O_dec_rc(chk_proxy->oh) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTDEC, FAIL, "can't decrement reference count on object header") - - /* Release the chunk proxy object */ - chk_proxy = H5FL_FREE(H5O_chunk_proxy_t, chk_proxy); - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5O__chunk_proxy_dest() */ - diff --git a/src/H5Ochunk.c b/src/H5Ochunk.c index 18561b3..50be171 100644 --- a/src/H5Ochunk.c +++ b/src/H5Ochunk.c @@ -93,7 +93,8 @@ H5FL_DEFINE(H5O_chunk_proxy_t); *------------------------------------------------------------------------- */ herr_t -H5O_chunk_add(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned idx) +H5O_chunk_add(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned idx, + unsigned cont_chunkno) { H5O_chunk_proxy_t *chk_proxy = NULL; /* Proxy for chunk, to mark it dirty in the cache */ herr_t ret_value = SUCCEED; /* Return value */ @@ -110,14 +111,16 @@ H5O_chunk_add(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned idx) if(NULL == (chk_proxy = H5FL_CALLOC(H5O_chunk_proxy_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") - /* Set the values in the chunk proxy */ - chk_proxy->oh = oh; - chk_proxy->chunkno = idx; - /* Increment reference count on object header */ if(H5O_inc_rc(oh) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINC, FAIL, "can't increment reference count on object header") + /* Set the values in the chunk proxy */ + chk_proxy->f = f; + chk_proxy->oh = oh; + chk_proxy->chunkno = idx; + chk_proxy->cont_chunkno = cont_chunkno; + /* Insert the chunk proxy into the cache */ if(H5AC_insert_entry(f, dxpl_id, H5AC_OHDR_CHK, oh->chunk[idx].addr, chk_proxy, H5AC__NO_FLAGS_SET) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINSERT, FAIL, "unable to cache object header chunk") @@ -126,8 +129,8 @@ H5O_chunk_add(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned idx) done: if(ret_value < 0) - if(chk_proxy) - chk_proxy = H5FL_FREE(H5O_chunk_proxy_t, chk_proxy); + if(chk_proxy && H5O__chunk_dest(chk_proxy) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTRELEASE, FAIL, "unable to destroy object header chunk") FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) } /* end H5O_chunk_add() */ @@ -172,6 +175,7 @@ H5O_chunk_protect(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned idx) HGOTO_ERROR(H5E_OHDR, H5E_CANTINC, NULL, "can't increment reference count on object header") /* Set chunk proxy fields */ + chk_proxy->f = f; chk_proxy->oh = oh; chk_proxy->chunkno = idx; } /* end if */ @@ -200,8 +204,8 @@ H5O_chunk_protect(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned idx) done: /* Cleanup on error */ if(!ret_value) - if(0 == idx && chk_proxy) - chk_proxy = H5FL_FREE(H5O_chunk_proxy_t, chk_proxy); + if(0 == idx && chk_proxy && H5O__chunk_dest(chk_proxy) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTRELEASE, NULL, "unable to destroy object header chunk") FUNC_LEAVE_NOAPI_TAG(ret_value, NULL) } /* end H5O_chunk_protect() */ @@ -399,7 +403,9 @@ H5O_chunk_delete(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned idx) HDassert(chk_proxy->oh == oh); HDassert(chk_proxy->chunkno == idx); - cache_flags |= H5AC__DIRTIED_FLAG | H5AC__FREE_FILE_SPACE_FLAG; + /* Only free file space if not doing SWMR writes */ + if(!oh->swmr_write) + cache_flags |= H5AC__DIRTIED_FLAG | H5AC__FREE_FILE_SPACE_FLAG; /* Release the chunk proxy from the cache, marking it deleted */ if(H5AC_unprotect(f, dxpl_id, H5AC_OHDR_CHK, oh->chunk[idx].addr, chk_proxy, cache_flags) < 0) @@ -409,3 +415,39 @@ done: FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) } /* end H5O_chunk_delete() */ + +/*------------------------------------------------------------------------- + * Function: H5O__chunk_dest + * + * Purpose: Destroy a chunk proxy object + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * July 13, 2008 + * + *------------------------------------------------------------------------- + */ +herr_t +H5O__chunk_dest(H5O_chunk_proxy_t *chk_proxy) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check arguments */ + HDassert(chk_proxy); + + /* Decrement reference count of object header */ + if(chk_proxy->oh && H5O_dec_rc(chk_proxy->oh) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTDEC, FAIL, "can't decrement reference count on object header") + + /* Release the chunk proxy object */ + chk_proxy = H5FL_FREE(H5O_chunk_proxy_t, chk_proxy); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5O__chunk_dest() */ + diff --git a/src/H5Ocopy.c b/src/H5Ocopy.c index 0ee8cfd..ddf375c 100644 --- a/src/H5Ocopy.c +++ b/src/H5Ocopy.c @@ -277,7 +277,7 @@ H5Ocopy(hid_t src_loc_id, const char *src_name, hid_t dst_loc_id, done: if(loc_found && H5G_loc_free(&src_loc) < 0) HDONE_ERROR(H5E_OHDR, H5E_CANTRELEASE, FAIL, "can't free location") - if(obj_open && H5O_close(&src_oloc) < 0) + if(obj_open && H5O_close(&src_oloc, NULL) < 0) HDONE_ERROR(H5E_OHDR, H5E_CLOSEERROR, FAIL, "unable to release object header") FUNC_LEAVE_API(ret_value) @@ -381,7 +381,7 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out*/, } /* end if */ /* Get source object header */ - if(NULL == (oh_src = H5O_protect(oloc_src, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh_src = H5O_protect(oloc_src, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header") /* Retrieve user data for particular type of object to copy */ @@ -450,6 +450,7 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out*/, oh_dst->attr_msgs_seen = oh_src->attr_msgs_seen; oh_dst->sizeof_size = H5F_SIZEOF_SIZE(oloc_dst->file); oh_dst->sizeof_addr = H5F_SIZEOF_ADDR(oloc_dst->file); + oh_dst->swmr_write = !!(H5F_INTENT(oloc_dst->file) & H5F_ACC_SWMR_WRITE); /* Copy time fields */ oh_dst->atime = oh_src->atime; @@ -461,6 +462,15 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out*/, oh_dst->max_compact = oh_src->max_compact; oh_dst->min_dense = oh_src->min_dense; + /* Create object header proxy if doing SWMR writes */ + if(oh_dst->swmr_write) { + /* Create virtual entry, for use as proxy */ + if(NULL == (oh_dst->proxy = H5AC_proxy_entry_create())) + HGOTO_ERROR(H5E_OHDR, H5E_CANTCREATE, FAIL, "can't create object header proxy") + } /* end if */ + else + oh_dst->proxy = NULL; + /* Initialize size of chunk array. Start off with zero chunks so this field * is consistent with the current state of the chunk array. This is * important if an error occurs. @@ -858,6 +868,10 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out*/, oh_dst->nlink += (unsigned)addr_map->inc_ref_count; } /* end if */ + /* Retag all copied metadata to apply the destination object's tag */ + if(H5AC_retag_copied_metadata(oloc_dst->file, oloc_dst->addr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "unable to re-tag metadata entries") + /* Set metadata tag for destination object's object header */ H5_BEGIN_TAG(dxpl_id, oloc_dst->addr, FAIL); @@ -870,10 +884,6 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out*/, /* Reset metadat tag */ H5_END_TAG(FAIL); - /* Retag all copied metadata to apply the destination object's tag */ - if(H5AC_retag_copied_metadata(oloc_dst->file, oloc_dst->addr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTTAG, FAIL, "unable to re-tag metadata entries") - /* Set obj_type and udata, if requested */ if(obj_type) { HDassert(udata); @@ -892,7 +902,7 @@ done: /* Free destination object header on failure */ if(ret_value < 0 && oh_dst && !inserted) { - if(H5O_free(oh_dst) < 0) + if(H5O__free(oh_dst) < 0) HDONE_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to destroy object header data") if(H5O_loc_reset(oloc_dst) < 0) HDONE_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to destroy object header data") diff --git a/src/H5Odbg.c b/src/H5Odbg.c index d9d4b8b..827f40c 100644 --- a/src/H5Odbg.c +++ b/src/H5Odbg.c @@ -578,7 +578,7 @@ H5O_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE *stream, int indent, int f loc.addr = addr; loc.holding_file = FALSE; - if(NULL == (oh = H5O_protect(&loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(&loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header") /* debug */ diff --git a/src/H5Oflush.c b/src/H5Oflush.c index 4fc9c8e..e399c6c 100644 --- a/src/H5Oflush.c +++ b/src/H5Oflush.c @@ -162,7 +162,7 @@ H5O_oh_tag(const H5O_loc_t *oloc, hid_t dxpl_id, haddr_t *tag) HDassert(oloc); /* Get object header for object */ - if(NULL == (oh = H5O_protect(oloc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(oloc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to protect object's object header") /* Get object header's address (i.e. the tag value for this object) */ @@ -262,7 +262,7 @@ H5O_refresh_metadata(hid_t oid, H5O_loc_t oloc, hid_t dxpl_id) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, FAIL, "unable to refresh object") /* Re-open the object, re-fetching its metadata */ - if((H5O_refresh_metadata_reopen(oid, &obj_loc, dxpl_id)) < 0) + if((H5O_refresh_metadata_reopen(oid, &obj_loc, dxpl_id, FALSE)) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, FAIL, "unable to refresh object") } /* end if */ @@ -360,7 +360,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5O_refresh_metadata_reopen(hid_t oid, H5G_loc_t *obj_loc, hid_t dxpl_id) +H5O_refresh_metadata_reopen(hid_t oid, H5G_loc_t *obj_loc, hid_t dxpl_id, hbool_t start_swmr) { void *object = NULL; /* Dataset for this operation */ H5I_type_t type; /* Type of object for the ID */ @@ -388,8 +388,9 @@ H5O_refresh_metadata_reopen(hid_t oid, H5G_loc_t *obj_loc, hid_t dxpl_id) /* Re-open the dataset */ if(NULL == (object = H5D_open(obj_loc, H5P_DATASET_ACCESS_DEFAULT, dxpl_id))) HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open dataset") - if(H5D_mult_refresh_reopen((H5D_t *)object, dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to finish refresh for dataset") + if(!start_swmr) /* No need to handle multiple opens when H5Fstart_swmr_write() */ + if(H5D_mult_refresh_reopen((H5D_t *)object, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to finish refresh for dataset") break; case(H5I_UNINIT): diff --git a/src/H5Omessage.c b/src/H5Omessage.c index d42896c..ee21e49 100644 --- a/src/H5Omessage.c +++ b/src/H5Omessage.c @@ -477,7 +477,7 @@ H5O_msg_read(const H5O_loc_t *loc, unsigned type_id, void *mesg, HDassert(type_id < NELMTS(H5O_msg_class_g)); /* Get the object header */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, NULL, "unable to protect object header") /* Call the "real" read routine */ @@ -803,7 +803,7 @@ H5O_msg_count(const H5O_loc_t *loc, unsigned type_id, hid_t dxpl_id) HDassert(type); /* Load the object header */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to protect object header") /* Count the messages of the correct type */ @@ -885,7 +885,7 @@ H5O_msg_exists(const H5O_loc_t *loc, unsigned type_id, hid_t dxpl_id) HDassert(type_id < NELMTS(H5O_msg_class_g)); /* Load the object header */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to protect object header") /* Call the "real" exists routine */ @@ -1225,7 +1225,7 @@ H5O_msg_iterate(const H5O_loc_t *loc, unsigned type_id, HDassert(op); /* Protect the object header to iterate over */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to protect object header") /* Call the "real" iterate routine */ @@ -2254,176 +2254,3 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_flush_msgs() */ - -/*------------------------------------------------------------------------- - * Function: H5O_msg_chunkno - * - * Purpose: Queries the object header chunk index for a message. - * - * Return: Success: >=0 value indicating the chunk number for - * the message - * Failure: <0 - * - * Programmer: Quincey Koziol - * koziol@hdfgroup.org - * Apr 22 2010 - * - *------------------------------------------------------------------------- - */ -int -H5O_msg_get_chunkno(const H5O_loc_t *loc, unsigned type_id, hid_t dxpl_id) -{ - H5O_t *oh = NULL; /* Object header to use */ - const H5O_msg_class_t *type; /* Actual H5O class type for the ID */ - H5O_mesg_t *idx_msg; /* Pointer to message to modify */ - unsigned idx; /* Index of message to modify */ - int ret_value = -1; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - /* check args */ - HDassert(loc); - HDassert(loc->file); - HDassert(H5F_addr_defined(loc->addr)); - HDassert(type_id < NELMTS(H5O_msg_class_g)); - type = H5O_msg_class_g[type_id]; /* map the type ID to the actual type object */ - HDassert(type); - - /* Get the object header */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) - HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to protect object header") - - /* Locate message of correct type */ - for(idx = 0, idx_msg = &oh->mesg[0]; idx < oh->nmesgs; idx++, idx_msg++) - if(type == idx_msg->type) - break; - if(idx == oh->nmesgs) - HGOTO_ERROR(H5E_OHDR, H5E_NOTFOUND, FAIL, "message type not found") - - /* Set return value */ - H5_CHECKED_ASSIGN(ret_value, int, idx_msg->chunkno, unsigned); - -done: - if(oh && H5O_unprotect(loc, dxpl_id, oh, H5AC__NO_FLAGS_SET) < 0) - HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to release object header") - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5O_msg_get_chunkno() */ - - -/*------------------------------------------------------------------------- - * Function: H5O_msg_lock - * - * Purpose: Locks a message into a particular chunk, preventing it from - * being moved into another chunk. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * koziol@hdfgroup.org - * Apr 22 2010 - * - *------------------------------------------------------------------------- - */ -herr_t -H5O_msg_lock(const H5O_loc_t *loc, unsigned type_id, hid_t dxpl_id) -{ - H5O_t *oh = NULL; /* Object header to use */ - const H5O_msg_class_t *type; /* Actual H5O class type for the ID */ - H5O_mesg_t *idx_msg; /* Pointer to message to modify */ - unsigned idx; /* Index of message to modify */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - /* check args */ - HDassert(loc); - HDassert(loc->file); - HDassert(H5F_addr_defined(loc->addr)); - HDassert(type_id < NELMTS(H5O_msg_class_g)); - type = H5O_msg_class_g[type_id]; /* map the type ID to the actual type object */ - HDassert(type); - - /* Get the object header */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) - HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to protect object header") - - /* Locate message of correct type */ - for(idx = 0, idx_msg = &oh->mesg[0]; idx < oh->nmesgs; idx++, idx_msg++) - if(type == idx_msg->type) - break; - if(idx == oh->nmesgs) - HGOTO_ERROR(H5E_OHDR, H5E_NOTFOUND, FAIL, "message type not found") - - /* Fail if the message is already locked */ - if(idx_msg->locked) - HGOTO_ERROR(H5E_OHDR, H5E_CANTLOCK, FAIL, "message already locked") - - /* Make the message locked */ - idx_msg->locked = TRUE; - -done: - if(oh && H5O_unprotect(loc, dxpl_id, oh, H5AC__NO_FLAGS_SET) < 0) - HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to release object header") - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5O_msg_lock() */ - - -/*------------------------------------------------------------------------- - * Function: H5O_msg_unlock - * - * Purpose: Unlocks a message, allowing it to be moved into another chunk. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * koziol@hdfgroup.org - * Apr 22 2010 - * - *------------------------------------------------------------------------- - */ -herr_t -H5O_msg_unlock(const H5O_loc_t *loc, unsigned type_id, hid_t dxpl_id) -{ - H5O_t *oh = NULL; /* Object header to use */ - const H5O_msg_class_t *type; /* Actual H5O class type for the ID */ - H5O_mesg_t *idx_msg; /* Pointer to message to modify */ - unsigned idx; /* Index of message to modify */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - /* check args */ - HDassert(loc); - HDassert(loc->file); - HDassert(H5F_addr_defined(loc->addr)); - HDassert(type_id < NELMTS(H5O_msg_class_g)); - type = H5O_msg_class_g[type_id]; /* map the type ID to the actual type object */ - HDassert(type); - - /* Get the object header */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) - HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to protect object header") - - /* Locate message of correct type */ - for(idx = 0, idx_msg = &oh->mesg[0]; idx < oh->nmesgs; idx++, idx_msg++) - if(type == idx_msg->type) - break; - if(idx == oh->nmesgs) - HGOTO_ERROR(H5E_OHDR, H5E_NOTFOUND, FAIL, "message type not found") - - /* Fail if the message is not locked */ - if(!idx_msg->locked) - HGOTO_ERROR(H5E_OHDR, H5E_CANTUNLOCK, FAIL, "message not locked") - - /* Make the message unlocked */ - idx_msg->locked = FALSE; - -done: - if(oh && H5O_unprotect(loc, dxpl_id, oh, H5AC__NO_FLAGS_SET) < 0) - HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to release object header") - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5O_msg_unlock() */ - diff --git a/src/H5Opkg.h b/src/H5Opkg.h index e40d28a..ef49535 100644 --- a/src/H5Opkg.h +++ b/src/H5Opkg.h @@ -217,6 +217,12 @@ #define H5O_SHARE_IS_SHARABLE 0x01 #define H5O_SHARE_IN_OHDR 0x02 +/* Set the object header size to speculatively read in */ +/* (needs to be more than the object header prefix size to work at all and + * should be larger than the largest object type's default object header + * size to save the extra I/O operations) */ +#define H5O_SPEC_READ_SIZE 512 + /* The "message class" type */ struct H5O_msg_class_t { @@ -245,7 +251,6 @@ struct H5O_msg_class_t { struct H5O_mesg_t { const H5O_msg_class_t *type; /*type of message */ hbool_t dirty; /*raw out of date wrt native */ - hbool_t locked; /*message is locked into chunk */ uint8_t flags; /*message flags */ H5O_msg_crt_idx_t crt_idx; /*message creation index */ unsigned chunkno; /*chunk number for this mesg */ @@ -254,11 +259,23 @@ struct H5O_mesg_t { size_t raw_size; /*size with alignment */ }; +/* Struct for storing information about "best" message to move to new chunk */ +typedef struct H5O_msg_alloc_info_t { + int msgno; /* Index in message array */ + unsigned id; /* Message type ID on disk */ + unsigned chunkno; /* Index in chunk array */ + size_t gap_size; /* Size of any "gap" in the chunk immediately after message */ + size_t null_size; /* Size of any null message in the chunk immediately after message */ + size_t total_size; /* Total size of "available" space around message */ + unsigned null_msgno; /* Message index of null message immediately after message */ +} H5O_msg_alloc_info_t; + typedef struct H5O_chunk_t { haddr_t addr; /*chunk file address */ size_t size; /*chunk size */ size_t gap; /*space at end of chunk too small for null message */ uint8_t *image; /*image of file */ + struct H5O_chunk_proxy_t *chunk_proxy; /* Pointer to a chunk's proxy when chunk protected */ } H5O_chunk_t; struct H5O_t { @@ -268,6 +285,7 @@ struct H5O_t { /* File-specific information (not stored) */ size_t sizeof_size; /* Size of file sizes */ size_t sizeof_addr; /* Size of file addresses */ + hbool_t swmr_write; /* Whether we are doing SWMR writes */ /* Debugging information (not stored) */ #ifdef H5O_ENABLE_BAD_MESG_COUNT @@ -282,9 +300,6 @@ struct H5O_t { /* Chunk management information (not stored) */ size_t rc; /* Reference count of [continuation] chunks using this structure */ - size_t chunk0_size; /* Size of serialized first chunk */ - hbool_t mesgs_modified; /* Whether any messages were modified when the object header was deserialized */ - hbool_t prefix_modified; /* Whether prefix was modified when the object header was deserialized */ /* Object information (stored) */ hbool_t has_refcount_msg; /* Whether the object has a ref. count message */ @@ -313,6 +328,10 @@ struct H5O_t { size_t nchunks; /*number of chunks */ size_t alloc_nchunks; /*chunks allocated */ H5O_chunk_t *chunk; /*array of chunks */ + hbool_t chunks_pinned; /* Whether chunks are pinned from ohdr protect */ + + /* Object header proxy information (not stored) */ + H5AC_proxy_entry_t *proxy; /* Proxy cache entry for all ohdr entries */ }; /* Class for types of objects in file */ @@ -352,7 +371,6 @@ typedef struct H5O_common_cache_ud_t { hid_t dxpl_id; /* DXPL for operation */ unsigned file_intent; /* Read/write intent for file */ unsigned merged_null_msgs; /* Number of null messages merged together */ - hbool_t mesgs_modified; /* Whether any messages were modified when the object header was deserialized */ H5O_cont_msgs_t *cont_msg_info; /* Pointer to continuation messages to work on */ haddr_t addr; /* Address of the prefix or chunk */ } H5O_common_cache_ud_t; @@ -361,6 +379,8 @@ typedef struct H5O_common_cache_ud_t { typedef struct H5O_cache_ud_t { hbool_t made_attempt; /* Whether the deserialize routine was already attempted */ unsigned v1_pfx_nmesgs; /* Number of messages from v1 prefix header */ + size_t chunk0_size; /* Size of serialized first chunk */ + H5O_t *oh; /* Partially deserialized object header, for later use */ H5O_common_cache_ud_t common; /* Common object header cache callback info */ } H5O_cache_ud_t; @@ -369,8 +389,24 @@ typedef struct H5O_chunk_proxy_t { H5AC_info_t cache_info; /* Information for metadata cache functions, _must_ be */ /* first field in structure */ + H5F_t *f; /* Pointer to file for object header/chunk */ H5O_t *oh; /* Object header for this chunk */ unsigned chunkno; /* Chunk number for this chunk */ + unsigned cont_chunkno; /* Chunk number for the chunk containing the continuation message that points to this chunk */ + + /* Flush depencency parent information (not stored) + * + * The following field is used to store a pointer + * to the in-core representation of the chunk proxy's flush dependency + * parent -- if it exists. If it does not exist, this field will + * contain NULL. + * + * If the file is opened in SWMR write mode, the flush dependency + * parent of the chunk proxy will be either its object header + * (if cont_chunkno == 0) or the chunk proxy indicated by the + * cont_chunkno field (if cont_chunkno > 0). + */ + void *parent; /* Pointer to flush dependency parent */ } H5O_chunk_proxy_t; /* Callback information for loading object header chunk from disk */ @@ -535,7 +571,7 @@ H5_DLL const H5O_obj_class_t * H5O_obj_class(const H5O_loc_t *loc, hid_t dxpl_id H5_DLL int H5O_link_oh(H5F_t *f, int adjust, hid_t dxpl_id, H5O_t *oh, hbool_t *deleted); H5_DLL herr_t H5O_inc_rc(H5O_t *oh); H5_DLL herr_t H5O_dec_rc(H5O_t *oh); -H5_DLL herr_t H5O_free(H5O_t *oh); +H5_DLL herr_t H5O__free(H5O_t *oh); /* Object header message routines */ H5_DLL herr_t H5O_msg_alloc(H5F_t *f, hid_t dxpl_id, H5O_t *oh, @@ -559,7 +595,8 @@ H5_DLL herr_t H5O_msg_iterate_real(H5F_t *f, H5O_t *oh, const H5O_msg_class_t *t const H5O_mesg_operator_t *op, void *op_data, hid_t dxpl_id); /* Object header chunk routines */ -H5_DLL herr_t H5O_chunk_add(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned idx); +H5_DLL herr_t H5O_chunk_add(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned idx, + unsigned cont_chunkno); H5_DLL H5O_chunk_proxy_t *H5O_chunk_protect(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned idx); H5_DLL herr_t H5O_chunk_unprotect(H5F_t *f, hid_t dxpl_id, @@ -567,6 +604,7 @@ H5_DLL herr_t H5O_chunk_unprotect(H5F_t *f, hid_t dxpl_id, H5_DLL herr_t H5O_chunk_update_idx(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned idx); H5_DLL herr_t H5O_chunk_resize(H5O_t *oh, H5O_chunk_proxy_t *chk_proxy); H5_DLL herr_t H5O_chunk_delete(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned idx); +H5_DLL herr_t H5O__chunk_dest(H5O_chunk_proxy_t *chunk_proxy); /* Collect storage info for btree and heap */ H5_DLL herr_t H5O_attr_bh_info(H5F_t *f, hid_t dxpl_id, H5O_t *oh, @@ -574,6 +612,8 @@ H5_DLL herr_t H5O_attr_bh_info(H5F_t *f, hid_t dxpl_id, H5O_t *oh, /* Object header allocation routines */ H5_DLL herr_t H5O_alloc_msgs(H5O_t *oh, size_t min_alloc); +H5_DLL herr_t H5O__alloc_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh, size_t size, + size_t found_null, const H5O_msg_alloc_info_t *found_msg, size_t *new_idx); H5_DLL herr_t H5O_alloc(H5F_t *f, hid_t dxpl_id, H5O_t *oh, const H5O_msg_class_t *type, const void *mesg, size_t *mesg_idx); H5_DLL herr_t H5O_condense_header(H5F_t *f, H5O_t *oh, hid_t dxpl_id); @@ -607,9 +647,6 @@ H5_DLL herr_t H5O_attr_link(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, void *_mesg H5_DLL herr_t H5O_attr_count_real(H5F_t *f, hid_t dxpl_id, H5O_t *oh, hsize_t *nattrs); -/* These functions operate on object locations */ -H5_DLL H5O_loc_t *H5O_get_loc(hid_t id); - /* Testing functions */ #ifdef H5O_TESTING H5_DLL htri_t H5O_is_attr_empty_test(hid_t oid); @@ -619,6 +656,9 @@ H5_DLL herr_t H5O_attr_dense_info_test(hid_t oid, hsize_t *name_count, hsize_t * H5_DLL herr_t H5O_check_msg_marked_test(hid_t oid, hbool_t flag_val); H5_DLL herr_t H5O_expunge_chunks_test(const H5O_loc_t *oloc, hid_t dxpl_id); H5_DLL herr_t H5O_get_rc(const H5O_loc_t *oloc, hid_t dxpl_id, unsigned *rc); +H5_DLL herr_t H5O_msg_get_chunkno_test(hid_t oid, unsigned msg_type, + unsigned *chunk_num); +H5_DLL herr_t H5O_msg_move_to_new_chunk_test(hid_t oid, unsigned msg_type); #endif /* H5O_TESTING */ /* Object header debugging routines */ diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 0acac8c..e430b1f 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -827,9 +827,10 @@ H5_DLL herr_t H5O_init(void); H5_DLL herr_t H5O_create(H5F_t *f, hid_t dxpl_id, size_t size_hint, size_t initial_rc, hid_t ocpl_id, H5O_loc_t *loc/*out*/); H5_DLL herr_t H5O_open(H5O_loc_t *loc); -H5_DLL herr_t H5O_close(H5O_loc_t *loc); +H5_DLL herr_t H5O_close(H5O_loc_t *loc, hbool_t *file_closed/*out*/); H5_DLL int H5O_link(const H5O_loc_t *loc, int adjust, hid_t dxpl_id); -H5_DLL H5O_t *H5O_protect(const H5O_loc_t *loc, hid_t dxpl_id, unsigned prot_flags); +H5_DLL H5O_t *H5O_protect(const H5O_loc_t *loc, hid_t dxpl_id, + unsigned prot_flags, hbool_t pin_all_chunks); H5_DLL H5O_t *H5O_pin(const H5O_loc_t *loc, hid_t dxpl_id); H5_DLL herr_t H5O_unpin(H5O_t *oh); H5_DLL herr_t H5O_dec_rc_by_loc(const H5O_loc_t *loc, hid_t dxpl_id); @@ -852,6 +853,7 @@ H5_DLL herr_t H5O_get_nlinks(const H5O_loc_t *loc, hid_t dxpl_id, hsize_t *nlink H5_DLL void *H5O_obj_create(H5F_t *f, H5O_type_t obj_type, void *crt_info, H5G_loc_t *obj_loc, hid_t dxpl_id); H5_DLL haddr_t H5O_get_oh_addr(const H5O_t *oh); H5_DLL herr_t H5O_get_rc_and_type(const H5O_loc_t *oloc, hid_t dxpl_id, unsigned *rc, H5O_type_t *otype); +H5_DLL H5AC_proxy_entry_t *H5O_get_proxy(const H5O_t *oh); /* Object header message routines */ H5_DLL herr_t H5O_msg_create(const H5O_loc_t *loc, unsigned type_id, unsigned mesg_flags, @@ -898,15 +900,12 @@ H5_DLL void* H5O_msg_decode(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, unsigned type_id, const unsigned char *buf); H5_DLL herr_t H5O_msg_delete(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, unsigned type_id, void *mesg); -H5_DLL int H5O_msg_get_chunkno(const H5O_loc_t *loc, unsigned type_id, hid_t dxpl_id); -H5_DLL herr_t H5O_msg_lock(const H5O_loc_t *loc, unsigned type_id, hid_t dxpl_id); -H5_DLL herr_t H5O_msg_unlock(const H5O_loc_t *loc, unsigned type_id, hid_t dxpl_id); /* Object metadata flush/refresh routines */ H5_DLL herr_t H5O_flush_common(H5O_loc_t *oloc, hid_t obj_id, hid_t dxpl_id); H5_DLL herr_t H5O_refresh_metadata(hid_t oid, H5O_loc_t oloc, hid_t dxpl_id); H5_DLL herr_t H5O_refresh_metadata_close(hid_t oid, H5O_loc_t oloc, H5G_loc_t *obj_loc, hid_t dxpl_id); -H5_DLL herr_t H5O_refresh_metadata_reopen(hid_t oid, H5G_loc_t *obj_loc, hid_t dxpl_id); +H5_DLL herr_t H5O_refresh_metadata_reopen(hid_t oid, H5G_loc_t *obj_loc, hid_t dxpl_id, hbool_t start_swmr); /* Object copying routines */ H5_DLL herr_t H5O_copy_header_map(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out */, @@ -926,6 +925,7 @@ H5_DLL herr_t H5O_loc_reset(H5O_loc_t *loc); H5_DLL herr_t H5O_loc_copy(H5O_loc_t *dst, H5O_loc_t *src, H5_copy_depth_t depth); H5_DLL herr_t H5O_loc_hold_file(H5O_loc_t *loc); H5_DLL herr_t H5O_loc_free(H5O_loc_t *loc); +H5_DLL H5O_loc_t *H5O_get_loc(hid_t id); /* EFL operators */ H5_DLL hsize_t H5O_efl_total_size(H5O_efl_t *efl); diff --git a/src/H5Otest.c b/src/H5Otest.c index c3e487d..8f8980a 100644 --- a/src/H5Otest.c +++ b/src/H5Otest.c @@ -108,7 +108,7 @@ H5O_is_attr_dense_test(hid_t oid) HGOTO_ERROR(H5E_SYM, H5E_NOTFOUND, FAIL, "object not found") /* Get the object header */ - if(NULL == (oh = H5O_protect(loc, H5AC_ind_read_dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, H5AC_ind_read_dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header") /* Check for attribute info stored */ @@ -173,7 +173,7 @@ H5O_is_attr_empty_test(hid_t oid) HGOTO_ERROR(H5E_SYM, H5E_NOTFOUND, FAIL, "object not found") /* Get the object header */ - if(NULL == (oh = H5O_protect(loc, H5AC_ind_read_dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, H5AC_ind_read_dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header") /* Check for attribute info stored */ @@ -266,7 +266,7 @@ H5O_num_attrs_test(hid_t oid, hsize_t *nattrs) HGOTO_ERROR(H5E_SYM, H5E_NOTFOUND, FAIL, "object not found") /* Get the object header */ - if(NULL == (oh = H5O_protect(loc, H5AC_ind_read_dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, H5AC_ind_read_dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header") /* Check for attribute info stored */ @@ -361,7 +361,7 @@ H5O_attr_dense_info_test(hid_t oid, hsize_t *name_count, hsize_t *corder_count) H5_BEGIN_TAG(H5AC_ind_read_dxpl_id, loc->addr, FAIL); /* Get the object header */ - if(NULL == (oh = H5O_protect(loc, H5AC_ind_read_dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, H5AC_ind_read_dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR_TAG(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header") /* Check for attribute info stored */ @@ -452,7 +452,7 @@ H5O_check_msg_marked_test(hid_t oid, hbool_t flag_val) HGOTO_ERROR(H5E_SYM, H5E_NOTFOUND, FAIL, "object not found") /* Get the object header */ - if(NULL == (oh = H5O_protect(loc, H5AC_ind_read_dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, H5AC_ind_read_dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header") /* Locate "unknown" message */ @@ -511,7 +511,7 @@ H5O_expunge_chunks_test(const H5O_loc_t *loc, hid_t dxpl_id) FUNC_ENTER_NOAPI(FAIL) /* Get the object header */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__NO_FLAGS_SET))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__NO_FLAGS_SET, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to protect object header") /* Safety check */ @@ -571,7 +571,7 @@ H5O_get_rc(const H5O_loc_t *loc, hid_t dxpl_id, unsigned *rc) HDassert(rc); /* Get the object header */ - if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to protect object header") /* Save the refcount for the object header */ @@ -585,3 +585,165 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5O_expunge_chunks_test() */ + +/*-------------------------------------------------------------------------- + NAME + H5O_msg_get_chunkno_test + PURPOSE + Retrieve the chunk number for an object header message of a given type. + USAGE + herr_t H5O_check_msg_marked_test(oid, chunk_num) + hid_t oid; IN: Object to check + unsigned msg_type; IN: Object header message type to check + unsigned *chunk_num; OUT: Object header chunk that the message is in + RETURNS + Non-negative on success, negative on failure + DESCRIPTION + Retrieves the chunk number for the first object header message of a given + type found in an object's header. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + DO NOT USE THIS FUNCTION FOR ANYTHING EXCEPT TESTING + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5O_msg_get_chunkno_test(hid_t oid, unsigned msg_type, unsigned *chunk_num) +{ + H5O_t *oh = NULL; /* Object header */ + H5O_loc_t *loc; /* Pointer to object's location */ + H5O_mesg_t *idx_msg; /* Pointer to message */ + unsigned idx; /* Index of message */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Get object location for object */ + if(NULL == (loc = H5O_get_loc(oid))) + HGOTO_ERROR(H5E_SYM, H5E_NOTFOUND, FAIL, "object not found") + + /* Get the object header */ + if(NULL == (oh = H5O_protect(loc, H5AC_ind_read_dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header") + + /* Locate first message of given type */ + for(idx = 0, idx_msg = &oh->mesg[0]; idx < oh->nmesgs; idx++, idx_msg++) + if(idx_msg->type->id == msg_type) { + /* Set the chunk number for the message */ + *chunk_num = idx_msg->chunkno; + + /* Break out of loop, to indicate that the message was found */ + break; + } /* end if */ + + /* Check for not finding a message of the given type*/ + if(idx == oh->nmesgs) + HGOTO_ERROR(H5E_OHDR, H5E_NOTFOUND, FAIL, "message of type not found") + +done: + if(oh && H5O_unprotect(loc, H5AC_ind_read_dxpl_id, oh, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to release object header") + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5O_msg_get_chunkno_test() */ + + +/*-------------------------------------------------------------------------- + NAME + H5O_msg_move_to_new_chunk_test + PURPOSE + Move a message into a new chunk + USAGE + herr_t H5O_msg_move_to_new_chunk_test(oid, msg_type) + hid_t oid; IN: Object to check + unsigned msg_type; IN: Object header message type to check + RETURNS + Non-negative on success, negative on failure + DESCRIPTION + Moves the first message of the given type to a new object header chunk. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + DO NOT USE THIS FUNCTION FOR ANYTHING EXCEPT TESTING + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5O_msg_move_to_new_chunk_test(hid_t oid, unsigned msg_type) +{ + H5O_t *oh = NULL; /* Object header */ + H5O_loc_t *loc; /* Pointer to object's location */ + H5O_mesg_t *curr_msg; /* Pointer to current message */ + unsigned idx; /* Index of message */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Get object location for object */ + if(NULL == (loc = H5O_get_loc(oid))) + HGOTO_ERROR(H5E_SYM, H5E_NOTFOUND, FAIL, "object not found") + + /* Get the object header */ + if(NULL == (oh = H5O_protect(loc, H5AC_ind_read_dxpl_id, H5AC__NO_FLAGS_SET, FALSE))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to load object header") + + /* Locate first message of given type */ + for(idx = 0, curr_msg = &oh->mesg[0]; idx < oh->nmesgs; idx++, curr_msg++) + if(curr_msg->type->id == msg_type) { + H5O_msg_alloc_info_t found_msg; /* Information about message to move */ + unsigned msg_chunkno = curr_msg->chunkno; /* Chunk that the message is in */ + uint8_t *end_chunk_data = (oh->chunk[msg_chunkno].image + oh->chunk[msg_chunkno].size) - (H5O_SIZEOF_CHKSUM_OH(oh) + oh->chunk[msg_chunkno].gap); /* End of message data in chunk */ + uint8_t *end_msg = curr_msg->raw + curr_msg->raw_size; /* End of current message */ + size_t gap_size = 0; /* Size of gap after current message */ + size_t null_size = 0; /* Size of NULL message after current message */ + unsigned null_msgno = 0; /* Index of NULL message after current message */ + size_t total_size; /* Total size of available space "around" current message */ + size_t new_idx; /* Index of new null message */ + + /* Check if the message is the last one in the chunk */ + if(end_msg == end_chunk_data) + gap_size = oh->chunk[msg_chunkno].gap; + else { + H5O_mesg_t *tmp_msg; /* Temp. pointer to message to operate on */ + unsigned v; /* Local index variable */ + + /* Check for null message after this message, in same chunk */ + for(v = 0, tmp_msg = &oh->mesg[0]; v < oh->nmesgs; v++, tmp_msg++) { + if(tmp_msg->type->id == H5O_NULL_ID && (tmp_msg->raw - H5O_SIZEOF_MSGHDR_OH(oh)) == end_msg) { + null_msgno = v; + null_size = (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + tmp_msg->raw_size; + break; + } /* end if */ + + /* XXX: Should also check for NULL message in front of current message... */ + + } /* end for */ + } /* end else */ + + /* Add up current message's total available space */ + total_size = curr_msg->raw_size + gap_size + null_size; + + /* Set up "found message" info for moving the message */ + found_msg.msgno = (int)idx; + found_msg.id = curr_msg->type->id; + found_msg.chunkno = msg_chunkno; + found_msg.gap_size = gap_size; + found_msg.null_size = null_size; + found_msg.total_size = total_size; + found_msg.null_msgno = null_msgno; + + /* Allocate and initialize new chunk in the file, moving the found message */ + /* (*new_idx returned from this routine is unused here) */ + if(H5O__alloc_chunk(loc->file, H5AC_ind_read_dxpl_id, oh, 40, oh->nmesgs, &found_msg, &new_idx) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "can't allocate new object header chunk") + + /* Break out of loop, the message was found */ + break; + } /* end if */ + +done: + if(oh && H5O_unprotect(loc, H5AC_ind_read_dxpl_id, oh, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to release object header") + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5O_msg_get_chunkno_test() */ + @@ -84,7 +84,7 @@ typedef const void *(__cdecl *H5PL_get_plugin_info_t)(void); #define H5PL_HANDLE void * /* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */ -#define H5PL_OPEN_DLIB(S) dlopen(S, RTLD_NOW) +#define H5PL_OPEN_DLIB(S) dlopen(S, RTLD_LAZY) /* Get the address of a symbol in dynamic library */ #define H5PL_GET_LIB_FUNC(H,N) dlsym(H,N) @@ -93,7 +93,7 @@ typedef const void *(__cdecl *H5PL_get_plugin_info_t)(void); #define H5PL_CLOSE_LIB(H) dlclose(H) /* Clear error */ -#define H5PL_CLR_ERROR dlerror() +#define H5PL_CLR_ERROR HERROR(H5E_PLUGIN, H5E_CANTGET, "can't dlopen:%s", dlerror()) typedef const void *(*H5PL_get_plugin_info_t)(void); #endif /* H5_HAVE_WIN32_API */ diff --git a/src/H5Pdxpl.c b/src/H5Pdxpl.c index e685c65..c094c20 100644 --- a/src/H5Pdxpl.c +++ b/src/H5Pdxpl.c @@ -273,7 +273,7 @@ static const void *H5D_def_vlen_alloc_info_g = H5D_XFER_VLEN_ALLOC_INFO_DEF; / static const H5MM_free_t H5D_def_vlen_free_g = H5D_XFER_VLEN_FREE_DEF; /* Default value for vlen free function */ static const void *H5D_def_vlen_free_info_g = H5D_XFER_VLEN_FREE_INFO_DEF; /* Default value for vlen free information */ static const size_t H5D_def_hyp_vec_size_g = H5D_XFER_HYPER_VECTOR_SIZE_DEF; /* Default value for vector size */ -static const H5C_tag_t H5D_def_tag_g = H5C_TAG_DEF; /* Default value for cache entry tag */ +static const haddr_t H5D_def_tag_g = H5AC_TAG_DEF; /* Default value for cache entry tag */ static const H5FD_mpio_xfer_t H5D_def_io_xfer_mode_g = H5D_XFER_IO_XFER_MODE_DEF; /* Default value for I/O transfer mode */ static const H5FD_mpio_chunk_opt_t H5D_def_mpio_chunk_opt_mode_g = H5D_XFER_MPIO_CHUNK_OPT_HARD_DEF; static const H5FD_mpio_collective_opt_t H5D_def_mpio_collective_opt_mode_g = H5D_XFER_MPIO_COLLECTIVE_OPT_DEF; @@ -326,7 +326,7 @@ H5P__dxfr_reg_prop(H5P_genclass_t *pclass) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") /* Register the cache tag property */ - if(H5P_register_real(pclass, H5C_TAG_NAME, H5C_TAG_SIZE, &H5D_def_tag_g, + if(H5P_register_real(pclass, H5AC_TAG_NAME, H5AC_TAG_SIZE, &H5D_def_tag_g, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 96de39c..91a0e03 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -178,9 +178,41 @@ #define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEF 524288 #define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_ENC H5P__encode_size_t #define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEC H5P__decode_size_t +/* Definition for # of metadata read attempts */ +#define H5F_ACS_METADATA_READ_ATTEMPTS_SIZE sizeof(unsigned) +#define H5F_ACS_METADATA_READ_ATTEMPTS_DEF 0 +#define H5F_ACS_METADATA_READ_ATTEMPTS_ENC H5P__encode_unsigned +#define H5F_ACS_METADATA_READ_ATTEMPTS_DEC H5P__decode_unsigned /* Definition for object flush callback */ #define H5F_ACS_OBJECT_FLUSH_CB_SIZE sizeof(H5F_object_flush_t) #define H5F_ACS_OBJECT_FLUSH_CB_DEF {NULL, NULL} +/* Definition for status_flags in the superblock */ +#define H5F_ACS_CLEAR_STATUS_FLAGS_SIZE sizeof(hbool_t) +#define H5F_ACS_CLEAR_STATUS_FLAGS_DEF FALSE +/* Definition for 'use metadata cache logging' flag */ +#define H5F_ACS_USE_MDC_LOGGING_SIZE sizeof(hbool_t) +#define H5F_ACS_USE_MDC_LOGGING_DEF FALSE +#define H5F_ACS_USE_MDC_LOGGING_ENC H5P__encode_hbool_t +#define H5F_ACS_USE_MDC_LOGGING_DEC H5P__decode_hbool_t +/* Definition for 'mdc log location' flag */ +#define H5F_ACS_MDC_LOG_LOCATION_SIZE sizeof(char *) +#define H5F_ACS_MDC_LOG_LOCATION_DEF NULL /* default is no log location */ +#define H5F_ACS_MDC_LOG_LOCATION_ENC H5P_facc_mdc_log_location_enc +#define H5F_ACS_MDC_LOG_LOCATION_DEC H5P_facc_mdc_log_location_dec +#define H5F_ACS_MDC_LOG_LOCATION_DEL H5P_facc_mdc_log_location_del +#define H5F_ACS_MDC_LOG_LOCATION_COPY H5P_facc_mdc_log_location_copy +#define H5F_ACS_MDC_LOG_LOCATION_CMP H5P_facc_mdc_log_location_cmp +#define H5F_ACS_MDC_LOG_LOCATION_CLOSE H5P_facc_mdc_log_location_close +/* Definition for 'start metadata cache logging on access' flag */ +#define H5F_ACS_START_MDC_LOG_ON_ACCESS_SIZE sizeof(hbool_t) +#define H5F_ACS_START_MDC_LOG_ON_ACCESS_DEF FALSE +#define H5F_ACS_START_MDC_LOG_ON_ACCESS_ENC H5P__encode_hbool_t +#define H5F_ACS_START_MDC_LOG_ON_ACCESS_DEC H5P__decode_hbool_t +/* Definition for evict on close property */ +#define H5F_ACS_EVICT_ON_CLOSE_FLAG_SIZE sizeof(hbool_t) +#define H5F_ACS_EVICT_ON_CLOSE_FLAG_DEF FALSE +#define H5F_ACS_EVICT_ON_CLOSE_FLAG_ENC H5P__encode_hbool_t +#define H5F_ACS_EVICT_ON_CLOSE_FLAG_DEC H5P__decode_hbool_t #ifdef H5_HAVE_PARALLEL /* Definition of collective metadata read mode flag */ #define H5F_ACS_COLL_MD_READ_FLAG_SIZE sizeof(H5P_coll_md_read_flag_t) @@ -239,6 +271,14 @@ static herr_t H5P__facc_fclose_degree_dec(const void **pp, void *value); static herr_t H5P__facc_multi_type_enc(const void *value, void **_pp, size_t *size); static herr_t H5P__facc_multi_type_dec(const void **_pp, void *value); +/* Metadata cache log location property callbacks */ +static herr_t H5P_facc_mdc_log_location_enc(const void *value, void **_pp, size_t *size); +static herr_t H5P_facc_mdc_log_location_dec(const void **_pp, void *value); +static herr_t H5P_facc_mdc_log_location_del(hid_t prop_id, const char *name, size_t size, void *value); +static herr_t H5P_facc_mdc_log_location_copy(const char *name, size_t size, void *value); +static int H5P_facc_mdc_log_location_cmp(const void *value1, const void *value2, size_t size); +static herr_t H5P_facc_mdc_log_location_close(const char *name, size_t size, void *value); + /*********************/ /* Package Variables */ @@ -295,7 +335,13 @@ static const unsigned H5F_def_efc_size_g = H5F_ACS_EFC_SIZE_DEF; static const H5FD_file_image_info_t H5F_def_file_image_info_g = H5F_ACS_FILE_IMAGE_INFO_DEF; /* Default file image info and callbacks */ static const hbool_t H5F_def_core_write_tracking_flag_g = H5F_ACS_CORE_WRITE_TRACKING_FLAG_DEF; /* Default setting for core VFD write tracking */ static const size_t H5F_def_core_write_tracking_page_size_g = H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEF; /* Default core VFD write tracking page size */ +static const unsigned H5F_def_metadata_read_attempts_g = H5F_ACS_METADATA_READ_ATTEMPTS_DEF; /* Default setting for the # of metadata read attempts */ static const H5F_object_flush_t H5F_def_object_flush_cb_g = H5F_ACS_OBJECT_FLUSH_CB_DEF; /* Default setting for object flush callback */ +static const hbool_t H5F_def_clear_status_flags_g = H5F_ACS_CLEAR_STATUS_FLAGS_DEF; /* Default to clear the superblock status_flags */ +static const hbool_t H5F_def_use_mdc_logging_g = H5F_ACS_USE_MDC_LOGGING_DEF; /* Default metadata cache logging flag */ +static const char *H5F_def_mdc_log_location_g = H5F_ACS_MDC_LOG_LOCATION_DEF; /* Default mdc log location */ +static const hbool_t H5F_def_start_mdc_log_on_access_g = H5F_ACS_START_MDC_LOG_ON_ACCESS_DEF; /* Default mdc log start on access flag */ +static const hbool_t H5F_def_evict_on_close_flag_g = H5F_ACS_EVICT_ON_CLOSE_FLAG_DEF; /* Default setting for evict on close property */ #ifdef H5_HAVE_PARALLEL static const H5P_coll_md_read_flag_t H5F_def_coll_md_read_flag_g = H5F_ACS_COLL_MD_READ_FLAG_DEF; /* Default setting for the collective metedata read flag */ static const hbool_t H5F_def_coll_md_write_flag_g = H5F_ACS_COLL_MD_WRITE_FLAG_DEF; /* Default setting for the collective metedata write flag */ @@ -456,12 +502,45 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the # of read attempts */ + if(H5P_register_real(pclass, H5F_ACS_METADATA_READ_ATTEMPTS_NAME, H5F_ACS_METADATA_READ_ATTEMPTS_SIZE, &H5F_def_metadata_read_attempts_g, + NULL, NULL, NULL, H5F_ACS_METADATA_READ_ATTEMPTS_ENC, H5F_ACS_METADATA_READ_ATTEMPTS_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register object flush callback */ /* (Note: this property should not have an encode/decode callback -QAK) */ if(H5P_register_real(pclass, H5F_ACS_OBJECT_FLUSH_CB_NAME, H5F_ACS_OBJECT_FLUSH_CB_SIZE, &H5F_def_object_flush_cb_g, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the private property of whether to clear the superblock status_flags. It's used by h5clear only. */ + if(H5P_register_real(pclass, H5F_ACS_CLEAR_STATUS_FLAGS_NAME, H5F_ACS_CLEAR_STATUS_FLAGS_SIZE, &H5F_def_clear_status_flags_g, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the metadata cache logging flag. */ + if(H5P_register_real(pclass, H5F_ACS_USE_MDC_LOGGING_NAME, H5F_ACS_USE_MDC_LOGGING_SIZE, &H5F_def_use_mdc_logging_g, + NULL, NULL, NULL, H5F_ACS_USE_MDC_LOGGING_ENC, H5F_ACS_USE_MDC_LOGGING_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the metadata cache log location. */ + if(H5P_register_real(pclass, H5F_ACS_MDC_LOG_LOCATION_NAME, H5F_ACS_MDC_LOG_LOCATION_SIZE, &H5F_def_mdc_log_location_g, + NULL, NULL, NULL, H5F_ACS_MDC_LOG_LOCATION_ENC, H5F_ACS_MDC_LOG_LOCATION_DEC, + H5F_ACS_MDC_LOG_LOCATION_DEL, H5F_ACS_MDC_LOG_LOCATION_COPY, H5F_ACS_MDC_LOG_LOCATION_CMP, H5F_ACS_MDC_LOG_LOCATION_CLOSE) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the flag that indicates whether mdc logging starts on file access. */ + if(H5P_register_real(pclass, H5F_ACS_START_MDC_LOG_ON_ACCESS_NAME, H5F_ACS_START_MDC_LOG_ON_ACCESS_SIZE, &H5F_def_start_mdc_log_on_access_g, + NULL, NULL, NULL, H5F_ACS_START_MDC_LOG_ON_ACCESS_ENC, H5F_ACS_START_MDC_LOG_ON_ACCESS_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the evict on close flag */ + if(H5P_register_real(pclass, H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME, H5F_ACS_EVICT_ON_CLOSE_FLAG_SIZE, &H5F_def_evict_on_close_flag_g, + NULL, NULL, NULL, H5F_ACS_EVICT_ON_CLOSE_FLAG_ENC, H5F_ACS_EVICT_ON_CLOSE_FLAG_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + #ifdef H5_HAVE_PARALLEL /* Register the metadata collective read flag */ if(H5P_register_real(pclass, H5_COLL_MD_READ_FLAG_NAME, H5F_ACS_COLL_MD_READ_FLAG_SIZE, &H5F_def_coll_md_read_flag_g, @@ -817,11 +896,11 @@ done: * *------------------------------------------------------------------------- */ -void * +const void * H5Pget_driver_info(hid_t plist_id) { - H5P_genplist_t *plist; /* Property list pointer */ - void *ret_value; /* Return value */ + H5P_genplist_t *plist = NULL; /* Property list pointer */ + const void *ret_value = NULL; /* Return value */ FUNC_ENTER_API(NULL) H5TRACE1("*x", "i", plist_id); @@ -830,7 +909,7 @@ H5Pget_driver_info(hid_t plist_id) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a property list") /* Get the driver info */ - if(NULL == (ret_value = (void *)H5P_peek_driver_info(plist))) + if(NULL == (ret_value = (const void *)H5P_peek_driver_info(plist))) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get driver info") done: @@ -2922,16 +3001,6 @@ H5P__facc_cache_config_cmp(const void *_config1, const void *_config2, size_t H5 if(config1->rpt_fcn_enabled < config2->rpt_fcn_enabled) HGOTO_DONE(-1); if(config1->rpt_fcn_enabled > config2->rpt_fcn_enabled) HGOTO_DONE(1); - if(config1->open_trace_file < config2->open_trace_file) HGOTO_DONE(-1); - if(config1->open_trace_file > config2->open_trace_file) HGOTO_DONE(1); - - if(config1->close_trace_file < config2->close_trace_file) HGOTO_DONE(-1); - if(config1->close_trace_file > config2->close_trace_file) HGOTO_DONE(1); - - if((ret_value = HDstrncmp(config1->trace_file_name, config2->trace_file_name, - (size_t)(H5AC__MAX_TRACE_FILE_NAME_LEN + 1))) != 0) - HGOTO_DONE(ret_value); - if(config1->evictions_enabled < config2->evictions_enabled) HGOTO_DONE(-1); if(config1->evictions_enabled > config2->evictions_enabled) HGOTO_DONE(1); @@ -3522,6 +3591,90 @@ done: /*------------------------------------------------------------------------- + * Function: H5Pset_metadata_read_attempts + * + * Purpose: Sets the # of read attempts in the file access property list + * when reading metadata with checksum. + * The # of read attempts set via this routine will only apply + * when opening a file with SWMR access. + * The # of read attempts set via this routine does not have + * any effect when opening a file with non-SWMR access; for this + * case, the # of read attempts will be always be 1. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Sept 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_metadata_read_attempts(hid_t plist_id, unsigned attempts) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "iIu", plist_id, attempts); + + /* Cannot set the # of attempts to 0 */ + if(attempts == 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "number of metadatata read attempts must be greater than 0"); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Set values */ + if(H5P_set(plist, H5F_ACS_METADATA_READ_ATTEMPTS_NAME, &attempts) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set # of metadata read attempts") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pset_metadata_read_attempts() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_metadata_read_attempts + * + * Purpose: Returns the # of metadata read attempts set in the file access property list. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Sept 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_metadata_read_attempts(hid_t plist_id, unsigned *attempts/*out*/) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ix", plist_id, attempts); + + /* Get values */ + if(attempts) { + H5P_genplist_t *plist; /* Property list pointer */ + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get the # of read attempts set */ + if(H5P_get(plist, H5F_ACS_METADATA_READ_ATTEMPTS_NAME, attempts) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get the number of metadata read attempts") + + /* If not set, return the default value */ + if(*attempts == H5F_ACS_METADATA_READ_ATTEMPTS_DEF) /* 0 */ + *attempts = H5F_METADATA_READ_ATTEMPTS; + } /* end if */ + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_metadata_read_attempts() */ + + +/*------------------------------------------------------------------------- * Function: H5Pset_obj_flush_cb * * Purpose: Sets the callback function to invoke and the user data when an @@ -3605,6 +3758,407 @@ done: FUNC_LEAVE_API(ret_value) } /* H5Pget_obj_flush_cb() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_mdc_log_options + * + * Purpose: Set metadata cache log options. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_mdc_log_options(hid_t plist_id, hbool_t is_enabled, const char *location, + hbool_t start_on_access) +{ + H5P_genplist_t *plist; /* Property list pointer */ + char * tmp_location; /* Working location pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE4("e", "ib*sb", plist_id, is_enabled, location, start_on_access); + + /* Check arguments */ + if(H5P_DEFAULT == plist_id) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "can't modify default property list") + if(!location) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "location cannot be NULL") + + /* Get the property list structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plist_id is not a file access property list") + + /* Get the current location string and free it */ + if(H5P_get(plist, H5F_ACS_MDC_LOG_LOCATION_NAME, &tmp_location) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get current log location") + H5MM_xfree(tmp_location); + + /* Make a copy of the passed-in location */ + if(NULL == (tmp_location = H5MM_xstrdup(location))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "can't copy passed-in log location") + + /* Set values */ + if(H5P_set(plist, H5F_ACS_USE_MDC_LOGGING_NAME, &is_enabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set is_enabled flag") + if(H5P_set(plist, H5F_ACS_MDC_LOG_LOCATION_NAME, &tmp_location) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set log location") + if(H5P_set(plist, H5F_ACS_START_MDC_LOG_ON_ACCESS_NAME, &start_on_access) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set start_on_access flag") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_mdc_log_options() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_mdc_log_options + * + * Purpose: Get metadata cache log options. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_mdc_log_options(hid_t plist_id, hbool_t *is_enabled, char *location, + size_t *location_size, hbool_t *start_on_access) +{ + H5P_genplist_t *plist; /* Property list pointer */ + char *location_ptr; /* Pointer to location string */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE5("e", "i*b*s*z*b", plist_id, is_enabled, location, location_size, + start_on_access); + + /* Get the property list structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plist_id is not a file access property list") + + /* Get simple values */ + if(is_enabled) + if(H5P_get(plist, H5F_ACS_USE_MDC_LOGGING_NAME, is_enabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get log location") + if(start_on_access) + if(H5P_get(plist, H5F_ACS_START_MDC_LOG_ON_ACCESS_NAME, start_on_access) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get start_on_access flag") + + /* Get the location */ + if(location || location_size) + if(H5P_get(plist, H5F_ACS_MDC_LOG_LOCATION_NAME, &location_ptr) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get log location") + + /* Copy log location to output buffer */ + if(location_ptr && location) + HDmemcpy(location, location_ptr, *location_size); + + /* Get location size, including terminating NULL */ + if(location_size) { + if(location_ptr) + *location_size = HDstrlen(location_ptr) + 1; + else + *location_size = 0; + } + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_mdc_log_options() */ + + +/*------------------------------------------------------------------------- + * Function: H5P_facc_mdc_log_location_enc + * + * Purpose: Callback routine which is called whenever the metadata + * cache log location property in the file access property + * list is encoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P_facc_mdc_log_location_enc(const void *value, void **_pp, size_t *size) +{ + const char *log_location = *(const char * const *)value; + uint8_t **pp = (uint8_t **)_pp; + size_t len = 0; + uint64_t enc_value; + unsigned enc_size; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDcompile_assert(sizeof(size_t) <= sizeof(uint64_t)); + + /* calculate prefix length */ + if(NULL != log_location) + len = HDstrlen(log_location); + + enc_value = (uint64_t)len; + enc_size = H5VM_limit_enc_size(enc_value); + HDassert(enc_size < 256); + + if(NULL != *pp) { + /* encode the length of the prefix */ + *(*pp)++ = (uint8_t)enc_size; + UINT64ENCODE_VAR(*pp, enc_value, enc_size); + + /* encode the prefix */ + if(NULL != log_location) { + HDmemcpy(*(char **)pp, log_location, len); + *pp += len; + } /* end if */ + } /* end if */ + + *size += (1 + enc_size); + if(NULL != log_location) + *size += len; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P_facc_mdc_log_location_enc() */ + + +/*------------------------------------------------------------------------- + * Function: H5P_facc_mdc_log_location_dec + * + * Purpose: Callback routine which is called whenever the metadata + * cache log location property in the file access property + * list is decoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P_facc_mdc_log_location_dec(const void **_pp, void *_value) +{ + char **log_location = (char **)_value; + const uint8_t **pp = (const uint8_t **)_pp; + size_t len; + uint64_t enc_value; /* Decoded property value */ + unsigned enc_size; /* Size of encoded property */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + HDassert(pp); + HDassert(*pp); + HDassert(log_location); + HDcompile_assert(sizeof(size_t) <= sizeof(uint64_t)); + + /* Decode the size */ + enc_size = *(*pp)++; + HDassert(enc_size < 256); + + /* Decode the value */ + UINT64DECODE_VAR(*pp, enc_value, enc_size); + len = enc_value; + + if(0 != len) { + /* Make a copy of the user's prefix string */ + if(NULL == (*log_location = (char *)H5MM_malloc(len + 1))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "memory allocation failed for prefix") + HDstrncpy(*log_location, *(const char **)pp, len); + (*log_location)[len] = '\0'; + + *pp += len; + } /* end if */ + else + *log_location = NULL; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P_facc_mdc_log_location_dec() */ + + +/*------------------------------------------------------------------------- + * Function: H5P_facc_mdc_log_location_del + * + * Purpose: Frees memory used to store the metadata cache log location. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P_facc_mdc_log_location_del(hid_t H5_ATTR_UNUSED prop_id, const char H5_ATTR_UNUSED *name, + size_t H5_ATTR_UNUSED size, void *value) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(value); + + H5MM_xfree(*(void **)value); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P_facc_mdc_log_location_del() */ + + +/*------------------------------------------------------------------------- + * Function: H5P_facc_mdc_log_location_copy + * + * Purpose: Creates a copy of the metadata cache log location string. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P_facc_mdc_log_location_copy(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *value) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(value); + + *(char **)value = H5MM_xstrdup(*(const char **)value); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P_facc_mdc_log_location_copy() */ + + +/*------------------------------------------------------------------------- + * Function: H5P_facc_mdc_log_location_cmp + * + * Purpose: Callback routine which is called whenever the metadata + * cache log location property in the file creation property + * list is compared. + * + * Return: zero if VALUE1 and VALUE2 are equal, non zero otherwise. + * + *------------------------------------------------------------------------- + */ +static int +H5P_facc_mdc_log_location_cmp(const void *value1, const void *value2, size_t H5_ATTR_UNUSED size) +{ + const char *pref1 = *(const char * const *)value1; + const char *pref2 = *(const char * const *)value2; + int ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + if(NULL == pref1 && NULL != pref2) + HGOTO_DONE(1); + if(NULL != pref1 && NULL == pref2) + HGOTO_DONE(-1); + if(NULL != pref1 && NULL != pref2) + ret_value = HDstrcmp(pref1, pref2); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P_facc_mdc_log_location_cmp() */ + + +/*------------------------------------------------------------------------- + * Function: H5P_facc_mdc_log_location_close + * + * Purpose: Frees memory used to store the metadata cache log location + * string + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P_facc_mdc_log_location_close(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *value) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(value); + + H5MM_xfree(*(void **)value); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P_facc_mdc_log_location_close() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_evict_on_close + * + * Purpose: Sets the evict_on_close property value. + * + * When this property is set, closing an HDF5 object will + * cause the object's metadata cache entries to be flushed + * and evicted from the cache. + * + * Currently only implemented for datasets. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Spring 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_evict_on_close(hid_t fapl_id, hbool_t evict_on_close) +{ + H5P_genplist_t *plist; /* property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ib", fapl_id, evict_on_close); + + /* Compare the property list's class against the other class */ + if(TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not a file access plist") + + /* Get the plist structure */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Set values */ + if(H5P_set(plist, H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME, &evict_on_close) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set evict on close property") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_evict_on_close() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_evict_on_close + * + * Purpose: Gets the evict_on_close property value. + * + * When this property is set, closing an HDF5 object will + * cause the object's metadata cache entries to be flushed + * and evicted from the cache. + * + * Currently only implemented for datasets. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Spring 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_evict_on_close(hid_t fapl_id, hbool_t *evict_on_close) +{ + H5P_genplist_t *plist; /* property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*b", fapl_id, evict_on_close); + + /* Compare the property list's class against the other class */ + if(TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not an access plist") + + /* Get the plist structure */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + if(H5P_get(plist, H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME, evict_on_close) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get evict on close property") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_evict_on_close() */ + #ifdef H5_HAVE_PARALLEL /*------------------------------------------------------------------------- diff --git a/src/H5Plapl.c b/src/H5Plapl.c index 266dc63..a0ec7f1 100644 --- a/src/H5Plapl.c +++ b/src/H5Plapl.c @@ -1168,7 +1168,9 @@ H5Pset_elink_acc_flags(hid_t lapl_id, unsigned flags) H5TRACE2("e", "iIu", lapl_id, flags); /* Check that flags are valid */ - if((flags != H5F_ACC_RDWR) && (flags != H5F_ACC_RDONLY) && (flags != H5F_ACC_DEFAULT)) + if(( flags != H5F_ACC_RDWR) && (flags != (H5F_ACC_RDWR | H5F_ACC_SWMR_WRITE)) + && (flags != H5F_ACC_RDONLY) && (flags != (H5F_ACC_RDONLY | H5F_ACC_SWMR_READ)) + && (flags != H5F_ACC_DEFAULT)) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file open flags") /* Get the plist structure */ diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 0318c8f..c736d7b 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -311,7 +311,7 @@ H5_DLL herr_t H5Pget_alignment(hid_t fapl_id, hsize_t *threshold/*out*/, H5_DLL herr_t H5Pset_driver(hid_t plist_id, hid_t driver_id, const void *driver_info); H5_DLL hid_t H5Pget_driver(hid_t plist_id); -H5_DLL void *H5Pget_driver_info(hid_t plist_id); +H5_DLL const void *H5Pget_driver_info(hid_t plist_id); H5_DLL herr_t H5Pset_family_offset(hid_t fapl_id, hsize_t offset); H5_DLL herr_t H5Pget_family_offset(hid_t fapl_id, hsize_t *offset); H5_DLL herr_t H5Pset_multi_type(hid_t fapl_id, H5FD_mem_t type); @@ -351,8 +351,14 @@ H5_DLL herr_t H5Pget_file_image_callbacks(hid_t fapl_id, H5FD_file_image_callbacks_t *callbacks_ptr); H5_DLL herr_t H5Pset_core_write_tracking(hid_t fapl_id, hbool_t is_enabled, size_t page_size); H5_DLL herr_t H5Pget_core_write_tracking(hid_t fapl_id, hbool_t *is_enabled, size_t *page_size); +H5_DLL herr_t H5Pset_metadata_read_attempts(hid_t plist_id, unsigned attempts); +H5_DLL herr_t H5Pget_metadata_read_attempts(hid_t plist_id, unsigned *attempts); H5_DLL herr_t H5Pset_object_flush_cb(hid_t plist_id, H5F_flush_cb_t func, void *udata); H5_DLL herr_t H5Pget_object_flush_cb(hid_t plist_id, H5F_flush_cb_t *func, void **udata); +H5_DLL herr_t H5Pset_mdc_log_options(hid_t plist_id, hbool_t is_enabled, const char *location, hbool_t start_on_access); +H5_DLL herr_t H5Pget_mdc_log_options(hid_t plist_id, hbool_t *is_enabled, char *location, size_t *location_size, hbool_t *start_on_access); +H5_DLL herr_t H5Pset_evict_on_close(hid_t fapl_id, hbool_t evict_on_close); +H5_DLL herr_t H5Pget_evict_on_close(hid_t fapl_id, hbool_t *evict_on_close); #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5Pset_all_coll_metadata_ops(hid_t plist_id, hbool_t is_collective); H5_DLL herr_t H5Pget_all_coll_metadata_ops(hid_t plist_id, hbool_t *is_collective); @@ -148,7 +148,7 @@ H5SM_init(H5F_t *f, H5P_genplist_t * fc_plist, const H5O_loc_t *ext_loc, hid_t d HGOTO_ERROR(H5E_SOHM, H5E_CANTSET, FAIL, "unable to set ring value") /* Initialize master table */ - if(NULL == (table = H5FL_MALLOC(H5SM_master_table_t))) + if(NULL == (table = H5FL_CALLOC(H5SM_master_table_t))) HGOTO_ERROR(H5E_SOHM, H5E_CANTALLOC, FAIL, "memory allocation failed for SOHM table") table->num_indexes = H5F_SOHM_NINDEXES(f); table->table_size = H5SM_TABLE_SIZE(f); @@ -657,7 +657,7 @@ H5SM_create_list(H5F_t *f, H5SM_index_header_t *header, hid_t dxpl_id) num_entries = header->list_max; /* Allocate list in memory */ - if(NULL == (list = H5FL_MALLOC(H5SM_list_t))) + if(NULL == (list = H5FL_CALLOC(H5SM_list_t))) HGOTO_ERROR(H5E_SOHM, H5E_NOSPACE, HADDR_UNDEF, "file allocation failed for SOHM list") if(NULL == (list->messages = (H5SM_sohm_t *)H5FL_ARR_CALLOC(H5SM_sohm_t, num_entries))) HGOTO_ERROR(H5E_SOHM, H5E_NOSPACE, HADDR_UNDEF, "file allocation failed for SOHM list") @@ -2436,7 +2436,7 @@ H5SM_read_mesg(H5F_t *f, const H5SM_sohm_t *mesg, H5HF_t *fheap, HGOTO_ERROR(H5E_SOHM, H5E_CANTLOAD, FAIL, "unable to open object header") /* Load the object header from the cache */ - if(NULL == (oh = H5O_protect(&oloc, dxpl_id, H5AC__READ_ONLY_FLAG))) + if(NULL == (oh = H5O_protect(&oloc, dxpl_id, H5AC__READ_ONLY_FLAG, FALSE))) HGOTO_ERROR(H5E_SOHM, H5E_CANTPROTECT, FAIL, "unable to load object header") } /* end if */ else @@ -2467,7 +2467,7 @@ done: if(oh && oh != open_oh) { if(oh && H5O_unprotect(&oloc, dxpl_id, oh, H5AC__NO_FLAGS_SET) < 0) HDONE_ERROR(H5E_SOHM, H5E_CANTUNPROTECT, FAIL, "unable to release object header") - if(H5O_close(&oloc) < 0) + if(H5O_close(&oloc, NULL) < 0) HDONE_ERROR(H5E_SOHM, H5E_CANTCLOSEOBJ, FAIL, "unable to close object header") } /* end if */ diff --git a/src/H5SMcache.c b/src/H5SMcache.c index 4de86e2..455dd1a 100644 --- a/src/H5SMcache.c +++ b/src/H5SMcache.c @@ -58,20 +58,20 @@ /********************/ /* Metadata cache (H5AC) callbacks */ -static herr_t H5SM__cache_table_get_load_size(const void *udata, size_t *image_len); +static herr_t H5SM__cache_table_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5SM__cache_table_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5SM__cache_table_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5SM__cache_table_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5SM__cache_table_image_len(const void *thing, size_t *image_len); static herr_t H5SM__cache_table_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5SM__cache_table_free_icr(void *thing); -static herr_t H5SM__cache_list_get_load_size(const void *udata, size_t *image_len); +static herr_t H5SM__cache_list_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5SM__cache_list_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5SM__cache_list_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5SM__cache_list_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5SM__cache_list_image_len(const void *thing, size_t *image_len); static herr_t H5SM__cache_list_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5SM__cache_list_free_icr(void *thing); @@ -87,14 +87,15 @@ const H5AC_class_t H5AC_SOHM_TABLE[1] = {{ "shared message table", /* Metadata client name (for debugging) */ H5FD_MEM_SOHM_TABLE, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5SM__cache_table_get_load_size, /* 'get_load_size' callback */ + H5SM__cache_table_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5SM__cache_table_verify_chksum, /* 'verify_chksum' callback */ H5SM__cache_table_deserialize, /* 'deserialize' callback */ H5SM__cache_table_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5SM__cache_table_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5SM__cache_table_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -103,14 +104,15 @@ const H5AC_class_t H5AC_SOHM_LIST[1] = {{ "shared message list", /* Metadata client name (for debugging) */ H5FD_MEM_SOHM_TABLE, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5SM__cache_list_get_load_size, /* 'get_load_size' callback */ + H5SM__cache_list_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5SM__cache_list_verify_chksum, /* 'verify_chksum' callback */ H5SM__cache_list_deserialize, /* 'deserialize' callback */ H5SM__cache_list_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5SM__cache_list_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5SM__cache_list_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -127,11 +129,10 @@ const H5AC_class_t H5AC_SOHM_LIST[1] = {{ /*------------------------------------------------------------------------- - * Function: H5SM__cache_table_get_load_size() + * Function: H5SM__cache_table_get_initial_load_size() * * Purpose: Return the size of the master table of Shared Object Header - * Message indexes on disk. As this cache client doesn't use - * speculative reads, this value should be accurate. + * Message indexes on disk. * * Return: Success: SUCCEED * Failure: FAIL @@ -142,9 +143,9 @@ const H5AC_class_t H5AC_SOHM_LIST[1] = {{ *------------------------------------------------------------------------- */ static herr_t -H5SM__cache_table_get_load_size(const void *_udata, size_t *image_len) +H5SM__cache_table_get_initial_load_size(void *_udata, size_t *image_len) { - const H5SM_table_cache_ud_t *udata = (const H5SM_table_cache_ud_t *)_udata; /* User data for callback */ + const H5SM_table_cache_ud_t *udata = (const H5SM_table_cache_ud_t *)_udata; /* User data for callback */ FUNC_ENTER_STATIC_NOERR @@ -153,10 +154,47 @@ H5SM__cache_table_get_load_size(const void *_udata, size_t *image_len) HDassert(udata->f); HDassert(image_len); + /* Set the image length size */ *image_len = H5SM_TABLE_SIZE(udata->f); FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5SM__cache_table_get_load_size() */ +} /* end H5SM__cache_table_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5SM__cache_table_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +htri_t +H5SM__cache_table_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5SM__cache_table_verify_chksum() */ /*------------------------------------------------------------------------- @@ -184,7 +222,6 @@ H5SM__cache_table_deserialize(const void *_image, size_t len, void *_udata, H5SM_table_cache_ud_t *udata = (H5SM_table_cache_ud_t *)_udata; /* Pointer to user data */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into input buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ size_t u; /* Counter variable for index headers */ void *ret_value = NULL; /* Return value */ @@ -260,19 +297,14 @@ H5SM__cache_table_deserialize(const void *_image, size_t len, void *_udata, table->indexes[u].list_size = H5SM_LIST_SIZE(f, table->indexes[u].list_max); } /* end for */ + /* checksum verification already done in verify_chksum cb */ + /* Read in checksum */ UINT32DECODE(image, stored_chksum); /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == table->table_size); - /* Compute checksum on entire header */ - computed_chksum = H5_checksum_metadata(_image, (table->table_size - H5SM_SIZEOF_CHECKSUM), 0); - - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_SOHM, H5E_BADVALUE, NULL, "incorrect metadata checksum for shared message table") - /* Set return value */ ret_value = table; @@ -301,8 +333,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5SM__cache_table_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5SM__cache_table_image_len(const void *_thing, size_t *image_len) { const H5SM_master_table_t *table = (const H5SM_master_table_t *)_thing; /* Shared message table to query */ @@ -319,10 +350,6 @@ H5SM__cache_table_image_len(const void *_thing, size_t *image_len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5SM__cache_table_image_len() */ -/***************************************/ -/* no H5SM_cache_table_pre_serialize() */ -/***************************************/ - /*------------------------------------------------------------------------- * Function: H5SM__cache_table_serialize @@ -451,7 +478,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5SM__cache_list_get_load_size() + * Function: H5SM__cache_list_get_initial_load_size() * * Purpose: Return the on disk size of list of SOHM messages. In this case, * we simply look up the size in the user data, and return that value @@ -466,9 +493,9 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5SM__cache_list_get_load_size(const void *_udata, size_t *image_len) +H5SM__cache_list_get_initial_load_size(void *_udata, size_t *image_len) { - const H5SM_list_cache_ud_t *udata = (const H5SM_list_cache_ud_t *)_udata; /* User data for callback */ + const H5SM_list_cache_ud_t *udata = (const H5SM_list_cache_ud_t *)_udata; /* User data for callback */ FUNC_ENTER_STATIC_NOERR @@ -478,10 +505,53 @@ H5SM__cache_list_get_load_size(const void *_udata, size_t *image_len) HDassert(udata->header->list_size > 0); HDassert(image_len); + /* Set the image length size */ *image_len = udata->header->list_size; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5SM__cache_list_get_load_size() */ +} /* end H5SM__cache_list_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5SM__cache_list_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +htri_t +H5SM__cache_list_verify_chksum(const void *_image, size_t H5_ATTR_UNUSED len, void *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5SM_list_cache_ud_t *udata = (H5SM_list_cache_ud_t *)_udata; /* User data for callback */ + size_t chk_size; /* Exact size of the node with checksum at the end */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + HDassert(udata); + + /* Exact size with checksum at the end */ + chk_size = H5SM_LIST_SIZE(udata->f, udata->header->num_messages); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, chk_size, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5SM__cache_list_verify_chksum() */ /*------------------------------------------------------------------------- @@ -508,7 +578,6 @@ H5SM__cache_list_deserialize(const void *_image, size_t len, void *_udata, H5SM_bt2_ctx_t ctx; /* Message encoding context */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into input buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ size_t u; /* Counter variable for messages in list */ void *ret_value = NULL; /* Return value */ @@ -546,19 +615,14 @@ H5SM__cache_list_deserialize(const void *_image, size_t len, void *_udata, image += H5SM_SOHM_ENTRY_SIZE(udata->f); } /* end for */ + /* checksum verification already done in verify_chksum cb */ + /* Read in checksum */ UINT32DECODE(image, stored_chksum); /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) <= udata->header->list_size); - /* Compute checksum on entire header */ - computed_chksum = H5_checksum_metadata(_image, ((size_t)(image - (const uint8_t *)_image) - H5SM_SIZEOF_CHECKSUM), 0); - - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_SOHM, H5E_BADVALUE, NULL, "incorrect metadata checksum for shared message list") - /* Initialize the rest of the array */ for(u = udata->header->num_messages; u < udata->header->list_max; u++) list->messages[u].location = H5SM_NO_LOC; @@ -591,8 +655,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5SM__cache_list_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5SM__cache_list_image_len(const void *_thing, size_t *image_len) { const H5SM_list_t *list = (const H5SM_list_t *)_thing; /* Shared message list to query */ @@ -610,10 +673,6 @@ H5SM__cache_list_image_len(const void *_thing, size_t *image_len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5SM__cache_list_image_len() */ -/**************************************/ -/* no H5SM_cache_list_pre_serialize() */ -/**************************************/ - /*------------------------------------------------------------------------- * Function: H5SM__cache_list_serialize diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index b00dc6b..d00b5be 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -183,8 +183,8 @@ typedef struct H5S_sel_iter_op_t { #define H5S_SELECT_IS_SINGLE(S) (H5S_select_is_single(S)) #define H5S_SELECT_IS_REGULAR(S) (H5S_select_is_regular(S)) #define H5S_SELECT_ADJUST_U(S,O) (H5S_select_adjust_u(S, O)) -#define H5S_SELECT_PROJECT_SCALAR(S,O) (H5S_select_project_scalar)(S, O)) -#define H5S_SELECT_PROJECT_SIMPLE(S,NS,O) (H5S_select_project_simple)(S, NS, O)) +#define H5S_SELECT_PROJECT_SCALAR(S,O) (H5S_select_project_scalar(S, O)) +#define H5S_SELECT_PROJECT_SIMPLE(S,NS,O) (H5S_select_project_simple(S, NS, O)) #define H5S_SELECT_ITER_COORDS(ITER,COORDS) (H5S_select_iter_coords(ITER,COORDS)) #define H5S_SELECT_ITER_BLOCK(ITER,START,END) (H5S_select_iter_block(ITER,START,END)) #define H5S_SELECT_ITER_NELMTS(ITER) (H5S_select_iter_nelmts(ITER)) @@ -3525,7 +3525,7 @@ H5T__free(H5T_t *dt) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTRELEASE, FAIL, "can't decrement count for object") if(H5FO_delete(dt->sh_loc.file, H5AC_ind_read_dxpl_id, dt->sh_loc.u.loc.oh_addr) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTRELEASE, FAIL, "can't remove datatype from list of open objects") - if(H5O_close(&dt->oloc) < 0) + if(H5O_close(&dt->oloc, NULL) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to close data type object header") dt->shared->state = H5T_STATE_NAMED; } /* end if */ @@ -3645,7 +3645,7 @@ H5T_close(H5T_t *dt) /* Check reference count for this object in the top file */ if(H5FO_top_count(dt->sh_loc.file, dt->sh_loc.u.loc.oh_addr) == 0) { /* Close object location for named datatype */ - if(H5O_close(&dt->oloc) < 0) + if(H5O_close(&dt->oloc, NULL) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to close") } /* end if */ else diff --git a/src/H5Tcommit.c b/src/H5Tcommit.c index 07fe371..3ded7af 100644 --- a/src/H5Tcommit.c +++ b/src/H5Tcommit.c @@ -203,7 +203,7 @@ done: HDONE_ERROR(H5E_DATASET, H5E_CANTRELEASE, FAIL, "can't remove dataset from list of open objects") /* Close the datatype object */ - if(H5O_close(&(dt->oloc)) < 0) + if(H5O_close(&(dt->oloc), NULL) < 0) HDONE_ERROR(H5E_DATATYPE, H5E_CLOSEERROR, FAIL, "unable to release object header") /* Remove the datatype's object header from the file */ @@ -401,7 +401,7 @@ done: if((type->shared->state == H5T_STATE_TRANSIENT || type->shared->state == H5T_STATE_RDONLY) && (type->sh_loc.type == H5O_SHARE_TYPE_COMMITTED)) { if(H5O_dec_rc_by_loc(&(type->oloc), dxpl_id) < 0) HDONE_ERROR(H5E_DATATYPE, H5E_CANTDEC, FAIL, "unable to decrement refcount on newly created object") - if(H5O_close(&(type->oloc)) < 0) + if(H5O_close(&(type->oloc), NULL) < 0) HDONE_ERROR(H5E_DATATYPE, H5E_CLOSEERROR, FAIL, "unable to release object header") if(H5O_delete(file, dxpl_id, type->sh_loc.u.loc.oh_addr) < 0) HDONE_ERROR(H5E_DATATYPE, H5E_CANTDELETE, FAIL, "unable to delete object header") @@ -823,7 +823,7 @@ H5T_open_oid(const H5G_loc_t *loc, hid_t dxpl_id) done: if(ret_value == NULL) if(dt == NULL) - H5O_close(loc->oloc); + H5O_close(loc->oloc, NULL); FUNC_LEAVE_NOAPI(ret_value) } /* end H5T_open_oid() */ diff --git a/src/H5Zscaleoffset.c b/src/H5Zscaleoffset.c index 91a6c00..1cca9b1 100644 --- a/src/H5Zscaleoffset.c +++ b/src/H5Zscaleoffset.c @@ -667,58 +667,6 @@ H5Z_class2_t H5Z_SCALEOFFSET[1] = {{ H5Z_scaleoffset_modify_4(i, type, pow_fun, buf, d_nelmts, min, D_val) \ } -/* Include our own rounding routine and alias it to the stdc macros, if they - * aren't available. - */ -#if !(defined(H5_HAVE_LLROUND) && defined(H5_HAVE_LLROUNDF) && defined(H5_HAVE_LROUND) && defined(H5_HAVE_LROUNDF) && defined(H5_HAVE_ROUND) && defined(H5_HAVE_ROUNDF)) -/* Round a floating-point value to the nearest integer value 4/19/05 */ -/* rounding to the bigger absolute value if val is in the middle, - 0.5 -> 1, -0.5 ->-1 -5/9/05, KY */ -static double -H5Z__scaleoffset_rnd(double val) -{ - double u_val, l_val; - - u_val = HDceil(val); - l_val = HDfloor(val); - - if(val > 0) { - if((u_val - val) <= (val - l_val)) - return u_val; - else - return l_val; - } /* end if */ - else { - if((val - l_val) <= (u_val - val)) - return l_val; - else - return u_val; - } -} /* H5Z__scaleoffset_rnd() */ - -/* Alias rounding macros to routine above, if not defined */ -#if !defined(H5_HAVE_LLROUND) -#define llround(x) H5Z__scaleoffset_rnd(x) -#endif /* !defined(H5_HAVE_LLROUND) */ -#if !defined(H5_HAVE_LLROUNDF) -#define llroundf(x) H5Z__scaleoffset_rnd(x) -#endif /* !defined(H5_HAVE_LLROUNDF) */ -#if !defined(H5_HAVE_LROUND) -#define lround(x) H5Z__scaleoffset_rnd(x) -#endif /* !defined(H5_HAVE_LROUND) */ -#if !defined(H5_HAVE_LROUNDF) -#define lroundf(x) H5Z__scaleoffset_rnd(x) -#endif /* !defined(H5_HAVE_LROUNDF) */ -#if !defined(H5_HAVE_ROUND) -#define round(x) H5Z__scaleoffset_rnd(x) -#endif /* !defined(H5_HAVE_ROUND) */ -#if !defined(H5_HAVE_ROUNDF) -#define roundf(x) H5Z__scaleoffset_rnd(x) -#endif /* !defined(H5_HAVE_ROUNDF) */ - -#endif /* !(defined(H5_HAVE_LLROUND) && defined(H5_HAVE_LLROUNDF) && defined(H5_HAVE_LROUND) && defined(H5_HAVE_LROUNDF) && defined(H5_HAVE_ROUND) && defined(H5_HAVE_ROUNDF)) */ - /*------------------------------------------------------------------------- * Function: H5Z_can_apply_scaleoffset diff --git a/src/H5Zszip.c b/src/H5Zszip.c index 2c2ed07..557e923 100644 --- a/src/H5Zszip.c +++ b/src/H5Zszip.c @@ -298,7 +298,7 @@ H5Z_filter_szip (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], /* Check arguments */ if (cd_nelmts!=4) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid deflate aggression level") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid number of filter parameters") /* Copy the filter parameters into the szip parameter block */ H5_CHECKED_ASSIGN(sz_param.options_mask, int, cd_values[H5Z_SZIP_PARM_MASK], unsigned); diff --git a/src/H5err.txt b/src/H5err.txt index 8219408..9531df9 100644 --- a/src/H5err.txt +++ b/src/H5err.txt @@ -169,12 +169,15 @@ MINOR, CACHE, H5E_CANTUNPROTECT, Unable to unprotect metadata MINOR, CACHE, H5E_CANTPIN, Unable to pin cache entry MINOR, CACHE, H5E_CANTUNPIN, Unable to un-pin cache entry MINOR, CACHE, H5E_CANTMARKDIRTY, Unable to mark a pinned entry as dirty +MINOR, CACHE, H5E_CANTMARKCLEAN, Unable to mark a pinned entry as clean MINOR, CACHE, H5E_CANTDIRTY, Unable to mark metadata as dirty +MINOR, CACHE, H5E_CANTCLEAN, Unable to mark metadata as clean MINOR, CACHE, H5E_CANTEXPUNGE, Unable to expunge a metadata cache entry MINOR, CACHE, H5E_CANTRESIZE, Unable to resize a metadata cache entry MINOR, CACHE, H5E_CANTDEPEND, Unable to create a flush dependency MINOR, CACHE, H5E_CANTUNDEPEND, Unable to destroy a flush dependency MINOR, CACHE, H5E_CANTNOTIFY, Unable to notify object about action +MINOR, CACHE, H5E_LOGFAIL, Failure in the cache logging framework MINOR, CACHE, H5E_CANTCORK, Unable to cork an object MINOR, CACHE, H5E_CANTUNCORK, Unable to uncork an object diff --git a/src/H5private.h b/src/H5private.h index 4c40965..47f6d78 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -813,7 +813,8 @@ H5_DLL H5_ATTR_CONST int Nflock(int fd, int operation); /* NOTE: flock(2) is not present on all POSIX systems. * If it is not present, we try a flock() equivalent based on * fcntl(2), then fall back to a function that always fails if - * it is not present at all. + * it is not present at all (Windows uses a separate Wflock() + * function). */ #if defined(H5_HAVE_FLOCK) #define HDflock(F,L) flock(F,L) @@ -1114,6 +1115,9 @@ typedef off_t h5_stat_size_t; #ifndef HDmodf #define HDmodf(X,Y) modf(X,Y) #endif /* HDmodf */ +#ifndef HDnanosleep + #define HDnanosleep(N, O) nanosleep(N, O) +#endif /* HDnanosleep */ #ifndef HDopen #ifdef _O_BINARY #define HDopen(S,F,M) open(S,F|_O_BINARY,M) @@ -2597,6 +2601,8 @@ H5_DLL uint32_t H5_hash_string(const char *str); /* Time related routines */ H5_DLL time_t H5_make_time(struct tm *tm); +H5_DLL void H5_nanosleep(uint64_t nanosec); +H5_DLL double H5_get_time(void); /* Functions for building paths, etc. */ H5_DLL herr_t H5_build_extpath(const char *name, char **extpath /*out*/); diff --git a/src/H5public.h b/src/H5public.h index 2f21648..f0eb63a 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -94,10 +94,10 @@ extern "C" { /* Version numbers */ #define H5_VERS_MAJOR 1 /* For major interface/format changes */ #define H5_VERS_MINOR 9 /* For minor interface/format changes */ -#define H5_VERS_RELEASE 234 /* For tweaks, bug-fixes, or development */ +#define H5_VERS_RELEASE 236 /* For tweaks, bug-fixes, or development */ #define H5_VERS_SUBRELEASE "" /* For pre-releases like snap0 */ /* Empty string for real releases. */ -#define H5_VERS_INFO "HDF5 library version: 1.9.234" /* Full version string */ +#define H5_VERS_INFO "HDF5 library version: 1.9.236" /* Full version string */ #define H5check() H5check_version(H5_VERS_MAJOR,H5_VERS_MINOR, \ H5_VERS_RELEASE) diff --git a/src/H5system.c b/src/H5system.c index e17373c..ac323c0 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -911,6 +911,76 @@ Wflock(int fd, int operation) { return 0; } /* end Wflock() */ + + /*-------------------------------------------------------------------------- + * Function: Wnanosleep + * + * Purpose: Sleep for a given # of nanoseconds (Windows version) + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Fall 2016 + *-------------------------------------------------------------------------- + */ +int +Wnanosleep(const struct timespec *req, struct timespec *rem) +{ + /* XXX: Currently just a placeholder */ + return 0; + +} /* end Wnanosleep() */ + + +/*------------------------------------------------------------------------- + * Function: Wllround, Wllroundf, Wlround, Wlroundf, Wround, Wroundf + * + * Purpose: Wrapper function for round functions for use with VS2012 + * and earlier. + * + * Return: The rounded value that was passed in. + * + * Programmer: Dana Robinson + * December 2016 + * + *------------------------------------------------------------------------- + */ +long long +Wllround(double arg) +{ + return (long long)(arg < 0.0 ? HDceil(arg - 0.5) : HDfloor(arg + 0.5)); +} + +long long +Wllroundf(float arg) +{ + return (long long)(arg < 0.0F ? HDceil(arg - 0.5F) : HDfloor(arg + 0.5F)); +} + +long +Wlround(double arg) +{ + return (long)(arg < 0.0 ? HDceil(arg - 0.5) : HDfloor(arg + 0.5)); +} + +long +Wlroundf(float arg) +{ + return (long)(arg < 0.0F ? HDceil(arg - 0.5F) : HDfloor(arg + 0.5F)); +} + +double +Wround(double arg) +{ + return arg < 0.0 ? HDceil(arg - 0.5) : HDfloor(arg + 0.5); +} + +float +Wroundf(float arg) +{ + return arg < 0.0F ? HDceil(arg - 0.5F) : HDfloor(arg + 0.5F); +} + #endif /* H5_HAVE_WIN32_API */ @@ -1106,5 +1176,65 @@ H5_combine_path(const char* path1, const char* path2, char **full_name /*out*/) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5_combine_name() */ +} /* end H5_combine_path() */ + + +/*-------------------------------------------------------------------------- + * Function: H5_nanosleep + * + * Purpose: Sleep for a given # of nanoseconds + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * October 01, 2016 + *-------------------------------------------------------------------------- + */ +void +H5_nanosleep(uint64_t nanosec) +{ + struct timespec sleeptime; /* Struct to hold time to sleep */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Set up time to sleep */ + sleeptime.tv_sec = 0; + sleeptime.tv_nsec = (long)nanosec; + + HDnanosleep(&sleeptime, NULL); + + FUNC_LEAVE_NOAPI_VOID +} /* end H5_nanosleep() */ + + +/*-------------------------------------------------------------------------- + * Function: H5_get_time + * + * Purpose: Get the current time, as the time of seconds after the UNIX epoch + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * October 05, 2016 + *-------------------------------------------------------------------------- + */ +double +H5_get_time(void) +{ +#ifdef H5_HAVE_GETTIMEOFDAY + struct timeval curr_time; +#endif /* H5_HAVE_GETTIMEOFDAY */ + double ret_value = (double)0.0f; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#ifdef H5_HAVE_GETTIMEOFDAY + HDgettimeofday(&curr_time, NULL); + + ret_value = (double)curr_time.tv_sec + ((double)curr_time.tv_usec / (double)1000000.0f); +#endif /* H5_HAVE_GETTIMEOFDAY */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5_get_time() */ + diff --git a/src/H5win32defs.h b/src/H5win32defs.h index a757832..63c3a16 100644 --- a/src/H5win32defs.h +++ b/src/H5win32defs.h @@ -42,6 +42,7 @@ typedef __int64 h5_stat_size_t; #define HDlseek(F,O,W) _lseeki64(F,O,W) #define HDlstat(S,B) _lstati64(S,B) #define HDmkdir(S,M) _mkdir(S) +#define HDnanosleep(N, O) Wnanosleep(N, O) #define HDoff_t __int64 /* _O_BINARY must be set in Windows to avoid CR-LF <-> LF EOL * transformations when performing I/O. @@ -65,13 +66,33 @@ typedef __int64 h5_stat_size_t; */ #define HDmemset(X,C,Z) memset((void*)(X),C,Z) -#endif /* H5_HAVE_VISUAL_STUDIO */ - struct timezone { int tz_minuteswest; int tz_dsttime; }; +/* time.h before VS2015 does not include timespec */ +#if (_MSC_VER < 1900) +struct timespec +{ + time_t tv_sec; // Seconds - >= 0 + long tv_nsec; // Nanoseconds - [0, 999999999] +}; +#endif /* MSC_VER < 1900 */ + +/* The round functions do not exist in VS2012 and earlier */ +#if (_MSC_VER <= 1700) +#define HDllround(V) Wllround(V) +#define HDllroundf(V) Wllroundf(V) +#define HDlround(V) Wlround(V) +#define HDlroundf(V) Wlroundf(V) +#define HDround(V) Wround(V) +#define HDroundf(V) Wroundf(V) +#endif /* MSC_VER < 1700 */ + +#endif /* H5_HAVE_VISUAL_STUDIO */ + + #ifdef __cplusplus extern "C" { #endif /* __cplusplus */ @@ -81,6 +102,19 @@ extern "C" { H5_DLL char* Wgetlogin(void); H5_DLL int c99_snprintf(char* str, size_t size, const char* format, ...); H5_DLL int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap); + H5_DLL int Wnanosleep(const struct timespec *req, struct timespec *rem); + + /* Round functions only needed for VS2012 and earlier. + * They are always built to ensure they don't go stale and + * can be deleted (along with their #defines, above) when we + * drop VS2012 support. + */ + H5_DLL long long Wllround(double arg); + H5_DLL long long Wllroundf(float arg); + H5_DLL long Wlround(double arg); + H5_DLL long Wlroundf(float arg); + H5_DLL double Wround(double arg); + H5_DLL float Wroundf(float arg); #ifdef __cplusplus } #endif /* __cplusplus */ diff --git a/src/Makefile.am b/src/Makefile.am index 7facf9a..939e151 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -42,10 +42,11 @@ DISTCLEANFILES=H5pubconf.h # library sources libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5A.c H5Abtree2.c H5Adense.c H5Adeprec.c H5Aint.c H5Atest.c \ - H5AC.c \ + H5AC.c H5ACdbg.c H5AClog.c H5ACproxy_entry.c \ H5B.c H5Bcache.c H5Bdbg.c \ - H5B2.c H5B2cache.c H5B2dbg.c H5B2hdr.c H5B2int.c H5B2stat.c H5B2test.c \ - H5C.c H5Cepoch.c H5Cquery.c H5Ctag.c H5Ctest.c \ + H5B2.c H5B2cache.c H5B2dbg.c H5B2hdr.c H5B2int.c H5B2internal.c \ + H5B2leaf.c H5B2stat.c H5B2test.c \ + H5C.c H5Cdbg.c H5Cepoch.c H5Clog.c H5Cquery.c H5Ctag.c H5Ctest.c \ H5CS.c \ H5D.c H5Dbtree.c H5Dbtree2.c H5Dchunk.c H5Dcompact.c H5Dcontig.c H5Ddbg.c \ H5Ddeprec.c H5Dearray.c H5Defl.c H5Dfarray.c H5Dfill.c H5Dint.c \ @@ -60,11 +61,12 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5Fmount.c H5Fquery.c \ H5Fsfile.c H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \ H5FA.c H5FAcache.c H5FAdbg.c H5FAdblock.c H5FAdblkpage.c H5FAhdr.c \ - H5FAstat.c H5FAtest.c \ + H5FAint.c H5FAstat.c H5FAtest.c \ H5FD.c H5FDcore.c \ H5FDfamily.c H5FDint.c H5FDlog.c \ - H5FDmulti.c H5FDsec2.c H5FDspace.c H5FDstdio.c \ - H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSsection.c H5FSstat.c H5FStest.c \ + H5FDmulti.c H5FDsec2.c H5FDspace.c H5FDstdio.c H5FDtest.c \ + H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSint.c H5FSsection.c \ + H5FSstat.c H5FStest.c \ H5G.c H5Gbtree2.c H5Gcache.c \ H5Gcompact.c H5Gdense.c H5Gdeprec.c H5Gent.c \ H5Gint.c H5Glink.c \ |