diff options
author | Dana Robinson <43805+derobins@users.noreply.github.com> | 2020-11-17 13:47:52 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-11-17 13:47:52 (GMT) |
commit | 5d0be1b196570ac2c08ffe0b61ac6a45b6ab7f3f (patch) | |
tree | dbe8a76bcb211babe9900647eb1f515d8623ad9b /src | |
parent | ec48fb8dfdb4f2f59f22a5efb6b950aafcac449d (diff) | |
download | hdf5-5d0be1b196570ac2c08ffe0b61ac6a45b6ab7f3f.zip hdf5-5d0be1b196570ac2c08ffe0b61ac6a45b6ab7f3f.tar.gz hdf5-5d0be1b196570ac2c08ffe0b61ac6a45b6ab7f3f.tar.bz2 |
Manual sync with develop (#95)
Brings all features from develop. Note that RELEASE.txt has not been
updated (will be done in a future PR).
Diffstat (limited to 'src')
57 files changed, 3799 insertions, 869 deletions
@@ -449,6 +449,14 @@ done: * Programmer: Robb Matzke * Jul 9 1997 * + * Changes: + * + * In the parallel case, added code to setup the MDC slist + * before the call to H5AC__flush_entries() and take it down + * afterwards. + * + * JRM -- 7/29/20 + * *------------------------------------------------------------------------- */ herr_t @@ -477,15 +485,20 @@ H5AC_dest(H5F_t *f) if (H5C_get_logging_status(f->shared->cache, &log_enabled, &curr_logging) < 0) HGOTO_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to get logging status") - if (log_enabled && curr_logging) + if (log_enabled && curr_logging) { + if (H5C_log_write_destroy_cache_msg(f->shared->cache) < 0) HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + } + /* Tear down logging */ - if (log_enabled) + if (log_enabled) { + if (H5C_log_tear_down(f->shared->cache) < 0) HGOTO_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "mdc logging tear-down failed") + } #ifdef H5_HAVE_PARALLEL @@ -495,19 +508,46 @@ H5AC_dest(H5F_t *f) HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5C_clear_coll_entries() failed") aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(f->shared->cache); - if (aux_ptr) + + if (aux_ptr) { + /* Sanity check */ HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - /* If the file was opened R/W, attempt to flush all entries - * from rank 0 & Bcast clean list to other ranks. - * - * Must not flush in the R/O case, as this will trigger the - * free space manager settle routines. - */ - if (H5F_ACC_RDWR & H5F_INTENT(f)) - if (H5AC__flush_entries(f) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush") + /* If the file was opened R/W, attempt to flush all entries + * from rank 0 & Bcast clean list to other ranks. + * + * Must not flush in the R/O case, as this will trigger the + * free space manager settle routines. + * + * Must also enable the skip list before the call to + * H5AC__flush_entries() and disable it afterwards, as the + * skip list will be disabled after the previous flush. + * + * Note that H5C_dest() does slist setup and take down as well. + * Unfortunately, we can't do the setup and take down just once, + * as H5C_dest() is called directly in the test code. + * + * Fortunately, the cache should be clean or close to it at this + * point, so the overhead should be minimal. + */ + if (H5F_ACC_RDWR & H5F_INTENT(f)) { + + /* enable and load the slist */ + if (H5C_set_slist_enabled(f->shared->cache, TRUE, FALSE) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "set slist enabled failed") + + if (H5AC__flush_entries(f) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush") + + /* disable the slist -- should be empty */ + if (H5C_set_slist_enabled(f->shared->cache, FALSE, FALSE) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "disable slist failed") + } + } #endif /* H5_HAVE_PARALLEL */ /* Destroy the cache */ @@ -902,9 +942,10 @@ H5AC_mark_entry_dirty(void *thing) done: /* If currently logging, generate a message */ - if (cache_ptr->log_info->logging) - if (H5C_log_write_mark_entry_dirty_msg(cache_ptr, entry_ptr, ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache_ptr != NULL && cache_ptr->log_info != NULL) + if (cache_ptr->log_info->logging) + if (H5C_log_write_mark_entry_dirty_msg(cache_ptr, entry_ptr, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_mark_entry_dirty() */ @@ -954,9 +995,10 @@ H5AC_mark_entry_clean(void *thing) done: /* If currently logging, generate a message */ - if (cache_ptr->log_info->logging) - if (H5C_log_write_mark_entry_clean_msg(cache_ptr, entry_ptr, ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache_ptr != NULL && cache_ptr->log_info != NULL) + if (cache_ptr->log_info->logging) + if (H5C_log_write_mark_entry_clean_msg(cache_ptr, entry_ptr, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_mark_entry_clean() */ @@ -995,9 +1037,10 @@ H5AC_mark_entry_unserialized(void *thing) done: /* If currently logging, generate a message */ - if (cache_ptr->log_info->logging) - if (H5C_log_write_mark_unserialized_entry_msg(cache_ptr, entry_ptr, ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache_ptr != NULL && cache_ptr->log_info != NULL) + if (cache_ptr->log_info->logging) + if (H5C_log_write_mark_unserialized_entry_msg(cache_ptr, entry_ptr, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_mark_entry_unserialized() */ @@ -1035,9 +1078,10 @@ H5AC_mark_entry_serialized(void *thing) done: /* If currently logging, generate a message */ - if (cache_ptr->log_info->logging) - if (H5C_log_write_mark_serialized_entry_msg(cache_ptr, entry_ptr, ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache_ptr != NULL && cache_ptr->log_info != NULL) + if (cache_ptr->log_info->logging) + if (H5C_log_write_mark_serialized_entry_msg(cache_ptr, entry_ptr, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_mark_entry_serialized() */ @@ -1134,9 +1178,10 @@ H5AC_pin_protected_entry(void *thing) done: /* If currently logging, generate a message */ - if (cache_ptr->log_info->logging) - if (H5C_log_write_pin_entry_msg(cache_ptr, entry_ptr, ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache_ptr != NULL && cache_ptr->log_info != NULL) + if (cache_ptr->log_info->logging) + if (H5C_log_write_pin_entry_msg(cache_ptr, entry_ptr, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_pin_protected_entry() */ @@ -1179,6 +1224,107 @@ done: } /* H5AC_prep_for_file_close() */ /*------------------------------------------------------------------------- + * + * Function: H5AC_prep_for_file_flush + * + * Purpose: This function should be called just prior to the first + * call to H5AC_flush() during a file flush. + * + * Its purpose is to handly any setup required prior to + * metadata cache flush. + * + * Initially, this means setting up the slist prior to the + * flush. We do this in a seperate call because + * H5F__flush_phase2() make repeated calls to H5AC_flush(). + * Handling this detail in separate calls allows us to avoid + * the overhead of setting up and taking down the skip list + * repeatedly. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 5/5/20 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_prep_for_file_flush(H5F_t *f) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->cache); + + if (H5C_set_slist_enabled(f->shared->cache, TRUE, FALSE) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "slist enabled failed") + +done: + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5AC_prep_for_file_flush() */ + +/*------------------------------------------------------------------------- + * + * Function: H5AC_secure_from_file_flush + * + * Purpose: This function should be called just after the last + * call to H5AC_flush() during a file flush. + * + * Its purpose is to perform any necessary cleanup after the + * metadata cache flush. + * + * The objective of the call is to allow the metadata cache + * to do any necessary necessary cleanup work after a cache + * flush. + * + * Initially, this means taking down the slist after the + * flush. We do this in a seperate call because + * H5F__flush_phase2() make repeated calls to H5AC_flush(). + * Handling this detail in separate calls allows us to avoid + * the overhead of setting up and taking down the skip list + * repeatedly. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 5/5/20 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_secure_from_file_flush(H5F_t *f) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->cache); + + if (H5C_set_slist_enabled(f->shared->cache, FALSE, FALSE) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "slist enabled failed") + +done: + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5AC_secure_from_file_flush() */ + +/*------------------------------------------------------------------------- + * * Function: H5AC_create_flush_dependency() * * Purpose: Create a flush dependency between two entries in the metadata @@ -1214,10 +1360,11 @@ H5AC_create_flush_dependency(void *parent_thing, void *child_thing) done: /* If currently logging, generate a message */ - if (cache_ptr->log_info->logging) - if (H5C_log_write_create_fd_msg(cache_ptr, (H5AC_info_t *)parent_thing, (H5AC_info_t *)child_thing, - ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache_ptr != NULL && cache_ptr->log_info != NULL) + if (cache_ptr->log_info->logging) + if (H5C_log_write_create_fd_msg(cache_ptr, (H5AC_info_t *)parent_thing, + (H5AC_info_t *)child_thing, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_create_flush_dependency() */ @@ -1345,9 +1492,10 @@ H5AC_resize_entry(void *thing, size_t new_size) done: /* If currently logging, generate a message */ - if (cache_ptr->log_info->logging) - if (H5C_log_write_resize_entry_msg(cache_ptr, entry_ptr, new_size, ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache_ptr != NULL && cache_ptr->log_info != NULL) + if (cache_ptr->log_info->logging) + if (H5C_log_write_resize_entry_msg(cache_ptr, entry_ptr, new_size, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_resize_entry() */ @@ -1387,9 +1535,10 @@ H5AC_unpin_entry(void *thing) done: /* If currently logging, generate a message */ - if (cache_ptr->log_info->logging) - if (H5C_log_write_unpin_entry_msg(cache_ptr, entry_ptr, ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache_ptr != NULL && cache_ptr->log_info != NULL) + if (cache_ptr->log_info->logging) + if (H5C_log_write_unpin_entry_msg(cache_ptr, entry_ptr, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_unpin_entry() */ @@ -1429,10 +1578,11 @@ H5AC_destroy_flush_dependency(void *parent_thing, void *child_thing) done: /* If currently logging, generate a message */ - if (cache_ptr->log_info->logging) - if (H5C_log_write_destroy_fd_msg(cache_ptr, (H5AC_info_t *)parent_thing, (H5AC_info_t *)child_thing, - ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache_ptr != NULL && cache_ptr->log_info != NULL) + if (cache_ptr->log_info->logging) + if (H5C_log_write_destroy_fd_msg(cache_ptr, (H5AC_info_t *)parent_thing, + (H5AC_info_t *)child_thing, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_destroy_flush_dependency() */ @@ -2581,9 +2731,10 @@ H5AC_remove_entry(void *_entry) done: /* If currently logging, generate a message */ - if (cache->log_info->logging) - if (H5C_log_write_remove_entry_msg(cache, entry, ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache != NULL && cache->log_info != NULL) + if (cache->log_info->logging) + if (H5C_log_write_remove_entry_msg(cache, entry, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_remove_entry() */ diff --git a/src/H5ACmpio.c b/src/H5ACmpio.c index 7ffa0bc..a806b35 100644 --- a/src/H5ACmpio.c +++ b/src/H5ACmpio.c @@ -773,7 +773,7 @@ H5AC__log_dirtied_entry(const H5AC_info_t *entry_ptr) else { aux_ptr->dirty_bytes += entry_ptr->size; #if H5AC_DEBUG_DIRTY_BYTES_CREATION - aux_ptr->unprotect_dirty_bytes += entry_size; + aux_ptr->unprotect_dirty_bytes += entry_ptr->size; aux_ptr->unprotect_dirty_bytes_updates += 1; #endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ } /* end else */ @@ -971,7 +971,7 @@ H5AC__log_inserted_entry(const H5AC_info_t *entry_ptr) aux_ptr->dirty_bytes += entry_ptr->size; #if H5AC_DEBUG_DIRTY_BYTES_CREATION - aux_ptr->insert_dirty_bytes += size; + aux_ptr->insert_dirty_bytes += entry_ptr->size; aux_ptr->insert_dirty_bytes_updates += 1; #endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ @@ -1849,6 +1849,8 @@ done: * Programmer: John Mainzer * April 28, 2010 * + * Changes: None. + * *------------------------------------------------------------------------- */ static herr_t @@ -1877,10 +1879,12 @@ H5AC__rsp__p0_only__flush(H5F_t *f) * However, when flushing from within the close operation from a file, * it's possible to skip this barrier (on the second flush of the cache). */ - if (!H5CX_get_mpi_file_flushing()) + if (!H5CX_get_mpi_file_flushing()) { + if (MPI_SUCCESS != (mpi_result = MPI_Barrier(aux_ptr->mpi_comm))) HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_result) + } /* Flush data to disk, from rank 0 process */ if (aux_ptr->mpi_rank == 0) { @@ -1904,8 +1908,10 @@ H5AC__rsp__p0_only__flush(H5F_t *f) * enforce POSIX semantics on the server that pretends to be a * file system in our parallel tests. */ - if (aux_ptr->write_done) + if (aux_ptr->write_done) { + (aux_ptr->write_done)(); + } } /* end if */ /* Propagate cleaned entries to other ranks. */ diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index 92f3fb3..7d6ec7b 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -385,6 +385,8 @@ H5_DLL herr_t H5AC_insert_entry(H5F_t *f, const H5AC_class_t *type, haddr_t addr unsigned int flags); H5_DLL herr_t H5AC_pin_protected_entry(void *thing); H5_DLL herr_t H5AC_prep_for_file_close(H5F_t *f); +H5_DLL herr_t H5AC_prep_for_file_flush(H5F_t *f); +H5_DLL herr_t H5AC_secure_from_file_flush(H5F_t *f); H5_DLL herr_t H5AC_create_flush_dependency(void *parent_thing, void *child_thing); H5_DLL void * H5AC_protect(H5F_t *f, const H5AC_class_t *type, haddr_t addr, void *udata, unsigned flags); H5_DLL herr_t H5AC_resize_entry(void *thing, size_t new_size); @@ -216,6 +216,82 @@ H5FL_SEQ_DEFINE_STATIC(H5C_cache_entry_ptr_t); * Programmer: John Mainzer * 6/2/04 * + * Modifications: + * + * JRM -- 7/20/04 + * Updated for the addition of the hash table. + * + * JRM -- 10/5/04 + * Added call to H5C_reset_cache_hit_rate_stats(). Also + * added initialization for cache_is_full flag and for + * resize_ctl. + * + * JRM -- 11/12/04 + * Added initialization for the new size_decreased field. + * + * JRM -- 11/17/04 + * Added/updated initialization for the automatic cache + * size control data structures. + * + * JRM -- 6/24/05 + * Added support for the new write_permitted field of + * the H5C_t structure. + * + * JRM -- 7/5/05 + * Added the new log_flush parameter and supporting code. + * + * JRM -- 9/21/05 + * Added the new aux_ptr parameter and supporting code. + * + * JRM -- 1/20/06 + * Added initialization of the new prefix field in H5C_t. + * + * JRM -- 3/16/06 + * Added initialization for the pinned entry related fields. + * + * JRM -- 5/31/06 + * Added initialization for the trace_file_ptr field. + * + * JRM -- 8/19/06 + * Added initialization for the flush_in_progress field. + * + * JRM -- 8/25/06 + * Added initialization for the slist_len_increase and + * slist_size_increase fields. These fields are used + * for sanity checking in the flush process, and are not + * compiled in unless H5C_DO_SANITY_CHECKS is TRUE. + * + * JRM -- 3/28/07 + * Added initialization for the new is_read_only and + * ro_ref_count fields. + * + * JRM -- 7/27/07 + * Added initialization for the new evictions_enabled + * field of H5C_t. + * + * JRM -- 12/31/07 + * Added initialization for the new flash cache size increase + * related fields of H5C_t. + * + * JRM -- 11/5/08 + * Added initialization for the new clean_index_size and + * dirty_index_size fields of H5C_t. + * + * + * Missing entries? + * + * + * JRM -- 4/20/20 + * Added initialization for the slist_enabled field. Recall + * that the slist is used to flush metadata cache entries + * in (roughly) increasing address order. While this is + * needed at flush and close, it is not used elsewhere. + * The slist_enabled field exists to allow us to construct + * the slist when needed, and leave it empty otherwise -- thus + * avoiding the overhead of maintaining it. + * + * JRM -- 4/29/20 + * *------------------------------------------------------------------------- */ H5C_t * @@ -306,6 +382,8 @@ H5C_create(size_t max_cache_size, size_t min_clean_size, int max_type_id, cache_ptr->ignore_tags = FALSE; cache_ptr->num_objs_corked = 0; + /* slist field initializations */ + cache_ptr->slist_enabled = !H5C__SLIST_OPT_ENABLED; cache_ptr->slist_changed = FALSE; cache_ptr->slist_len = 0; cache_ptr->slist_size = (size_t)0; @@ -750,6 +828,20 @@ done: * Programmer: John Mainzer * 6/2/04 * + * Modifications: + * + * JRM -- 5/15/20 + * + * Updated the function to enable the slist prior to the + * call to H5C__flush_invalidate_cache(). + * + * Arguably, it shouldn't be necessary to re-enable the + * slist after the call to H5C__flush_invalidate_cache(), as + * the metadata cache should be discarded. However, in the + * test code, we make multiple calls to H5C_dest(). Thus + * we re-enable the slist on failure if it and the cache + * still exist. + * *------------------------------------------------------------------------- */ herr_t @@ -770,21 +862,32 @@ H5C_dest(H5F_t *f) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't display cache image stats") #endif /* H5AC_DUMP_IMAGE_STATS_ON_CLOSE */ + /* Enable the slist, as it is needed in the flush */ + if (H5C_set_slist_enabled(f->shared->cache, TRUE, FALSE) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "set slist enabled failed") + /* Flush and invalidate all cache entries */ if (H5C__flush_invalidate_cache(f, H5C__NO_FLAGS_SET) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache") /* Generate & write cache image if requested */ - if (cache_ptr->image_ctl.generate_image) + if (cache_ptr->image_ctl.generate_image) { + if (H5C__generate_cache_image(f, cache_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, FAIL, "Can't generate metadata cache image") + } /* Question: Is it possible for cache_ptr->slist be non-null at this * point? If no, shouldn't this if statement be an assert? */ if (cache_ptr->slist_ptr != NULL) { + + HDassert(cache_ptr->slist_len == 0); + HDassert(cache_ptr->slist_size == 0); + H5SL_close(cache_ptr->slist_ptr); cache_ptr->slist_ptr = NULL; @@ -799,14 +902,19 @@ H5C_dest(H5F_t *f) } /* end if */ - if (cache_ptr->log_info != NULL) + if (cache_ptr->log_info != NULL) { + H5MM_xfree(cache_ptr->log_info); + } #ifndef NDEBUG #if H5C_DO_SANITY_CHECKS - if (cache_ptr->get_entry_ptr_from_addr_counter > 0) - HDfprintf(stdout, "*** %ld calls to H5C_get_entry_ptr_from_add(). ***\n", + + if (cache_ptr->get_entry_ptr_from_addr_counter > 0) { + + HDfprintf(stdout, "*** %" PRId64 " calls to H5C_get_entry_ptr_from_add(). ***\n", cache_ptr->get_entry_ptr_from_addr_counter); + } #endif /* H5C_DO_SANITY_CHECKS */ cache_ptr->magic = 0; @@ -815,6 +923,16 @@ H5C_dest(H5F_t *f) cache_ptr = H5FL_FREE(H5C_t, cache_ptr); done: + + if ((ret_value < 0) && (cache_ptr) && (cache_ptr->slist_ptr)) { + + /* need this for test code -- see change note for details */ + + if (H5C_set_slist_enabled(f->shared->cache, FALSE, FALSE) < 0) + + HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "disable slist on flush dest failure failed") + } + FUNC_LEAVE_NOAPI(ret_value) } /* H5C_dest() */ @@ -829,6 +947,14 @@ done: * Programmer: Vailin Choi * Dec 2013 * + * Modifications: + * + * JRM -- 5/5/20 + * + * Added code to enable the skip list prior to the call + * to H5C__flush_invalidate_cache(), and disable it + * afterwards. + * *------------------------------------------------------------------------- */ herr_t @@ -841,11 +967,21 @@ H5C_evict(H5F_t *f) /* Sanity check */ HDassert(f); + /* Enable the slist, as it is needed in the flush */ + if (H5C_set_slist_enabled(f->shared->cache, TRUE, FALSE) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "set slist enabled failed") + /* Flush and invalidate all cache entries except the pinned entries */ if (H5C__flush_invalidate_cache(f, H5C__EVICT_ALLOW_LAST_PINS_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to evict entries in the cache") + /* Disable the slist */ + if (H5C_set_slist_enabled(f->shared->cache, FALSE, TRUE) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "set slist disabled failed") + done: FUNC_LEAVE_NOAPI(ret_value) } /* H5C_evict() */ @@ -2765,6 +2901,174 @@ done: } /* H5C_set_evictions_enabled() */ /*------------------------------------------------------------------------- + * + * Function: H5C_set_slist_enabled() + * + * Purpose: Enable or disable the slist as directed. + * + * The slist (skip list) is an address ordered list of + * dirty entries in the metadata cache. However, this + * list is only needed during flush and close, where we + * use it to write entries in more or less increasing + * address order. + * + * This function sets up and enables further operations + * on the slist, or disable the slist. This in turn + * allows us to avoid the overhead of maintaining the + * slist when it is not needed. + * + * + * If the slist_enabled parameter is TRUE, the function + * + * 1) Verifies that the slist is empty. + * + * 2) Scans the index list, and inserts all dirty entries + * into the slist. + * + * 3) Sets cache_ptr->slist_enabled = TRUE. + * + * Note that the clear_slist parameter is ignored if + * the slist_enabed parameter is TRUE. + * + * + * If the slist_enabled_parameter is FALSE, the function + * shuts down the slist. + * + * Normally the slist will be empty at this point, however + * that need not be the case if H5C_flush_cache() has been + * called with the H5C__FLUSH_MARKED_ENTRIES_FLAG. + * + * Thus shutdown proceeds as follows: + * + * 1) Test to see if the slist is empty. If it is, proceed + * to step 3. + * + * 2) Test to see if the clear_slist parameter is TRUE. + * + * If it is, remove all entries from the slist. + * + * If it isn't, throw an error. + * + * 3) set cache_ptr->slist_enabled = FALSE. + * + * Return: SUCCEED on success, and FAIL on failure. + * + * Programmer: John Mainzer + * 5/1/20 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_set_slist_enabled(H5C_t *cache_ptr, hbool_t slist_enabled, hbool_t clear_slist) +{ + H5C_cache_entry_t *entry_ptr; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + if ((cache_ptr == NULL) || (cache_ptr->magic != H5C__H5C_T_MAGIC)) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Bad cache_ptr on entry") + +#if H5C__SLIST_OPT_ENABLED + + if (slist_enabled) { + + if (cache_ptr->slist_enabled) { + + HDassert(FALSE); + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "slist already enabled?") + } + + if ((cache_ptr->slist_len != 0) || (cache_ptr->slist_size != 0)) { + + HDassert(FALSE); + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "slist not empty (1)?") + } + + /* set cache_ptr->slist_enabled to TRUE so that the slist + * mainenance macros will be enabled. + */ + cache_ptr->slist_enabled = TRUE; + + /* scan the index list and insert all dirty entries in the slist */ + entry_ptr = cache_ptr->il_head; + + while (entry_ptr != NULL) { + + HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + + if (entry_ptr->is_dirty) { + + H5C__INSERT_ENTRY_IN_SLIST(cache_ptr, entry_ptr, FAIL) + } + + entry_ptr = entry_ptr->il_next; + } + + /* we don't maintain a dirty index len, so we can't do a cross + * check against it. Note that there is no point in cross checking + * against the dirty LRU size, as the dirty LRU may not be maintained, + * and in any case, there is no requirement that all dirty entries + * will reside on the dirty LRU. + */ + HDassert(cache_ptr->dirty_index_size == cache_ptr->slist_size); + } + else { /* take down the skip list */ + + if (!cache_ptr->slist_enabled) { + + HDassert(FALSE); + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "slist already disabled?") + } + + if ((cache_ptr->slist_len != 0) || (cache_ptr->slist_size != 0)) { + + if (clear_slist) { + + H5SL_node_t *node_ptr; + + node_ptr = H5SL_first(cache_ptr->slist_ptr); + + while (node_ptr != NULL) { + + entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); + + H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, FALSE); + + node_ptr = H5SL_first(cache_ptr->slist_ptr); + } + } + else { + + HDassert(FALSE); + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "slist not empty (2)?") + } + } + + cache_ptr->slist_enabled = FALSE; + + HDassert(0 == cache_ptr->slist_len); + HDassert(0 == cache_ptr->slist_size); + } + +#else /* H5C__SLIST_OPT_ENABLED is FALSE */ + + HDassert(cache_ptr->slist_enabled); + +#endif /* H5C__SLIST_OPT_ENABLED is FALSE */ + +done: + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5C_set_slist_enabled() */ + +/*------------------------------------------------------------------------- * Function: H5C_unpin_entry() * * Purpose: Unpin a cache entry. The entry can be either protected or @@ -2838,6 +3142,81 @@ done: * Programmer: John Mainzer * 6/2/04 * + * Modifications: + * + * JRM -- 7/21/04 + * Updated for the addition of the hash table. + * + * JRM -- 10/28/04 + * Added code to set cache_full to TRUE whenever we try to + * make space in the cache. + * + * JRM -- 11/12/04 + * Added code to call to H5C_make_space_in_cache() after the + * call to H5C__auto_adjust_cache_size() if that function + * sets the size_decreased flag is TRUE. + * + * JRM -- 4/25/05 + * The size_decreased flag can also be set to TRUE in + * H5C_set_cache_auto_resize_config() if a new configuration + * forces an immediate reduction in cache size. Modified + * the code to deal with this eventuallity. + * + * JRM -- 6/24/05 + * Added support for the new write_permitted field of H5C_t. + * + * JRM -- 10/22/05 + * Hand optimizations. + * + * JRM -- 5/3/06 + * Added code to set the new dirtied field in + * H5C_cache_entry_t to FALSE prior to return. + * + * JRM -- 6/23/06 + * Modified code to allow dirty entries to be loaded from + * disk. This is necessary as a bug fix in the object + * header code requires us to modify a header as it is read. + * + * JRM -- 3/28/07 + * Added the flags parameter and supporting code. At least + * for now, this parameter is used to allow the entry to + * be protected read only, thus allowing multiple protects. + * + * Also added code to allow multiple read only protects + * of cache entries. + * + * JRM -- 7/27/07 + * Added code supporting the new evictions_enabled field + * in H5C_t. + * + * JRM -- 1/3/08 + * Added to do a flash cache size increase if appropriate + * when a large entry is loaded. + * + * JRM -- 11/13/08 + * Modified function to call H5C_make_space_in_cache() when + * the min_clean_size is violated, not just when there isn't + * enough space for and entry that has just been loaded. + * + * The purpose of this modification is to avoid "metadata + * blizzards" in the write only case. In such instances, + * the cache was allowed to fill with dirty metadata. When + * we finally needed to evict an entry to make space, we had + * to flush out a whole cache full of metadata -- which has + * interesting performance effects. We hope to avoid (or + * perhaps more accurately hide) this effect by maintaining + * the min_clean_size, which should force us to start flushing + * entries long before we actually have to evict something + * to make space. + * + * + * Missing entries? + * + * + * JRM -- 5/8/20 + * Updated for the possibility that the slist will be + * disabled. + * *------------------------------------------------------------------------- */ herr_t @@ -2879,9 +3258,15 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags) HDassert(H5F_addr_defined(addr)); HDassert(thing); HDassert(!(pin_entry && unpin_entry)); - HDassert((!free_file_space) || (deleted)); /* deleted flag must accompany free_file_space */ - HDassert((!take_ownership) || (deleted)); /* deleted flag must accompany take_ownership */ - HDassert(!(free_file_space && take_ownership)); /* can't have both free_file_space & take_ownership */ + + /* deleted flag must accompany free_file_space */ + HDassert((!free_file_space) || (deleted)); + + /* deleted flag must accompany take_ownership */ + HDassert((!take_ownership) || (deleted)); + + /* can't have both free_file_space & take_ownership */ + HDassert(!(free_file_space && take_ownership)); entry_ptr = (H5C_cache_entry_t *)thing; @@ -2983,16 +3368,22 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags) /* Mark the entry as dirty if appropriate */ entry_ptr->is_dirty = (entry_ptr->is_dirty || dirtied); - if (dirtied) + if (dirtied) { + if (entry_ptr->image_up_to_date) { entry_ptr->image_up_to_date = FALSE; - if (entry_ptr->flush_dep_nparents > 0) + + if (entry_ptr->flush_dep_nparents > 0) { + if (H5C__mark_flush_dep_unserialized(entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "Can't propagate serialization status to fd parents") - } /* end if */ + + } /* end if */ + } /* end if */ + } /* end if */ /* Check for newly dirtied entry */ if (was_clean && entry_ptr->is_dirty) { @@ -3004,16 +3395,20 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags) * 'entry dirtied' notice now that the entry is fully * integrated into the cache. */ - if (entry_ptr->type->notify && - (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_DIRTIED, entry_ptr) < 0) + if ((entry_ptr->type->notify) && + ((entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_DIRTIED, entry_ptr) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag set") /* Propagate the flush dep dirty flag up the flush dependency chain - * if appropriate */ - if (entry_ptr->flush_dep_nparents > 0) + * if appropriate + */ + if (entry_ptr->flush_dep_nparents > 0) { + if (H5C__mark_flush_dep_dirty(entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep dirty flag") + } } /* end if */ /* Check for newly clean entry */ else if (!was_clean && !entry_ptr->is_dirty) { @@ -3022,17 +3417,21 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags) * 'entry cleaned' notice now that the entry is fully * integrated into the cache. */ - if (entry_ptr->type->notify && - (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_CLEANED, entry_ptr) < 0) + if ((entry_ptr->type->notify) && + ((entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_CLEANED, entry_ptr) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag cleared") /* Propagate the flush dep clean flag up the flush dependency chain - * if appropriate */ - if (entry_ptr->flush_dep_nparents > 0) + * if appropriate + */ + if (entry_ptr->flush_dep_nparents > 0) { + if (H5C__mark_flush_dep_clean(entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep dirty flag") + } } /* end else-if */ /* Pin or unpin the entry as requested. */ @@ -3064,8 +3463,12 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags) if (entry_ptr->is_dirty) { entry_ptr->flush_marker |= set_flush_marker; - if (!entry_ptr->in_slist) + + if (!entry_ptr->in_slist) { + + /* this is a no-op if cache_ptr->slist_enabled is FALSE */ H5C__INSERT_ENTRY_IN_SLIST(cache_ptr, entry_ptr, FAIL) + } } /* end if */ /* this implementation of the "deleted" option is a bit inefficient, as @@ -3094,17 +3497,22 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags) "hash table contains multiple entries for addr?!?") /* Set the 'free file space' flag for the flush, if needed */ - if (free_file_space) + if (free_file_space) { + flush_flags |= H5C__FREE_FILE_SPACE_FLAG; + } /* Set the "take ownership" flag for the flush, if needed */ - if (take_ownership) + if (take_ownership) { + flush_flags |= H5C__TAKE_OWNERSHIP_FLAG; + } /* Delete the entry from the skip list on destroy */ flush_flags |= H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG; - HDassert(((!was_clean) || dirtied) == entry_ptr->in_slist); + HDassert((!cache_ptr->slist_enabled) || (((!was_clean) || dirtied) == (entry_ptr->in_slist))); + if (H5C__flush_single_entry(f, entry_ptr, flush_flags) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "Can't flush entry") @@ -4868,6 +5276,7 @@ done: } /* H5C__flash_increase_cache_size() */ /*------------------------------------------------------------------------- + * * Function: H5C__flush_invalidate_cache * * Purpose: Flush and destroy the entries contained in the target @@ -4897,6 +5306,51 @@ done: * Programmer: John Mainzer * 3/24/065 * + * Modifications: + * + * To support the fractal heap, the cache must now deal with + * entries being dirtied, resized, and/or renamed inside + * flush callbacks. Updated function to support this. + * + * -- JRM 8/27/06 + * + * Added code to detect and manage the case in which a + * flush callback changes the s-list out from under + * the function. The only way I can think of in which this + * can happen is if a flush function loads an entry + * into the cache that isn't there already. Quincey tells + * me that this will never happen, but I'm not sure I + * believe him. + * + * Note that this is a pretty bad scenario if it ever + * happens. The code I have added should allow us to + * handle the situation under all but the worst conditions, + * but one can argue that we should just scream and die if + * we ever detect the condition. + * + * -- JRM 10/13/07 + * + * Missing entries? + * + * + * Added support for the H5C__EVICT_ALLOW_LAST_PINS_FLAG. + * This flag is used to flush and evict all entries in + * the metadata cache that are not pinned -- typically, + * everything other than the superblock. + * + * ??? -- ??/??/?? + * + * Added sanity checks to verify that the skip list is + * enabled on entry. On the face of it, it would make + * sense to enable the slist on entry, and disable it + * on exit, as this function is not called repeatedly. + * However, since this function can be called from + * H5C_flush_cache(), this would create cases in the test + * code where we would have to check the flags to determine + * whether we must setup and take down the slist. + * + * JRM -- 5/5/20 + * *------------------------------------------------------------------------- */ static herr_t @@ -4914,6 +5368,7 @@ H5C__flush_invalidate_cache(H5F_t *f, unsigned flags) HDassert(cache_ptr); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); HDassert(cache_ptr->slist_ptr); + HDassert(cache_ptr->slist_enabled); #if H5C_DO_SANITY_CHECKS { @@ -4933,6 +5388,7 @@ H5C__flush_invalidate_cache(H5F_t *f, unsigned flags) HDassert(cache_ptr->slist_ring_size[H5C_RING_UNDEFINED] == (size_t)0); for (i = H5C_RING_USER; i < H5C_RING_NTYPES; i++) { + index_len += cache_ptr->index_ring_len[i]; index_size += cache_ptr->index_ring_size[i]; clean_index_size += cache_ptr->clean_index_ring_size[i]; @@ -4940,6 +5396,7 @@ H5C__flush_invalidate_cache(H5F_t *f, unsigned flags) slist_len += cache_ptr->slist_ring_len[i]; slist_size += cache_ptr->slist_ring_size[i]; + } /* end for */ HDassert(cache_ptr->index_len == index_len); @@ -4952,49 +5409,66 @@ H5C__flush_invalidate_cache(H5F_t *f, unsigned flags) #endif /* H5C_DO_SANITY_CHECKS */ /* remove ageout markers if present */ - if (cache_ptr->epoch_markers_active > 0) + if (cache_ptr->epoch_markers_active > 0) { + if (H5C__autoadjust__ageout__remove_all_markers(cache_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "error removing all epoch markers") + } /* flush invalidate each ring, starting from the outermost ring and * working inward. */ ring = H5C_RING_USER; + while (ring < H5C_RING_NTYPES) { + if (H5C__flush_invalidate_ring(f, ring, flags) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "flush invalidate ring failed") ring++; + } /* end while */ /* Invariants, after destroying all entries in the hash table */ if (!(flags & H5C__EVICT_ALLOW_LAST_PINS_FLAG)) { + HDassert(cache_ptr->index_size == 0); HDassert(cache_ptr->clean_index_size == 0); HDassert(cache_ptr->pel_len == 0); HDassert(cache_ptr->pel_size == 0); + } /* end if */ else { + H5C_cache_entry_t *entry_ptr; /* Cache entry */ unsigned u; /* Local index variable */ /* All rings except ring 4 should be empty now */ /* (Ring 4 has the superblock) */ for (u = H5C_RING_USER; u < H5C_RING_SB; u++) { + HDassert(cache_ptr->index_ring_len[u] == 0); HDassert(cache_ptr->index_ring_size[u] == 0); HDassert(cache_ptr->clean_index_ring_size[u] == 0); + } /* end for */ /* Check that any remaining pinned entries are in the superblock ring */ + entry_ptr = cache_ptr->pel_head_ptr; + while (entry_ptr) { + /* Check ring */ HDassert(entry_ptr->ring == H5C_RING_SB); /* Advance to next entry in pinned entry list */ entry_ptr = entry_ptr->next; + } /* end while */ } /* end else */ + HDassert(cache_ptr->dirty_index_size == 0); HDassert(cache_ptr->slist_len == 0); HDassert(cache_ptr->slist_size == 0); @@ -5004,42 +5478,58 @@ H5C__flush_invalidate_cache(H5F_t *f, unsigned flags) HDassert(cache_ptr->LRU_list_size == 0); done: + FUNC_LEAVE_NOAPI(ret_value) + } /* H5C__flush_invalidate_cache() */ /*------------------------------------------------------------------------- * Function: H5C__flush_invalidate_ring * - * Purpose: Flush and destroy the entries contained in the target - * cache and ring. + * Purpose: Flush and destroy the entries contained in the target + * cache and ring. * - * If the ring contains protected entries, the function will - * fail, as protected entries cannot be either flushed or - * destroyed. However all unprotected entries should be - * flushed and destroyed before the function returns failure. + * If the ring contains protected entries, the function will + * fail, as protected entries cannot be either flushed or + * destroyed. However all unprotected entries should be + * flushed and destroyed before the function returns failure. * - * While pinned entries can usually be flushed, they cannot - * be destroyed. However, they should be unpinned when all - * the entries that reference them have been destroyed (thus - * reduding the pinned entry's reference count to 0, allowing - * it to be unpinned). + * While pinned entries can usually be flushed, they cannot + * be destroyed. However, they should be unpinned when all + * the entries that reference them have been destroyed (thus + * reduding the pinned entry's reference count to 0, allowing + * it to be unpinned). * - * If pinned entries are present, the function makes repeated - * passes through the cache, flushing all dirty entries - * (including the pinned dirty entries where permitted) and - * destroying all unpinned entries. This process is repeated - * until either the cache is empty, or the number of pinned - * entries stops decreasing on each pass. + * If pinned entries are present, the function makes repeated + * passes through the cache, flushing all dirty entries + * (including the pinned dirty entries where permitted) and + * destroying all unpinned entries. This process is repeated + * until either the cache is empty, or the number of pinned + * entries stops decreasing on each pass. * - * If flush dependencies appear in the target ring, the - * function makes repeated passes through the cache flushing - * entries in flush dependency order. + * If flush dependencies appear in the target ring, the + * function makes repeated passes through the cache flushing + * entries in flush dependency order. * * Return: Non-negative on success/Negative on failure or if there was - * a request to flush all items and something was protected. + * a request to flush all items and something was protected. * * Programmer: John Mainzer - * 9/1/15 + * 9/1/15 + * + * Changes: Added support for the H5C__EVICT_ALLOW_LAST_PINS_FLAG. + * This flag is used to flush and evict all entries in + * the metadata cache that are not pinned -- typically, + * everything other than the superblock. + * + * ??? -- ??/??/?? + * + * A recent optimization turns off the slist unless a flush + * is in progress. This should not effect this function, as + * it is only called during a flush. Added an assertion to + * verify this. + * + * JRM -- 5/6/20 * *------------------------------------------------------------------------- */ @@ -5067,9 +5557,12 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) HDassert(f); HDassert(f->shared); + cache_ptr = f->shared->cache; + HDassert(cache_ptr); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(cache_ptr->slist_enabled); HDassert(cache_ptr->slist_ptr); HDassert(ring > H5C_RING_UNDEFINED); HDassert(ring < H5C_RING_NTYPES); @@ -5111,19 +5604,25 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) */ /* compute the number of pinned entries in this ring */ + entry_ptr = cache_ptr->pel_head_ptr; cur_ring_pel_len = 0; + while (entry_ptr != NULL) { + HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(entry_ptr->ring >= ring); if (entry_ptr->ring == ring) cur_ring_pel_len++; entry_ptr = entry_ptr->next; + } /* end while */ old_ring_pel_len = cur_ring_pel_len; + while (cache_ptr->index_ring_len[ring] > 0) { + /* first, try to flush-destroy any dirty entries. Do this by * making a scan through the slist. Note that new dirty entries * may be created by the flush call backs. Thus it is possible @@ -5166,25 +5665,32 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) /* this done, start the scan of the slist */ restart_slist_scan = TRUE; + while (restart_slist_scan || (node_ptr != NULL)) { + if (restart_slist_scan) { + restart_slist_scan = FALSE; /* Start at beginning of skip list */ node_ptr = H5SL_first(cache_ptr->slist_ptr); + if (node_ptr == NULL) /* the slist is empty -- break out of inner loop */ break; /* Get cache entry for this node */ next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); + if (NULL == next_entry_ptr) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "next_entry_ptr == NULL ?!?!") HDassert(next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(next_entry_ptr->is_dirty); HDassert(next_entry_ptr->in_slist); HDassert(next_entry_ptr->ring >= ring); + } /* end if */ entry_ptr = next_entry_ptr; @@ -5210,18 +5716,25 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) * from the slist. */ node_ptr = H5SL_next(node_ptr); + if (node_ptr != NULL) { + next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); + if (NULL == next_entry_ptr) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "next_entry_ptr == NULL ?!?!") + HDassert(next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(next_entry_ptr->is_dirty); HDassert(next_entry_ptr->in_slist); HDassert(next_entry_ptr->ring >= ring); HDassert(entry_ptr != next_entry_ptr); } /* end if */ - else + else { + next_entry_ptr = NULL; + } /* Note that we now remove nodes from the slist as we flush * the associated entries, instead of leaving them there @@ -5236,17 +5749,23 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) if (((!entry_ptr->flush_me_last) || ((entry_ptr->flush_me_last) && (cache_ptr->num_last_entries >= cache_ptr->slist_len))) && (entry_ptr->flush_dep_nchildren == 0) && (entry_ptr->ring == ring)) { + if (entry_ptr->is_protected) { + /* we have major problems -- but lets flush * everything we can before we flag an error. */ protected_entries++; + } /* end if */ else if (entry_ptr->is_pinned) { + if (H5C__flush_single_entry(f, entry_ptr, H5C__DURING_FLUSH_FLAG) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "dirty pinned entry flush failed") if (cache_ptr->slist_changed) { + /* The slist has been modified by something * other than the simple removal of the * of the flushed entry after the flush. @@ -5257,16 +5776,20 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) restart_slist_scan = TRUE; cache_ptr->slist_changed = FALSE; H5C__UPDATE_STATS_FOR_SLIST_SCAN_RESTART(cache_ptr); + } /* end if */ } /* end else-if */ else { + if (H5C__flush_single_entry(f, entry_ptr, (cooked_flags | H5C__DURING_FLUSH_FLAG | H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "dirty entry flush destroy failed") if (cache_ptr->slist_changed) { + /* The slist has been modified by something * other than the simple removal of the * of the flushed entry after the flush. @@ -5293,8 +5816,10 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) */ if (node_ptr == NULL) { + HDassert(cache_ptr->slist_len == (uint32_t)((int32_t)initial_slist_len + cache_ptr->slist_len_increase)); + HDassert(cache_ptr->slist_size == (size_t)((ssize_t)initial_slist_size + cache_ptr->slist_size_increase)); } /* end if */ @@ -5320,7 +5845,9 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) cache_ptr->entries_relocated_counter = 0; next_entry_ptr = cache_ptr->il_head; + while (next_entry_ptr != NULL) { + entry_ptr = next_entry_ptr; HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(entry_ptr->ring >= ring); @@ -5328,19 +5855,25 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) next_entry_ptr = entry_ptr->il_next; HDassert((next_entry_ptr == NULL) || (next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC)); - if ((!entry_ptr->flush_me_last || - (entry_ptr->flush_me_last && cache_ptr->num_last_entries >= cache_ptr->slist_len)) && - entry_ptr->flush_dep_nchildren == 0 && entry_ptr->ring == ring) { + if (((!entry_ptr->flush_me_last) || + (entry_ptr->flush_me_last && (cache_ptr->num_last_entries >= cache_ptr->slist_len))) && + (entry_ptr->flush_dep_nchildren == 0) && (entry_ptr->ring == ring)) { + if (entry_ptr->is_protected) { + /* we have major problems -- but lets flush and * destroy everything we can before we flag an * error. */ protected_entries++; - if (!entry_ptr->in_slist) + + if (!entry_ptr->in_slist) { + HDassert(!(entry_ptr->is_dirty)); + } } /* end if */ else if (!(entry_ptr->is_pinned)) { + /* if *entry_ptr is dirty, it is possible * that one or more other entries may be * either removed from the cache, loaded @@ -5373,6 +5906,7 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) (cooked_flags | H5C__DURING_FLUSH_FLAG | H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Entry flush destroy failed") /* Restart the index list scan if necessary. Must @@ -5383,7 +5917,7 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) * if this results in the size of the pinned entry * failing to decline during the pass. */ - if ((NULL != next_entry_ptr && NULL == cache_ptr->entry_watched_for_removal) || + if (((NULL != next_entry_ptr) && (NULL == cache_ptr->entry_watched_for_removal)) || (cache_ptr->entries_loaded_counter > 0) || (cache_ptr->entries_inserted_counter > 0) || (cache_ptr->entries_relocated_counter > 0)) { @@ -5395,9 +5929,12 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) cache_ptr->entries_relocated_counter = 0; H5C__UPDATE_STATS_FOR_INDEX_SCAN_RESTART(cache_ptr) + } /* end if */ - else + else { + cache_ptr->entry_watched_for_removal = NULL; + } } /* end if */ } /* end if */ } /* end for loop scanning hash table */ @@ -5413,23 +5950,31 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) old_ring_pel_len = cur_ring_pel_len; entry_ptr = cache_ptr->pel_head_ptr; cur_ring_pel_len = 0; + while (entry_ptr != NULL) { + HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(entry_ptr->ring >= ring); - if (entry_ptr->ring == ring) + if (entry_ptr->ring == ring) { + cur_ring_pel_len++; + } entry_ptr = entry_ptr->next; + } /* end while */ /* Check if the number of pinned entries in the ring is positive, and * it is not declining. Scream and die if so. */ - if (cur_ring_pel_len > 0 && cur_ring_pel_len >= old_ring_pel_len) { + if ((cur_ring_pel_len > 0) && (cur_ring_pel_len >= old_ring_pel_len)) { + /* Don't error if allowed to have pinned entries remaining */ - if (evict_flags) + if (evict_flags) { + HGOTO_DONE(TRUE) + } HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Pinned entry count not decreasing, cur_ring_pel_len = %d, old_ring_pel_len = " @@ -5439,14 +5984,17 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) HDassert(protected_entries == cache_ptr->pl_len); - if (protected_entries > 0 && protected_entries == cache_ptr->index_len) + if ((protected_entries > 0) && (protected_entries == cache_ptr->index_len)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Only protected entries left in cache, protected_entries = %d", (int)protected_entries) + } /* main while loop */ /* Invariants, after destroying all entries in the ring */ for (i = (int)H5C_RING_UNDEFINED; i <= (int)ring; i++) { + HDassert(cache_ptr->index_ring_len[i] == 0); HDassert(cache_ptr->index_ring_size[i] == (size_t)0); HDassert(cache_ptr->clean_index_ring_size[i] == (size_t)0); @@ -5454,40 +6002,56 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) HDassert(cache_ptr->slist_ring_len[i] == 0); HDassert(cache_ptr->slist_ring_size[i] == (size_t)0); + } /* end for */ HDassert(protected_entries <= cache_ptr->pl_len); - if (protected_entries > 0) + if (protected_entries > 0) { + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Cache has protected entries") - else if (cur_ring_pel_len > 0) + } + else if (cur_ring_pel_len > 0) { + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't unpin all pinned entries in ring") + } done: + FUNC_LEAVE_NOAPI(ret_value) + } /* H5C__flush_invalidate_ring() */ /*------------------------------------------------------------------------- + * * Function: H5C__flush_ring * - * Purpose: Flush the entries contained in the specified cache and - * ring. All entries in rings outside the specified ring - * must have been flushed on entry. + * Purpose: Flush the entries contained in the specified cache and + * ring. All entries in rings outside the specified ring + * must have been flushed on entry. * - * If the cache contains protected entries in the specified - * ring, the function will fail, as protected entries cannot - * be flushed. However all unprotected entries in the target - * ring should be flushed before the function returns failure. + * If the cache contains protected entries in the specified + * ring, the function will fail, as protected entries cannot + * be flushed. However all unprotected entries in the target + * ring should be flushed before the function returns failure. * - * If flush dependencies appear in the target ring, the - * function makes repeated passes through the slist flushing - * entries in flush dependency order. + * If flush dependencies appear in the target ring, the + * function makes repeated passes through the slist flushing + * entries in flush dependency order. * * Return: Non-negative on success/Negative on failure or if there was - * a request to flush all items and something was protected. + * a request to flush all items and something was protected. * * Programmer: John Mainzer - * 9/1/15 + * 9/1/15 + * + * Changes: A recent optimization turns off the slist unless a flush + * is in progress. This should not effect this function, as + * it is only called during a flush. Added an assertion to + * verify this. + * + * JRM -- 5/6/20 + * * *------------------------------------------------------------------------- */ @@ -5515,6 +6079,7 @@ H5C__flush_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) HDassert(cache_ptr); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(cache_ptr->slist_enabled); HDassert(cache_ptr->slist_ptr); HDassert((flags & H5C__FLUSH_INVALIDATE_FLAG) == 0); HDassert(ring > H5C_RING_UNDEFINED); @@ -5523,15 +6088,20 @@ H5C__flush_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) #if H5C_DO_EXTREME_SANITY_CHECKS if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ ignore_protected = ((flags & H5C__FLUSH_IGNORE_PROTECTED_FLAG) != 0); flush_marked_entries = ((flags & H5C__FLUSH_MARKED_ENTRIES_FLAG) != 0); - if (!flush_marked_entries) - for (i = (int)H5C_RING_UNDEFINED; i < (int)ring; i++) + if (!flush_marked_entries) { + + for (i = (int)H5C_RING_UNDEFINED; i < (int)ring; i++) { + HDassert(cache_ptr->slist_ring_len[i] == 0); + } + } HDassert(cache_ptr->flush_in_progress); @@ -5553,6 +6123,7 @@ H5C__flush_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) cache_ptr->slist_changed = FALSE; while ((cache_ptr->slist_ring_len[ring] > 0) && (protected_entries == 0) && (flushed_entries_last_pass)) { + flushed_entries_last_pass = FALSE; #if H5C_DO_SANITY_CHECKS @@ -5597,25 +6168,31 @@ H5C__flush_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) restart_slist_scan = TRUE; while ((restart_slist_scan) || (node_ptr != NULL)) { + if (restart_slist_scan) { + restart_slist_scan = FALSE; /* Start at beginning of skip list */ node_ptr = H5SL_first(cache_ptr->slist_ptr); - if (node_ptr == NULL) + if (node_ptr == NULL) { + /* the slist is empty -- break out of inner loop */ break; + } /* Get cache entry for this node */ next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); if (NULL == next_entry_ptr) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "next_entry_ptr == NULL ?!?!") HDassert(next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(next_entry_ptr->is_dirty); HDassert(next_entry_ptr->in_slist); + } /* end if */ entry_ptr = next_entry_ptr; @@ -5640,52 +6217,69 @@ H5C__flush_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(entry_ptr->in_slist); HDassert(entry_ptr->is_dirty); - if (!flush_marked_entries || entry_ptr->flush_marker) + + if ((!flush_marked_entries) || (entry_ptr->flush_marker)) { + HDassert(entry_ptr->ring >= ring); + } /* Advance node pointer now, before we delete its target * from the slist. */ node_ptr = H5SL_next(node_ptr); + if (node_ptr != NULL) { + next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); + if (NULL == next_entry_ptr) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "next_entry_ptr == NULL ?!?!") HDassert(next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(next_entry_ptr->is_dirty); HDassert(next_entry_ptr->in_slist); - if (!flush_marked_entries || next_entry_ptr->flush_marker) + if (!flush_marked_entries || next_entry_ptr->flush_marker) { + HDassert(next_entry_ptr->ring >= ring); + } HDassert(entry_ptr != next_entry_ptr); + } /* end if */ - else + else { + next_entry_ptr = NULL; + } if ((!flush_marked_entries || entry_ptr->flush_marker) && - (!entry_ptr->flush_me_last || - (entry_ptr->flush_me_last && (cache_ptr->num_last_entries >= cache_ptr->slist_len || - (flush_marked_entries && entry_ptr->flush_marker)))) && - (entry_ptr->flush_dep_nchildren == 0 || entry_ptr->flush_dep_ndirty_children == 0) && - entry_ptr->ring == ring) { + ((!entry_ptr->flush_me_last) || + ((entry_ptr->flush_me_last) && ((cache_ptr->num_last_entries >= cache_ptr->slist_len) || + (flush_marked_entries && entry_ptr->flush_marker)))) && + ((entry_ptr->flush_dep_nchildren == 0) || (entry_ptr->flush_dep_ndirty_children == 0)) && + (entry_ptr->ring == ring)) { HDassert(entry_ptr->flush_dep_nunser_children == 0); if (entry_ptr->is_protected) { + /* we probably have major problems -- but lets * flush everything we can before we decide * whether to flag an error. */ tried_to_flush_protected_entry = TRUE; protected_entries++; + } /* end if */ else { + if (H5C__flush_single_entry(f, entry_ptr, (flags | H5C__DURING_FLUSH_FLAG)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush entry") if (cache_ptr->slist_changed) { + /* The slist has been modified by something * other than the simple removal of the * of the flushed entry after the flush. @@ -5696,9 +6290,11 @@ H5C__flush_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) restart_slist_scan = TRUE; cache_ptr->slist_changed = FALSE; H5C__UPDATE_STATS_FOR_SLIST_SCAN_RESTART(cache_ptr) + } /* end if */ flushed_entries_last_pass = TRUE; + } /* end else */ } /* end if */ } /* while ( ( restart_slist_scan ) || ( node_ptr != NULL ) ) */ @@ -5710,22 +6306,28 @@ H5C__flush_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) HDassert((size_t)((ssize_t)initial_slist_size + cache_ptr->slist_size_increase) == cache_ptr->slist_size); #endif /* H5C_DO_SANITY_CHECKS */ - } /* while */ + + } /* while */ HDassert(protected_entries <= cache_ptr->pl_len); if (((cache_ptr->pl_len > 0) && (!ignore_protected)) || (tried_to_flush_protected_entry)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "cache has protected items") #if H5C_DO_SANITY_CHECKS if (!flush_marked_entries) { + HDassert(cache_ptr->slist_ring_len[ring] == 0); HDassert(cache_ptr->slist_ring_size[ring] == 0); + } /* end if */ #endif /* H5C_DO_SANITY_CHECKS */ done: + FUNC_LEAVE_NOAPI(ret_value) + } /* H5C__flush_ring() */ /*------------------------------------------------------------------------- @@ -5733,32 +6335,89 @@ done: * Function: H5C__flush_single_entry * * Purpose: Flush or clear (and evict if requested) the cache entry - * with the specified address and type. If the type is NULL, - * any unprotected entry at the specified address will be - * flushed (and possibly evicted). + * with the specified address and type. If the type is NULL, + * any unprotected entry at the specified address will be + * flushed (and possibly evicted). * - * Attempts to flush a protected entry will result in an - * error. + * Attempts to flush a protected entry will result in an + * error. * - * If the H5C__FLUSH_INVALIDATE_FLAG flag is set, the entry will - * be cleared and not flushed, and the call can't be part of a + * If the H5C__FLUSH_INVALIDATE_FLAG flag is set, the entry will + * be cleared and not flushed, and the call can't be part of a * sequence of flushes. * - * If the caller knows the address of the skip list node at - * which the target entry resides, it can avoid a lookup - * by supplying that address in the tgt_node_ptr parameter. - * If this parameter is NULL, the function will do a skip list - * search for the entry instead. - * - * The function does nothing silently if there is no entry - * at the supplied address, or if the entry found has the - * wrong type. + * The function does nothing silently if there is no entry + * at the supplied address, or if the entry found has the + * wrong type. * * Return: Non-negative on success/Negative on failure or if there was - * an attempt to flush a protected item. + * an attempt to flush a protected item. * * Programmer: John Mainzer, 5/5/04 * + * Modifications: + * + * JRM -- 7/21/04 + * Updated function for the addition of the hash table. + * + * QAK -- 11/26/04 + * Updated function for the switch from TBBTs to skip lists. + * + * JRM -- 1/6/05 + * Updated function to reset the flush_marker field. + * Also replace references to H5F_FLUSH_INVALIDATE and + * H5F_FLUSH_CLEAR_ONLY with references to + * H5C__FLUSH_INVALIDATE_FLAG and H5C__FLUSH_CLEAR_ONLY_FLAG + * respectively. + * + * JRM -- 6/24/05 + * Added code to remove dirty entries from the slist after + * they have been flushed. Also added a sanity check that + * will scream if we attempt a write when writes are + * completely disabled. + * + * JRM -- 7/5/05 + * Added code to call the new log_flush callback whenever + * a dirty entry is written to disk. Note that the callback + * is not called if the H5C__FLUSH_CLEAR_ONLY_FLAG is set, + * as there is no write to file in this case. + * + * JRM -- 8/21/06 + * Added code maintaining the flush_in_progress and + * destroy_in_progress fields in H5C_cache_entry_t. + * + * Also added flush_flags parameter to the call to + * type_ptr->flush() so that the flush routine can report + * whether the entry has been resized or renamed. Added + * code using the flush_flags variable to detect the case + * in which the target entry is resized during flush, and + * update the caches data structures accordingly. + * + * JRM -- 3/29/07 + * Added sanity checks on the new is_read_only and + * ro_ref_count fields. + * + * QAK -- 2/07/08 + * Separated "destroy entry" concept from "remove entry from + * cache" concept, by adding the 'take_ownership' flag and + * the "destroy_entry" variable. + * + * JRM -- 11/5/08 + * Added call to H5C__UPDATE_INDEX_FOR_ENTRY_CLEAN() to + * maintain the new clean_index_size and clean_index_size + * fields of H5C_t. + * + * + * Missing entries?? + * + * + * JRM -- 5/8/20 + * Updated sanity checks for the possibility that the slist + * is disabled. + * + * Also updated main comment to conform more closely with + * the current state of the code. + * *------------------------------------------------------------------------- */ herr_t @@ -5805,18 +6464,26 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) /* Set the flag for destroying the entry, based on the 'take ownership' * and 'destroy' flags */ - if (take_ownership) + if (take_ownership) { + destroy_entry = FALSE; - else + } + else { + destroy_entry = destroy; + } /* we will write the entry to disk if it exists, is dirty, and if the * clear only flag is not set. */ - if (entry_ptr->is_dirty && !clear_only) + if (entry_ptr->is_dirty && !clear_only) { + write_entry = TRUE; - else + } + else { + write_entry = FALSE; + } /* if we have received close warning, and we have been instructed to * generate a metadata cache image, and we have actually constructed @@ -5825,8 +6492,9 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * Set suppress_image_entry_writes to TRUE if indicated by the * image_ctl flags. */ - if (cache_ptr->close_warning_received && cache_ptr->image_ctl.generate_image && - cache_ptr->num_entries_in_image > 0 && cache_ptr->image_entries) { + if ((cache_ptr->close_warning_received) && (cache_ptr->image_ctl.generate_image) && + (cache_ptr->num_entries_in_image > 0) && (cache_ptr->image_entries != NULL)) { + /* Sanity checks */ HDassert(entry_ptr->image_up_to_date || !(entry_ptr->include_in_image)); HDassert(entry_ptr->image_ptr || !(entry_ptr->include_in_image)); @@ -5836,32 +6504,56 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) suppress_image_entry_frees = TRUE; - if (cache_ptr->image_ctl.flags & H5C_CI__SUPRESS_ENTRY_WRITES) + if (cache_ptr->image_ctl.flags & H5C_CI__SUPRESS_ENTRY_WRITES) { + suppress_image_entry_writes = TRUE; - } /* end if */ + + } /* end if */ + } /* end if */ /* run initial sanity checks */ #if H5C_DO_SANITY_CHECKS - if (entry_ptr->in_slist) { - HDassert(entry_ptr->is_dirty); + if (cache_ptr->slist_enabled) { - if ((entry_ptr->flush_marker) && (!entry_ptr->is_dirty)) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry in slist failed sanity checks") - } /* end if */ - else { - HDassert(!entry_ptr->is_dirty); - HDassert(!entry_ptr->flush_marker); + if (entry_ptr->in_slist) { + + HDassert(entry_ptr->is_dirty); + + if ((entry_ptr->flush_marker) && (!entry_ptr->is_dirty)) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry in slist failed sanity checks") + } /* end if */ + else { + + HDassert(!entry_ptr->is_dirty); + HDassert(!entry_ptr->flush_marker); + + if ((entry_ptr->is_dirty) || (entry_ptr->flush_marker)) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry failed sanity checks") + + } /* end else */ + } + else { /* slist is disabled */ + + HDassert(!entry_ptr->in_slist); + + if (!entry_ptr->is_dirty) { + + if (entry_ptr->flush_marker) - if ((entry_ptr->is_dirty) || (entry_ptr->flush_marker)) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry failed sanity checks") - } /* end else */ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "flush marked clean entry?") + } + } #endif /* H5C_DO_SANITY_CHECKS */ if (entry_ptr->is_protected) { + HDassert(!entry_ptr->is_protected); /* Attempt to flush a protected entry -- scream and die. */ HGOTO_ERROR(H5E_CACHE, H5E_PROTECT, FAIL, "Attempt to flush a protected entry") + } /* end if */ /* Set entry_ptr->flush_in_progress = TRUE and set @@ -5881,25 +6573,33 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * entry. */ if (write_entry || generate_image) { + HDassert(entry_ptr->is_dirty); if (NULL == entry_ptr->image_ptr) { + if (NULL == (entry_ptr->image_ptr = H5MM_malloc(entry_ptr->size + H5C_IMAGE_EXTRA_SPACE))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for on disk image buffer") + #if H5C_DO_MEMORY_SANITY_CHECKS H5MM_memcpy(((uint8_t *)entry_ptr->image_ptr) + entry_ptr->size, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); -#endif /* H5C_DO_MEMORY_SANITY_CHECKS */ +#endif /* H5C_DO_MEMORY_SANITY_CHECKS */ + } /* end if */ if (!(entry_ptr->image_up_to_date)) { + /* Sanity check */ HDassert(!entry_ptr->prefetched); /* Generate the entry's image */ if (H5C__generate_image(f, cache_ptr, entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "can't generate entry's image") + } /* end if ( ! (entry_ptr->image_up_to_date) ) */ } /* end if */ @@ -5910,10 +6610,12 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * flag should only be used in test code. */ if (write_entry) { + HDassert(entry_ptr->is_dirty); #if H5C_DO_SANITY_CHECKS - if (cache_ptr->check_write_permitted && !(cache_ptr->write_permitted)) + if ((cache_ptr->check_write_permitted) && (!(cache_ptr->write_permitted))) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Write when writes are always forbidden!?!?!") #endif /* H5C_DO_SANITY_CHECKS */ @@ -5924,38 +6626,50 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * H5AC__CLASS_SKIP_WRITES is set in the entry's type. This * flag should only be used in test code */ - if ((!suppress_image_entry_writes || !entry_ptr->include_in_image) && + if (((!suppress_image_entry_writes) || (!entry_ptr->include_in_image)) && (((entry_ptr->type->flags) & H5C__CLASS_SKIP_WRITES) == 0)) { + H5FD_mem_t mem_type = H5FD_MEM_DEFAULT; #ifdef H5_HAVE_PARALLEL if (cache_ptr->coll_write_list) { + if (H5SL_insert(cache_ptr->coll_write_list, entry_ptr, &entry_ptr->addr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "unable to insert skip list item") } /* end if */ else { #endif /* H5_HAVE_PARALLEL */ if (entry_ptr->prefetched) { + HDassert(entry_ptr->type->id == H5AC_PREFETCHED_ENTRY_ID); + mem_type = cache_ptr->class_table_ptr[entry_ptr->prefetch_type_id]->mem_type; } /* end if */ - else + else { + mem_type = entry_ptr->type->mem_type; + } if (H5F_block_write(f, mem_type, entry_ptr->addr, entry_ptr->size, entry_ptr->image_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't write image to file") + #ifdef H5_HAVE_PARALLEL } -#endif /* H5_HAVE_PARALLEL */ +#endif /* H5_HAVE_PARALLEL */ + } /* end if */ /* if the entry has a notify callback, notify it that we have * just flushed the entry. */ - if (entry_ptr->type->notify && - (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_AFTER_FLUSH, entry_ptr) < 0) + if ((entry_ptr->type->notify) && + ((entry_ptr->type->notify)(H5C_NOTIFY_ACTION_AFTER_FLUSH, entry_ptr) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client of entry flush") + } /* if ( write_entry ) */ /* At this point, all pre-serialize and serialize calls have been @@ -5968,16 +6682,21 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) /* start by updating the statistics */ if (clear_only) { + /* only log a clear if the entry was dirty */ if (was_dirty) { + H5C__UPDATE_STATS_FOR_CLEAR(cache_ptr, entry_ptr) + } /* end if */ } else if (write_entry) { + HDassert(was_dirty); /* only log a flush if we actually wrote to disk */ H5C__UPDATE_STATS_FOR_FLUSH(cache_ptr, entry_ptr) + } /* end else if */ /* Note that the algorithm below is (very) similar to the set of operations @@ -5987,11 +6706,17 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) /* Update the cache internal data structures. */ if (destroy) { + /* Sanity checks */ - if (take_ownership) + if (take_ownership) { + HDassert(!destroy_entry); - else + } + else { + HDassert(destroy_entry); + } + HDassert(!entry_ptr->is_pinned); /* Update stats, while entry is still in the cache */ @@ -6001,8 +6726,9 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * to be removed from the cache, send a 'before eviction' notice while * the entry is still fully integrated in the cache. */ - if (entry_ptr->type->notify && - (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_BEFORE_EVICT, entry_ptr) < 0) + if ((entry_ptr->type->notify) && + ((entry_ptr->type->notify)(H5C_NOTIFY_ACTION_BEFORE_EVICT, entry_ptr) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry to evict") /* Update the cache internal data structures as appropriate @@ -6023,14 +6749,19 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) */ H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr, FAIL) - if (entry_ptr->in_slist && del_from_slist_on_destroy) + if ((entry_ptr->in_slist) && (del_from_slist_on_destroy)) { + H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush) + } #ifdef H5_HAVE_PARALLEL /* Check for collective read access flag */ if (entry_ptr->coll_access) { + entry_ptr->coll_access = FALSE; + H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL) + } /* end if */ #endif /* H5_HAVE_PARALLEL */ @@ -6038,16 +6769,19 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) /* Remove entry from tag list */ if (H5C__untag_entry(cache_ptr, entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove entry from tag list") /* verify that the entry is no longer part of any flush dependencies */ HDassert(entry_ptr->flush_dep_nparents == 0); HDassert(entry_ptr->flush_dep_nchildren == 0); + } /* end if */ else { + HDassert(clear_only || write_entry); HDassert(entry_ptr->is_dirty); - HDassert(entry_ptr->in_slist); + HDassert((!cache_ptr->slist_enabled) || (entry_ptr->in_slist)); /* We are either doing a flush or a clear. * @@ -6055,7 +6789,7 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * view of the replacement policy and the slist. * Hence no differentiation between them. * - * JRM -- 7/7/07 + * JRM -- 7/7/07 */ H5C__UPDATE_RP_FOR_FLUSH(cache_ptr, entry_ptr, FAIL) @@ -6072,20 +6806,31 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) /* Check for entry changing status and do notifications, etc. */ if (was_dirty) { - /* If the entry's type has a 'notify' callback send a 'entry cleaned' - * notice now that the entry is fully integrated into the cache. + + /* If the entry's type has a 'notify' callback send a + * 'entry cleaned' notice now that the entry is fully + * integrated into the cache. */ - if (entry_ptr->type->notify && - (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_CLEANED, entry_ptr) < 0) + if ((entry_ptr->type->notify) && + ((entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_CLEANED, entry_ptr) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag cleared") - /* Propagate the clean flag up the flush dependency chain if appropriate */ - if (entry_ptr->flush_dep_ndirty_children != 0) + /* Propagate the clean flag up the flush dependency chain + * if appropriate + */ + if (entry_ptr->flush_dep_ndirty_children != 0) { + HDassert(entry_ptr->flush_dep_ndirty_children == 0); - if (entry_ptr->flush_dep_nparents > 0) + } + + if (entry_ptr->flush_dep_nparents > 0) { + if (H5C__mark_flush_dep_clean(entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKCLEAN, FAIL, "Can't propagate flush dep clean flag") + } } /* end if */ } /* end else */ @@ -6093,7 +6838,8 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) entry_ptr->flush_in_progress = FALSE; /* capture the cache entry address for the log_flush call at the - end before the entry_ptr gets freed */ + * end before the entry_ptr gets freed + */ entry_addr = entry_ptr->addr; /* Internal cache data structures should now be up to date, and @@ -6102,6 +6848,7 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * Now discard the entry if appropriate. */ if (destroy) { + /* Sanity check */ HDassert(0 == entry_ptr->flush_dep_nparents); @@ -6112,10 +6859,14 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * * Otherwise, free the buffer if it exists. */ - if (suppress_image_entry_frees && entry_ptr->include_in_image) + if (suppress_image_entry_frees && entry_ptr->include_in_image) { + entry_ptr->image_ptr = NULL; - else if (entry_ptr->image_ptr != NULL) + } + else if (entry_ptr->image_ptr != NULL) { + entry_ptr->image_ptr = H5MM_xfree(entry_ptr->image_ptr); + } /* If the entry is not a prefetched entry, verify that the flush * dependency parents addresses array has been transferred. @@ -6124,14 +6875,17 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * the flush dependency parents addresses array if necessary. */ if (!entry_ptr->prefetched) { + HDassert(0 == entry_ptr->fd_parent_count); HDassert(NULL == entry_ptr->fd_parent_addrs); + } /* end if */ /* Check whether we should free the space in the file that * the entry occupies */ if (free_file_space) { + hsize_t fsf_size; /* Sanity checks */ @@ -6152,15 +6906,22 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * Otherwise use entry_ptr->size. */ if (entry_ptr->type->fsf_size) { + if ((entry_ptr->type->fsf_size)((void *)entry_ptr, &fsf_size) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "unable to get file space free size") - } /* end if */ - else /* no file space free size callback -- use entry size */ + + } /* end if */ + else { /* no file space free size callback -- use entry size */ + fsf_size = entry_ptr->size; + } /* Release the space on disk */ if (H5MF_xfree(f, entry_ptr->type->mem_type, entry_ptr->addr, fsf_size) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "unable to free file space for cache entry") + } /* end if ( free_file_space ) */ /* Reset the pointer to the cache the entry is within. -QAK */ @@ -6183,23 +6944,31 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) */ cache_ptr->entries_removed_counter++; cache_ptr->last_entry_removed_ptr = entry_ptr; - if (entry_ptr == cache_ptr->entry_watched_for_removal) + + if (entry_ptr == cache_ptr->entry_watched_for_removal) { + cache_ptr->entry_watched_for_removal = NULL; + } /* Check for actually destroying the entry in memory */ /* (As opposed to taking ownership of it) */ if (destroy_entry) { + if (entry_ptr->is_dirty) { + /* Reset dirty flag */ entry_ptr->is_dirty = FALSE; - /* If the entry's type has a 'notify' callback send a 'entry cleaned' - * notice now that the entry is fully integrated into the cache. + /* If the entry's type has a 'notify' callback send a + * 'entry cleaned' notice now that the entry is fully + * integrated into the cache. */ - if (entry_ptr->type->notify && - (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_CLEANED, entry_ptr) < 0) + if ((entry_ptr->type->notify) && + ((entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_CLEANED, entry_ptr) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag cleared") + } /* end if */ /* we are about to discard the in core representation -- @@ -6212,9 +6981,12 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) HDassert(entry_ptr->image_ptr == NULL); if (entry_ptr->type->free_icr((void *)entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "free_icr callback failed") + } /* end if */ else { + HDassert(take_ownership); /* client is taking ownership of the entry. @@ -6222,6 +6994,7 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * unless the entry is re-inserted properly */ entry_ptr->magic = H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC; + } /* end else */ } /* if (destroy) */ @@ -6229,25 +7002,36 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * so it doesn't go out of date */ if (update_page_buffer) { + /* Sanity check */ HDassert(!destroy); HDassert(entry_ptr->image_ptr); - if (f->shared->page_buf && f->shared->page_buf->page_size >= entry_ptr->size) + if ((f->shared->page_buf) && (f->shared->page_buf->page_size >= entry_ptr->size)) { + if (H5PB_update_entry(f->shared->page_buf, entry_ptr->addr, entry_ptr->size, entry_ptr->image_ptr) > 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Failed to update PB with metadata cache") - } /* end if */ + } /* end if */ + } /* end if */ + + if (cache_ptr->log_flush) { - if (cache_ptr->log_flush) if ((cache_ptr->log_flush)(cache_ptr, entry_addr, was_dirty, flags) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "log_flush callback failed") + } /* end if */ + done: + HDassert((ret_value != SUCCEED) || (destroy_entry) || (!entry_ptr->flush_in_progress)); + HDassert((ret_value != SUCCEED) || (destroy_entry) || (take_ownership) || (!entry_ptr->is_dirty)); FUNC_LEAVE_NOAPI(ret_value) + } /* H5C__flush_single_entry() */ /*------------------------------------------------------------------------- @@ -7251,8 +8035,18 @@ H5C__entry_in_skip_list(H5C_t *cache_ptr, H5C_cache_entry_t *target_ptr) * Programmer: Mike McGreevy * November 3, 2010 * + * Changes: Modified function to setup the slist before calling + * H%C_flush_cache(), and take it down afterwards. Note + * that the slist need not be empty after the call to + * H5C_flush_cache() since we are only flushing marked + * entries. Thus must set the clear_slist parameter + * of H5C_set_slist_enabled to TRUE. + * + * JRM -- 5/6/20 + * *------------------------------------------------------------------------- */ + herr_t H5C__flush_marked_entries(H5F_t *f) { @@ -7263,12 +8057,28 @@ H5C__flush_marked_entries(H5F_t *f) /* Assertions */ HDassert(f != NULL); + /* Enable the slist, as it is needed in the flush */ + if (H5C_set_slist_enabled(f->shared->cache, TRUE, FALSE) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "set slist enabled failed") + /* Flush all marked entries */ if (H5C_flush_cache(f, H5C__FLUSH_MARKED_ENTRIES_FLAG | H5C__FLUSH_IGNORE_PROTECTED_FLAG) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush cache") + /* Disable the slist. Set the clear_slist parameter to TRUE + * since we called H5C_flush_cache() with the + * H5C__FLUSH_MARKED_ENTRIES_FLAG. + */ + if (H5C_set_slist_enabled(f->shared->cache, FALSE, TRUE) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "disable slist failed") + done: + FUNC_LEAVE_NOAPI(ret_value) + } /* H5C__flush_marked_entries */ /*------------------------------------------------------------------------- @@ -8135,6 +8945,10 @@ done: * Programmer: Mohamad Chaarawi * 2/10/16 * + * Changes: Updated sanity checks for the possibility that the skip + * list is disabled. + * JRM 5/16/20 + * *------------------------------------------------------------------------- */ static herr_t @@ -8163,15 +8977,18 @@ H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr) old_addr = entry_ptr->addr; /* Call client's pre-serialize callback, if there's one */ - if (entry_ptr->type->pre_serialize && - (entry_ptr->type->pre_serialize)(f, (void *)entry_ptr, entry_ptr->addr, entry_ptr->size, &new_addr, - &new_len, &serialize_flags) < 0) + if ((entry_ptr->type->pre_serialize) && + ((entry_ptr->type->pre_serialize)(f, (void *)entry_ptr, entry_ptr->addr, entry_ptr->size, &new_addr, + &new_len, &serialize_flags) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to pre-serialize entry") /* Check for any flags set in the pre-serialize callback */ if (serialize_flags != H5C__SERIALIZE_NO_FLAGS_SET) { + /* Check for unexpected flags from serialize callback */ if (serialize_flags & ~(H5C__SERIALIZE_RESIZED_FLAG | H5C__SERIALIZE_MOVED_FLAG)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unknown serialize flag(s)") #ifdef H5_HAVE_PARALLEL @@ -8202,20 +9019,25 @@ H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr) * tests will be necessary. */ if (cache_ptr->aux_ptr != NULL) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "resize/move in serialize occurred in parallel case") #endif /* If required, resize the buffer and update the entry and the cache - * data structures */ + * data structures + */ if (serialize_flags & H5C__SERIALIZE_RESIZED_FLAG) { + /* Sanity check */ HDassert(new_len > 0); /* Allocate a new image buffer */ if (NULL == (entry_ptr->image_ptr = H5MM_realloc(entry_ptr->image_ptr, new_len + H5C_IMAGE_EXTRA_SPACE))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for on disk image buffer") + #if H5C_DO_MEMORY_SANITY_CHECKS H5MM_memcpy(((uint8_t *)entry_ptr->image_ptr) + new_len, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); @@ -8237,25 +9059,31 @@ H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr) /* As we haven't updated the cache data structures for * for the flush or flush destroy yet, the entry should - * be in the slist. Thus update it for the size change. + * be in the slist if the slist is enabled. Since + * H5C__UPDATE_SLIST_FOR_SIZE_CHANGE() is a no-op if the + * slist is enabled, call it un-conditionally. */ HDassert(entry_ptr->is_dirty); - HDassert(entry_ptr->in_slist); + HDassert((entry_ptr->in_slist) || (!cache_ptr->slist_enabled)); + H5C__UPDATE_SLIST_FOR_SIZE_CHANGE(cache_ptr, entry_ptr->size, new_len); /* Finally, update the entry for its new size */ entry_ptr->size = new_len; + } /* end if */ /* If required, udate the entry and the cache data structures * for a move */ if (serialize_flags & H5C__SERIALIZE_MOVED_FLAG) { + /* Update stats and entries relocated counter */ H5C__UPDATE_STATS_FOR_MOVE(cache_ptr, entry_ptr) /* We must update cache data structures for the change in address */ if (entry_ptr->addr == old_addr) { + /* Delete the entry from the hash table and the slist */ H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr, FAIL); H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, FALSE); @@ -8266,19 +9094,25 @@ H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr) /* And then reinsert in the index and slist */ H5C__INSERT_IN_INDEX(cache_ptr, entry_ptr, FAIL); H5C__INSERT_ENTRY_IN_SLIST(cache_ptr, entry_ptr, FAIL); - } /* end if */ - else /* move is already done for us -- just do sanity checks */ + + } /* end if */ + else { /* move is already done for us -- just do sanity checks */ + HDassert(entry_ptr->addr == new_addr); + } } /* end if */ } /* end if(serialize_flags != H5C__SERIALIZE_NO_FLAGS_SET) */ /* Serialize object into buffer */ if (entry_ptr->type->serialize(f, entry_ptr->image_ptr, entry_ptr->size, (void *)entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to serialize entry") + #if H5C_DO_MEMORY_SANITY_CHECKS HDassert(0 == HDmemcmp(((uint8_t *)entry_ptr->image_ptr) + entry_ptr->size, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE)); #endif /* H5C_DO_MEMORY_SANITY_CHECKS */ + entry_ptr->image_up_to_date = TRUE; /* Propagate the fact that the entry is serialized up the @@ -8288,9 +9122,13 @@ H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr) * for flush dependency parents. */ HDassert(entry_ptr->flush_dep_nunser_children == 0); - if (entry_ptr->flush_dep_nparents > 0) + + if (entry_ptr->flush_dep_nparents > 0) { + if (H5C__mark_flush_dep_serialized(entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "Can't propagate serialization status to fd parents") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Cdbg.c b/src/H5Cdbg.c index 0752673..a8917ea 100644 --- a/src/H5Cdbg.c +++ b/src/H5Cdbg.c @@ -260,6 +260,12 @@ H5C_dump_cache_LRU(H5C_t *cache_ptr, const char *cache_name) * Programmer: John Mainzer * 11/15/14 * + * Changes: Updated function for the slist_enabled field in H5C_t. + * Recall that to minimize slist overhead, the slist is + * empty and not maintained if cache_ptr->slist_enabled is + * false. + * JRM -- 5/6/20 + * *------------------------------------------------------------------------- */ #ifndef NDEBUG @@ -278,6 +284,7 @@ H5C_dump_cache_skip_list(H5C_t *cache_ptr, char *calling_fcn) HDassert(calling_fcn != NULL); HDfprintf(stdout, "\n\nDumping metadata cache skip list from %s.\n", calling_fcn); + HDfprintf(stdout, " slist %s.\n", cache_ptr->slist_enabled ? "enabled" : "disabled"); HDfprintf(stdout, " slist len = %" PRIu32 ".\n", cache_ptr->slist_len); HDfprintf(stdout, " slist size = %zu.\n", cache_ptr->slist_size); diff --git a/src/H5Cimage.c b/src/H5Cimage.c index e876265..b3f6c12 100644 --- a/src/H5Cimage.c +++ b/src/H5Cimage.c @@ -453,6 +453,10 @@ done: * * Programmer: John Mainzer, 8/10/15 * + * Changes: Updated sanity checks for possibility that the slist + * is disabled. + * JRM -- 5/17/20 + * *------------------------------------------------------------------------- */ herr_t @@ -693,7 +697,10 @@ H5C__deserialize_prefetched_entry(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t pf_entry_ptr->image_ptr = NULL; if (pf_entry_ptr->is_dirty) { - HDassert(pf_entry_ptr->in_slist); + + HDassert(((cache_ptr->slist_enabled) && (pf_entry_ptr->in_slist)) || + ((!cache_ptr->slist_enabled) && (!pf_entry_ptr->in_slist))); + flush_flags |= H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG; } /* end if */ diff --git a/src/H5Cmpio.c b/src/H5Cmpio.c index 6c0c015..0489ad2 100644 --- a/src/H5Cmpio.c +++ b/src/H5Cmpio.c @@ -154,6 +154,10 @@ static herr_t H5C__flush_candidates_in_ring(H5F_t *f, H5C_ring_t ring, unsigned * Programmer: John Mainzer * 3/17/10 * + * Changes: Updated sanity checks to allow for the possibility that + * the slist is disabled. + * JRM -- 8/3/20 + * *------------------------------------------------------------------------- */ herr_t @@ -190,7 +194,7 @@ H5C_apply_candidate_list(H5F_t *f, H5C_t *cache_ptr, unsigned num_candidates, ha HDassert(cache_ptr != NULL); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); HDassert(num_candidates > 0); - HDassert(num_candidates <= cache_ptr->slist_len); + HDassert((!cache_ptr->slist_enabled) || (num_candidates <= cache_ptr->slist_len)); HDassert(candidates_list_ptr != NULL); HDassert(0 <= mpi_rank); HDassert(mpi_rank < mpi_size); @@ -416,6 +420,16 @@ done: * Programmer: John Mainzer * 3/17/10 * + * Changes: With the slist optimization, the slist is not maintained + * unless a flush is in progress. Thus we can not longer use + * cache_ptr->slist_size to determine the total size of + * the entries we must insert in the candidate list. + * + * To address this, we now use cache_ptr->dirty_index_size + * instead. + * + * JRM -- 7/27/20 + * *------------------------------------------------------------------------- */ herr_t @@ -429,18 +443,22 @@ H5C_construct_candidate_list__clean_cache(H5C_t *cache_ptr) HDassert(cache_ptr != NULL); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - /* As a sanity check, set space needed to the size of the skip list. - * This should be the sum total of the sizes of all the dirty entries - * in the metadata cache. + /* As a sanity check, set space needed to the dirty_index_size. This + * should be the sum total of the sizes of all the dirty entries + * in the metadata cache. Note that if the slist is enabled, + * cache_ptr->slist_size should equal cache_ptr->dirty_index_size. */ - space_needed = cache_ptr->slist_size; + space_needed = cache_ptr->dirty_index_size; + + HDassert((!cache_ptr->slist_enabled) || (space_needed == cache_ptr->slist_size)); /* Recall that while we shouldn't have any protected entries at this * point, it is possible that some dirty entries may reside on the * pinned list at this point. */ - HDassert(cache_ptr->slist_size <= (cache_ptr->dLRU_list_size + cache_ptr->pel_size)); - HDassert(cache_ptr->slist_len <= (cache_ptr->dLRU_list_len + cache_ptr->pel_len)); + HDassert(cache_ptr->dirty_index_size <= (cache_ptr->dLRU_list_size + cache_ptr->pel_size)); + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_len <= (cache_ptr->dLRU_list_len + cache_ptr->pel_len))); if (space_needed > 0) { /* we have work to do */ @@ -449,7 +467,7 @@ H5C_construct_candidate_list__clean_cache(H5C_t *cache_ptr) size_t nominated_entries_size = 0; haddr_t nominated_addr; - HDassert(cache_ptr->slist_len > 0); + HDassert((!cache_ptr->slist_enabled) || (cache_ptr->slist_len > 0)); /* Scan the dirty LRU list from tail forward and nominate sufficient * entries to free up the necessary space. @@ -457,14 +475,14 @@ H5C_construct_candidate_list__clean_cache(H5C_t *cache_ptr) entry_ptr = cache_ptr->dLRU_tail_ptr; while ((nominated_entries_size < space_needed) && - (nominated_entries_count < cache_ptr->slist_len) && + ((!cache_ptr->slist_enabled) || (nominated_entries_count < cache_ptr->slist_len)) && (entry_ptr != NULL)) { HDassert(!(entry_ptr->is_protected)); HDassert(!(entry_ptr->is_read_only)); HDassert(entry_ptr->ro_ref_count == 0); HDassert(entry_ptr->is_dirty); - HDassert(entry_ptr->in_slist); + HDassert((!cache_ptr->slist_enabled) || (entry_ptr->in_slist)); nominated_addr = entry_ptr->addr; @@ -484,7 +502,9 @@ H5C_construct_candidate_list__clean_cache(H5C_t *cache_ptr) * protected entry list as well -- scan it too if necessary */ entry_ptr = cache_ptr->pel_head_ptr; - while ((nominated_entries_size < space_needed) && (nominated_entries_count < cache_ptr->slist_len) && + + while ((nominated_entries_size < space_needed) && + ((!cache_ptr->slist_enabled) || (nominated_entries_count < cache_ptr->slist_len)) && (entry_ptr != NULL)) { if (entry_ptr->is_dirty) { @@ -510,7 +530,7 @@ H5C_construct_candidate_list__clean_cache(H5C_t *cache_ptr) } /* end while */ - HDassert(nominated_entries_count == cache_ptr->slist_len); + HDassert((!cache_ptr->slist_enabled) || (nominated_entries_count == cache_ptr->slist_len)); HDassert(nominated_entries_size == space_needed); } /* end if */ @@ -537,6 +557,12 @@ done: * Programmer: John Mainzer * 3/17/10 * + * Changes: With the slist optimization, the slist is not maintained + * unless a flush is in progress. Updated sanity checks to + * reflect this. + * + * JRM -- 7/27/20 + * *------------------------------------------------------------------------- */ herr_t @@ -556,17 +582,26 @@ H5C_construct_candidate_list__min_clean(H5C_t *cache_ptr) if (cache_ptr->max_cache_size > cache_ptr->index_size) { if (((cache_ptr->max_cache_size - cache_ptr->index_size) + cache_ptr->cLRU_list_size) >= - cache_ptr->min_clean_size) + cache_ptr->min_clean_size) { + space_needed = 0; - else + } + else { + space_needed = cache_ptr->min_clean_size - ((cache_ptr->max_cache_size - cache_ptr->index_size) + cache_ptr->cLRU_list_size); + } } /* end if */ else { - if (cache_ptr->min_clean_size <= cache_ptr->cLRU_list_size) + + if (cache_ptr->min_clean_size <= cache_ptr->cLRU_list_size) { + space_needed = 0; - else + } + else { + space_needed = cache_ptr->min_clean_size - cache_ptr->cLRU_list_size; + } } /* end else */ if (space_needed > 0) { /* we have work to do */ @@ -575,13 +610,15 @@ H5C_construct_candidate_list__min_clean(H5C_t *cache_ptr) unsigned nominated_entries_count = 0; size_t nominated_entries_size = 0; - HDassert(cache_ptr->slist_len > 0); + HDassert((!cache_ptr->slist_enabled) || (cache_ptr->slist_len > 0)); /* Scan the dirty LRU list from tail forward and nominate sufficient * entries to free up the necessary space. */ entry_ptr = cache_ptr->dLRU_tail_ptr; - while ((nominated_entries_size < space_needed) && (nominated_entries_count < cache_ptr->slist_len) && + + while ((nominated_entries_size < space_needed) && + ((!cache_ptr->slist_enabled) || (nominated_entries_count < cache_ptr->slist_len)) && (entry_ptr != NULL) && (!entry_ptr->flush_me_last)) { haddr_t nominated_addr; @@ -590,7 +627,7 @@ H5C_construct_candidate_list__min_clean(H5C_t *cache_ptr) HDassert(!(entry_ptr->is_read_only)); HDassert(entry_ptr->ro_ref_count == 0); HDassert(entry_ptr->is_dirty); - HDassert(entry_ptr->in_slist); + HDassert((!cache_ptr->slist_enabled) || (entry_ptr->in_slist)); nominated_addr = entry_ptr->addr; @@ -603,7 +640,9 @@ H5C_construct_candidate_list__min_clean(H5C_t *cache_ptr) entry_ptr = entry_ptr->aux_prev; } /* end while */ - HDassert(nominated_entries_count <= cache_ptr->slist_len); + + HDassert((!cache_ptr->slist_enabled) || (nominated_entries_count <= cache_ptr->slist_len)); + HDassert(nominated_entries_size <= cache_ptr->dirty_index_size); HDassert(nominated_entries_size >= space_needed); } /* end if */ diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h index 409dccc..410fc3e 100644 --- a/src/H5Cpkg.h +++ b/src/H5Cpkg.h @@ -59,6 +59,13 @@ /* Initial allocated size of the "flush_dep_parent" array */ #define H5C_FLUSH_DEP_PARENT_INIT 8 + +/* Set to TRUE to enable the slist optimization. If this field is TRUE, + * the slist is disabled whenever a flush is not in progress. + */ +#define H5C__SLIST_OPT_ENABLED TRUE + + /**************************************************************************** * * We maintain doubly linked lists of instances of H5C_cache_entry_t for a @@ -1573,9 +1580,27 @@ if ( ( (cache_ptr)->index_size != \ * Added code to maintain the cache_ptr->slist_ring_len * and cache_ptr->slist_ring_size arrays. * + * JRM -- 4/29/20 + * Reworked macro to support the slist_enabled field + * of H5C_t. If slist_enabled == TRUE, the macro + * functions as before. Otherwise, the macro is a no-op, + * and the slist must be empty. + * *------------------------------------------------------------------------- */ +/* NOTE: The H5C__INSERT_ENTRY_IN_SLIST() macro is set up so that + * + * H5C_DO_SANITY_CHECKS + * + * and + * + * H5C_DO_SLIST_SANITY_CHECKS + * + * can be selected independantly. This is easy to miss as the + * two #defines are easy to confuse. + */ + #if H5C_DO_SLIST_SANITY_CHECKS #define ENTRY_IN_SLIST(cache_ptr, entry_ptr) \ @@ -1594,33 +1619,43 @@ if ( ( (cache_ptr)->index_size != \ { \ HDassert( (cache_ptr) ); \ HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ - HDassert( (entry_ptr) ); \ - HDassert( (entry_ptr)->size > 0 ); \ - HDassert( H5F_addr_defined((entry_ptr)->addr) ); \ - HDassert( !((entry_ptr)->in_slist) ); \ - HDassert( !ENTRY_IN_SLIST((cache_ptr), (entry_ptr)) ); \ - HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ - HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ - HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_len ); \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_size ); \ \ - if(H5SL_insert((cache_ptr)->slist_ptr, entry_ptr, &(entry_ptr)->addr) < 0) \ - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, (fail_val), "can't insert entry in skip list") \ + if ( (cache_ptr)->slist_enabled ) { \ \ - (entry_ptr)->in_slist = TRUE; \ - (cache_ptr)->slist_changed = TRUE; \ - (cache_ptr)->slist_len++; \ - (cache_ptr)->slist_size += (entry_ptr)->size; \ - ((cache_ptr)->slist_ring_len[(entry_ptr)->ring])++; \ - ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) += (entry_ptr)->size; \ - (cache_ptr)->slist_len_increase++; \ - (cache_ptr)->slist_size_increase += (int64_t)((entry_ptr)->size); \ + HDassert( (entry_ptr) ); \ + HDassert( (entry_ptr)->size > 0 ); \ + HDassert( H5F_addr_defined((entry_ptr)->addr) ); \ + HDassert( !((entry_ptr)->in_slist) ); \ + HDassert( ! ENTRY_IN_SLIST((cache_ptr), (entry_ptr)) ); \ + HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ + HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ + HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_len ); \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_size ); \ \ - HDassert( (cache_ptr)->slist_len > 0 ); \ - HDassert( (cache_ptr)->slist_size > 0 ); \ + if ( H5SL_insert((cache_ptr)->slist_ptr, entry_ptr, \ + &((entry_ptr)->addr)) < 0) \ + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, (fail_val), \ + "can't insert entry in skip list") \ \ + (entry_ptr)->in_slist = TRUE; \ + (cache_ptr)->slist_changed = TRUE; \ + (cache_ptr)->slist_len++; \ + (cache_ptr)->slist_size += (entry_ptr)->size; \ + ((cache_ptr)->slist_ring_len[(entry_ptr)->ring])++; \ + ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) += (entry_ptr)->size;\ + (cache_ptr)->slist_len_increase++; \ + (cache_ptr)->slist_size_increase += (int64_t)((entry_ptr)->size); \ + \ + HDassert( (cache_ptr)->slist_len > 0 ); \ + HDassert( (cache_ptr)->slist_size > 0 ); \ + \ + } else { /* slist disabled */ \ + \ + HDassert( (cache_ptr)->slist_len == 0 ); \ + HDassert( (cache_ptr)->slist_size == 0 ); \ + } \ } /* H5C__INSERT_ENTRY_IN_SLIST */ #else /* H5C_DO_SANITY_CHECKS */ @@ -1629,31 +1664,42 @@ if ( ( (cache_ptr)->index_size != \ { \ HDassert( (cache_ptr) ); \ HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ - HDassert( (entry_ptr) ); \ - HDassert( (entry_ptr)->size > 0 ); \ - HDassert( H5F_addr_defined((entry_ptr)->addr) ); \ - HDassert( !((entry_ptr)->in_slist) ); \ - HDassert( !ENTRY_IN_SLIST((cache_ptr), (entry_ptr)) ); \ - HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ - HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ - HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_len ); \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_size ); \ \ - if(H5SL_insert((cache_ptr)->slist_ptr, entry_ptr, &(entry_ptr)->addr) < 0) \ - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, (fail_val), "can't insert entry in skip list") \ + if ( (cache_ptr)->slist_enabled ) { \ + \ + HDassert( (entry_ptr) ); \ + HDassert( (entry_ptr)->size > 0 ); \ + HDassert( ! ENTRY_IN_SLIST((cache_ptr), (entry_ptr)) ); \ + HDassert( H5F_addr_defined((entry_ptr)->addr) ); \ + HDassert( !((entry_ptr)->in_slist) ); \ + HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ + HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ + HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_len ); \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_size ); \ + HDassert( (cache_ptr)->slist_ptr ); \ \ - (entry_ptr)->in_slist = TRUE; \ - (cache_ptr)->slist_changed = TRUE; \ - (cache_ptr)->slist_len++; \ - (cache_ptr)->slist_size += (entry_ptr)->size; \ - ((cache_ptr)->slist_ring_len[(entry_ptr)->ring])++; \ - ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) += (entry_ptr)->size; \ + if ( H5SL_insert((cache_ptr)->slist_ptr, entry_ptr, \ + &((entry_ptr)->addr)) < 0) \ + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, (fail_val), \ + "can't insert entry in skip list") \ \ - HDassert( (cache_ptr)->slist_len > 0 ); \ - HDassert( (cache_ptr)->slist_size > 0 ); \ + (entry_ptr)->in_slist = TRUE; \ + (cache_ptr)->slist_changed = TRUE; \ + (cache_ptr)->slist_len++; \ + (cache_ptr)->slist_size += (entry_ptr)->size; \ + ((cache_ptr)->slist_ring_len[(entry_ptr)->ring])++; \ + ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) += (entry_ptr)->size;\ \ + HDassert( (cache_ptr)->slist_len > 0 ); \ + HDassert( (cache_ptr)->slist_size > 0 ); \ + \ + } else { /* slist disabled */ \ + \ + HDassert( (cache_ptr)->slist_len == 0 ); \ + HDassert( (cache_ptr)->slist_size == 0 ); \ + } \ } /* H5C__INSERT_ENTRY_IN_SLIST */ #endif /* H5C_DO_SANITY_CHECKS */ @@ -1671,79 +1717,127 @@ if ( ( (cache_ptr)->index_size != \ * * Programmer: John Mainzer, 5/10/04 * + * Modifications: + * + * JRM -- 7/21/04 + * Updated function for the addition of the hash table. + * + * JRM - 7/27/04 + * Converted from the function H5C_remove_entry_from_tree() + * to the macro H5C__REMOVE_ENTRY_FROM_TREE in the hopes of + * wringing a little more performance out of the cache. + * + * QAK -- 11/27/04 + * Switched over to using skip list routines. + * + * JRM -- 3/28/07 + * Updated sanity checks for the new is_read_only and + * ro_ref_count fields in H5C_cache_entry_t. + * + * JRM -- 12/13/14 + * Added code to set cache_ptr->slist_changed to TRUE + * when an entry is removed from the slist. + * + * JRM -- 4/29/20 + * Reworked macro to support the slist_enabled field + * of H5C_t. If slist_enabled == TRUE, the macro + * functions as before. Otherwise, the macro is a no-op, + * and the slist must be empty. + * *------------------------------------------------------------------------- */ #if H5C_DO_SANITY_CHECKS -#define H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush) \ -{ \ - HDassert( (cache_ptr) ); \ - HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ - HDassert( (entry_ptr) ); \ - HDassert( !((entry_ptr)->is_read_only) ); \ - HDassert( ((entry_ptr)->ro_ref_count) == 0 ); \ - HDassert( (entry_ptr)->size > 0 ); \ - HDassert( (entry_ptr)->in_slist ); \ - HDassert( (cache_ptr)->slist_ptr ); \ - HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ - HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ - HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_len ); \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_size ); \ - \ - if ( H5SL_remove((cache_ptr)->slist_ptr, &(entry_ptr)->addr) \ - != (entry_ptr) ) \ - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "can't delete entry from skip list") \ - \ - HDassert( (cache_ptr)->slist_len > 0 ); \ - if(!(during_flush)) \ - (cache_ptr)->slist_changed = TRUE; \ - (cache_ptr)->slist_len--; \ - HDassert( (cache_ptr)->slist_size >= (entry_ptr)->size ); \ - (cache_ptr)->slist_size -= (entry_ptr)->size; \ - ((cache_ptr)->slist_ring_len[(entry_ptr)->ring])--; \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr->ring)] >= \ - (entry_ptr)->size ); \ - ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) -= (entry_ptr)->size; \ - (cache_ptr)->slist_len_increase--; \ - (cache_ptr)->slist_size_increase -= (int64_t)((entry_ptr)->size); \ - (entry_ptr)->in_slist = FALSE; \ +#define H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush) \ +{ \ + HDassert( (cache_ptr) ); \ + HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ + \ + if ( (cache_ptr)->slist_enabled ) { \ + \ + HDassert( (entry_ptr) ); \ + HDassert( !((entry_ptr)->is_read_only) ); \ + HDassert( ((entry_ptr)->ro_ref_count) == 0 ); \ + HDassert( (entry_ptr)->size > 0 ); \ + HDassert( (entry_ptr)->in_slist ); \ + HDassert( (cache_ptr)->slist_ptr ); \ + HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ + HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ + HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_len ); \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_size ); \ + HDassert( (cache_ptr)->slist_size >= (entry_ptr)->size ); \ + \ + if ( H5SL_remove((cache_ptr)->slist_ptr, &(entry_ptr)->addr) \ + != (entry_ptr) ) \ + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, \ + "can't delete entry from skip list") \ + \ + HDassert( (cache_ptr)->slist_len > 0 ); \ + if(!(during_flush)) \ + (cache_ptr)->slist_changed = TRUE; \ + (cache_ptr)->slist_len--; \ + HDassert( (cache_ptr)->slist_size >= (entry_ptr)->size ); \ + (cache_ptr)->slist_size -= (entry_ptr)->size; \ + ((cache_ptr)->slist_ring_len[(entry_ptr)->ring])--; \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr->ring)] >= \ + (entry_ptr)->size ); \ + ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) -= (entry_ptr)->size;\ + (cache_ptr)->slist_len_increase--; \ + (cache_ptr)->slist_size_increase -= (int64_t)((entry_ptr)->size); \ + (entry_ptr)->in_slist = FALSE; \ + \ + } else { /* slist disabled */ \ + \ + HDassert( (cache_ptr)->slist_len == 0 ); \ + HDassert( (cache_ptr)->slist_size == 0 ); \ + } \ } /* H5C__REMOVE_ENTRY_FROM_SLIST */ #else /* H5C_DO_SANITY_CHECKS */ -#define H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush) \ -{ \ - HDassert( (cache_ptr) ); \ - HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ - HDassert( (entry_ptr) ); \ - HDassert( !((entry_ptr)->is_read_only) ); \ - HDassert( ((entry_ptr)->ro_ref_count) == 0 ); \ - HDassert( (entry_ptr)->in_slist ); \ - HDassert( (cache_ptr)->slist_ptr ); \ - HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ - HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ - HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_len ); \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_size ); \ - \ - if ( H5SL_remove((cache_ptr)->slist_ptr, &(entry_ptr)->addr) \ - != (entry_ptr) ) \ - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "can't delete entry from skip list") \ - \ - HDassert( (cache_ptr)->slist_len > 0 ); \ - if(!(during_flush)) \ - (cache_ptr)->slist_changed = TRUE; \ - (cache_ptr)->slist_len--; \ - HDassert( (cache_ptr)->slist_size >= (entry_ptr)->size ); \ - (cache_ptr)->slist_size -= (entry_ptr)->size; \ - ((cache_ptr)->slist_ring_len[(entry_ptr)->ring])--; \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr->ring)] >= \ - (entry_ptr)->size ); \ - ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) -= (entry_ptr)->size; \ - (entry_ptr)->in_slist = FALSE; \ +#define H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush) \ +{ \ + HDassert( (cache_ptr) ); \ + HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ + \ + if ( (cache_ptr)->slist_enabled ) { \ + \ + HDassert( (entry_ptr) ); \ + HDassert( !((entry_ptr)->is_read_only) ); \ + HDassert( ((entry_ptr)->ro_ref_count) == 0 ); \ + HDassert( (entry_ptr)->in_slist ); \ + HDassert( (cache_ptr)->slist_ptr ); \ + HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ + HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ + HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_len ); \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_size ); \ + \ + if ( H5SL_remove((cache_ptr)->slist_ptr, &(entry_ptr)->addr) \ + != (entry_ptr) ) \ + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, \ + "can't delete entry from skip list") \ + \ + HDassert( (cache_ptr)->slist_len > 0 ); \ + if(!(during_flush)) \ + (cache_ptr)->slist_changed = TRUE; \ + (cache_ptr)->slist_len--; \ + HDassert( (cache_ptr)->slist_size >= (entry_ptr)->size ); \ + (cache_ptr)->slist_size -= (entry_ptr)->size; \ + ((cache_ptr)->slist_ring_len[(entry_ptr)->ring])--; \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr->ring)] >= \ + (entry_ptr)->size ); \ + ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) -= (entry_ptr)->size;\ + (entry_ptr)->in_slist = FALSE; \ + \ + } else { /* slist disabled */ \ + \ + HDassert( (cache_ptr)->slist_len == 0 ); \ + HDassert( (cache_ptr)->slist_size == 0 ); \ + } \ } /* H5C__REMOVE_ENTRY_FROM_SLIST */ #endif /* H5C_DO_SANITY_CHECKS */ @@ -1781,6 +1875,12 @@ if ( ( (cache_ptr)->index_size != \ * Added code to maintain the cache_ptr->slist_ring_len * and cache_ptr->slist_ring_size arrays. * + * JRM -- 4/29/20 + * Reworked macro to support the slist_enabled field + * of H5C_t. If slist_enabled == TRUE, the macro + * functions as before. Otherwise, the macro is a no-op, + * and the slist must be empty. + * *------------------------------------------------------------------------- */ @@ -1790,32 +1890,43 @@ if ( ( (cache_ptr)->index_size != \ { \ HDassert( (cache_ptr) ); \ HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ - HDassert( (old_size) > 0 ); \ - HDassert( (new_size) > 0 ); \ - HDassert( (old_size) <= (cache_ptr)->slist_size ); \ - HDassert( (cache_ptr)->slist_len > 0 ); \ - HDassert( ((cache_ptr)->slist_len > 1) || \ - ( (cache_ptr)->slist_size == (old_size) ) ); \ - HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ - HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ - HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_len ); \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_size ); \ \ - (cache_ptr)->slist_size -= (old_size); \ - (cache_ptr)->slist_size += (new_size); \ + if ( (cache_ptr)->slist_enabled ) { \ + \ + HDassert( (old_size) > 0 ); \ + HDassert( (new_size) > 0 ); \ + HDassert( (old_size) <= (cache_ptr)->slist_size ); \ + HDassert( (cache_ptr)->slist_len > 0 ); \ + HDassert( ((cache_ptr)->slist_len > 1) || \ + ( (cache_ptr)->slist_size == (old_size) ) ); \ + HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ + HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ + HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_len ); \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_size ); \ + \ + (cache_ptr)->slist_size -= (old_size); \ + (cache_ptr)->slist_size += (new_size); \ + \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr->ring)] \ + >= (old_size) ); \ \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr->ring)] >=(old_size) ); \ - ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) -= (old_size); \ - ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) += (new_size); \ + ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) -= (old_size); \ + ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) += (new_size); \ \ - (cache_ptr)->slist_size_increase -= (int64_t)(old_size); \ - (cache_ptr)->slist_size_increase += (int64_t)(new_size); \ + (cache_ptr)->slist_size_increase -= (int64_t)(old_size); \ + (cache_ptr)->slist_size_increase += (int64_t)(new_size); \ \ - HDassert( (new_size) <= (cache_ptr)->slist_size ); \ - HDassert( ( (cache_ptr)->slist_len > 1 ) || \ - ( (cache_ptr)->slist_size == (new_size) ) ); \ + HDassert( (new_size) <= (cache_ptr)->slist_size ); \ + HDassert( ( (cache_ptr)->slist_len > 1 ) || \ + ( (cache_ptr)->slist_size == (new_size) ) ); \ + \ + } else { /* slist disabled */ \ + \ + HDassert( (cache_ptr)->slist_len == 0 ); \ + HDassert( (cache_ptr)->slist_size == 0 ); \ + } \ } /* H5C__UPDATE_SLIST_FOR_SIZE_CHANGE */ #else /* H5C_DO_SANITY_CHECKS */ @@ -1824,29 +1935,39 @@ if ( ( (cache_ptr)->index_size != \ { \ HDassert( (cache_ptr) ); \ HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ - HDassert( (old_size) > 0 ); \ - HDassert( (new_size) > 0 ); \ - HDassert( (old_size) <= (cache_ptr)->slist_size ); \ - HDassert( (cache_ptr)->slist_len > 0 ); \ - HDassert( ((cache_ptr)->slist_len > 1) || \ - ( (cache_ptr)->slist_size == (old_size) ) ); \ - HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ - HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ - HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_len ); \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_size ); \ \ - (cache_ptr)->slist_size -= (old_size); \ - (cache_ptr)->slist_size += (new_size); \ + if ( (cache_ptr)->slist_enabled ) { \ + \ + HDassert( (old_size) > 0 ); \ + HDassert( (new_size) > 0 ); \ + HDassert( (old_size) <= (cache_ptr)->slist_size ); \ + HDassert( (cache_ptr)->slist_len > 0 ); \ + HDassert( ((cache_ptr)->slist_len > 1) || \ + ( (cache_ptr)->slist_size == (old_size) ) ); \ + HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ + HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ + HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_len ); \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_size ); \ + \ + (cache_ptr)->slist_size -= (old_size); \ + (cache_ptr)->slist_size += (new_size); \ \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr->ring)] >=(old_size) ); \ - ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) -= (old_size); \ - ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) += (new_size); \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr->ring)] >= \ + (old_size) ); \ + ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) -= (old_size); \ + ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) += (new_size); \ \ - HDassert( (new_size) <= (cache_ptr)->slist_size ); \ - HDassert( ( (cache_ptr)->slist_len > 1 ) || \ - ( (cache_ptr)->slist_size == (new_size) ) ); \ + HDassert( (new_size) <= (cache_ptr)->slist_size ); \ + HDassert( ( (cache_ptr)->slist_len > 1 ) || \ + ( (cache_ptr)->slist_size == (new_size) ) ); \ + \ + } else { /* slist disabled */ \ + \ + HDassert( (cache_ptr)->slist_len == 0 ); \ + HDassert( (cache_ptr)->slist_size == 0 ); \ + } \ } /* H5C__UPDATE_SLIST_FOR_SIZE_CHANGE */ #endif /* H5C_DO_SANITY_CHECKS */ @@ -3752,6 +3873,36 @@ typedef struct H5C_tag_info_t { * are flushed. (this has been changed -- dirty entries are now removed from * the skip list as they are flushed. JRM - 10/25/05) * + * Update 4/21/20: + * + * Profiling indicates that the cost of maintaining the skip list is + * significant. As it is only used on flush and close, maintaining it + * only when needed is an obvious optimization. + * + * To do this, we add a flag to control maintenanace of the skip list. + * This flag is initially set to FALSE, which disables all operations + * on the skip list. + * + * At the beginning of either flush or close, we scan the index list, + * insert all dirtly entries in the skip list, and enable operations + * on skip list by setting above control flag to true. + * + * At the end of a complete flush, we verify that the skip list is empty, + * and set the control flag back to false, so as to avoid skip list + * maintenance overhead until the next flush or close. + * + * In the case of a partial flush (i.e. flush marked entries), we remove + * all remaining entries from the skip list, and then set the control flag + * back to false -- again avoiding skip list maintenance overhead until + * the next flush or close. + * + * slist_enabled: Boolean flag used to control operation of the skip + * list. If this filed is FALSE, operations on the + * slist are no-ops, and the slist must be empty. If + * it is TRUE, operations on the slist proceed as usual, + * and all dirty entries in the metadata cache must be + * listed in the slist. + * * slist_changed: Boolean flag used to indicate whether the contents of * the slist has changed since the last time this flag was * reset. This is used in the cache flush code to detect @@ -4698,6 +4849,7 @@ struct H5C_t { H5C_cache_entry_t * entry_watched_for_removal; /* Fields for maintaining list of in-order entries, for flushing */ + hbool_t slist_enabled; hbool_t slist_changed; uint32_t slist_len; size_t slist_size; diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index 80ab5ff..ac597bf 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -2270,6 +2270,7 @@ H5_DLL herr_t H5C_resize_entry(void *thing, size_t new_size); H5_DLL herr_t H5C_set_cache_auto_resize_config(H5C_t *cache_ptr, H5C_auto_size_ctl_t *config_ptr); H5_DLL herr_t H5C_set_cache_image_config(const H5F_t *f, H5C_t *cache_ptr, H5C_cache_image_ctl_t *config_ptr); H5_DLL herr_t H5C_set_evictions_enabled(H5C_t *cache_ptr, hbool_t evictions_enabled); +H5_DLL herr_t H5C_set_slist_enabled(H5C_t *cache_ptr, hbool_t slist_enabled, hbool_t clear_slist); H5_DLL herr_t H5C_set_prefix(H5C_t *cache_ptr, char *prefix); H5_DLL herr_t H5C_stats(H5C_t *cache_ptr, const char *cache_name, hbool_t display_detailed_stats); H5_DLL void H5C_stats__reset(H5C_t *cache_ptr); diff --git a/src/H5Dint.c b/src/H5Dint.c index cfed336..7a2b93b 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -1277,18 +1277,24 @@ H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id, hid_t /* Check if the dataset has a non-default DCPL & get important values, if so */ if (new_dset->shared->dcpl_id != H5P_DATASET_CREATE_DEFAULT) { - H5O_layout_t *layout; /* Dataset's layout information */ - H5O_pline_t * pline; /* Dataset's I/O pipeline information */ - H5O_fill_t * fill; /* Dataset's fill value info */ - H5O_efl_t * efl; /* Dataset's external file list info */ + H5O_layout_t *layout; /* Dataset's layout information */ + H5O_pline_t * pline; /* Dataset's I/O pipeline information */ + H5O_fill_t * fill; /* Dataset's fill value info */ + H5O_efl_t * efl; /* Dataset's external file list info */ + htri_t ignore_filters = FALSE; /* Ignore optional filters or not */ - /* Check if the filters in the DCPL can be applied to this dataset */ - if (H5Z_can_apply(new_dset->shared->dcpl_id, new_dset->shared->type_id) < 0) - HGOTO_ERROR(H5E_ARGS, H5E_CANTINIT, NULL, "I/O filters can't operate on this dataset") + if ((ignore_filters = H5Z_ignore_filters(new_dset->shared->dcpl_id, dt, space)) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_CANTINIT, NULL, "H5Z_has_optional_filter() failed") - /* Make the "set local" filter callbacks for this dataset */ - if (H5Z_set_local(new_dset->shared->dcpl_id, new_dset->shared->type_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to set local filter parameters") + if (FALSE == ignore_filters) { + /* Check if the filters in the DCPL can be applied to this dataset */ + if (H5Z_can_apply(new_dset->shared->dcpl_id, new_dset->shared->type_id) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_CANTINIT, NULL, "I/O filters can't operate on this dataset") + + /* Make the "set local" filter callbacks for this dataset */ + if (H5Z_set_local(new_dset->shared->dcpl_id, new_dset->shared->type_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to set local filter parameters") + } /* ignore_filters */ /* Get new dataset's property list object */ if (NULL == (dc_plist = (H5P_genplist_t *)H5I_object(new_dset->shared->dcpl_id))) @@ -1312,9 +1318,11 @@ H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id, hid_t HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, NULL, "can't retrieve external file list") efl_copied = TRUE; - /* Check that chunked layout is used if filters are enabled */ - if (pline->nused > 0 && H5D_CHUNKED != layout->type) - HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, NULL, "filters can only be used with chunked layout") + if (FALSE == ignore_filters) { + /* Check that chunked layout is used if filters are enabled */ + if (pline->nused > 0 && H5D_CHUNKED != layout->type) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, NULL, "filters can only be used with chunked layout") + } /* Check if the alloc_time is the default and error out */ if (fill->alloc_time == H5D_ALLOC_TIME_DEFAULT) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 8c2cace..2588d1c 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -302,22 +302,23 @@ done: herr_t H5D_virtual_update_min_dims(H5O_layout_t *layout, size_t idx) { - H5S_sel_type sel_type; - int rank; - hsize_t bounds_start[H5S_MAX_RANK]; - hsize_t bounds_end[H5S_MAX_RANK]; - int i; - herr_t ret_value = SUCCEED; + H5O_storage_virtual_t * virt = &layout->storage.u.virt; + H5O_storage_virtual_ent_t *ent = &virt->list[idx]; + H5S_sel_type sel_type; + int rank; + hsize_t bounds_start[H5S_MAX_RANK]; + hsize_t bounds_end[H5S_MAX_RANK]; + int i; + herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI(FAIL) HDassert(layout); HDassert(layout->type == H5D_VIRTUAL); - HDassert(idx < layout->storage.u.virt.list_nalloc); + HDassert(idx < virt->list_nalloc); /* Get type of selection */ - if (H5S_SEL_ERROR == - (sel_type = H5S_GET_SELECT_TYPE(layout->storage.u.virt.list[idx].source_dset.virtual_select))) + if (H5S_SEL_ERROR == (sel_type = H5S_GET_SELECT_TYPE(ent->source_dset.virtual_select))) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection type") /* Do not update min_dims for "all" or "none" selections */ @@ -325,20 +326,18 @@ H5D_virtual_update_min_dims(H5O_layout_t *layout, size_t idx) HGOTO_DONE(SUCCEED) /* Get rank of vspace */ - if ((rank = H5S_GET_EXTENT_NDIMS(layout->storage.u.virt.list[idx].source_dset.virtual_select)) < 0) + if ((rank = H5S_GET_EXTENT_NDIMS(ent->source_dset.virtual_select)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get number of dimensions") /* Get selection bounds */ - if (H5S_SELECT_BOUNDS(layout->storage.u.virt.list[idx].source_dset.virtual_select, bounds_start, - bounds_end) < 0) + if (H5S_SELECT_BOUNDS(ent->source_dset.virtual_select, bounds_start, bounds_end) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds") /* Update min_dims */ for (i = 0; i < rank; i++) /* Don't check unlimited dimensions in the selection */ - if ((i != layout->storage.u.virt.list[idx].unlim_dim_virtual) && - (bounds_end[i] >= layout->storage.u.virt.min_dims[i])) - layout->storage.u.virt.min_dims[i] = bounds_end[i] + (hsize_t)1; + if ((i != ent->unlim_dim_virtual) && (bounds_end[i] >= virt->min_dims[i])) + virt->min_dims[i] = bounds_end[i] + (hsize_t)1; done: FUNC_LEAVE_NOAPI(ret_value) @@ -410,31 +409,31 @@ done: herr_t H5D__virtual_store_layout(H5F_t *f, H5O_layout_t *layout) { - uint8_t *heap_block = NULL; /* Block to add to heap */ - size_t * str_size = NULL; /* Array for VDS entry string lengths */ - uint8_t *heap_block_p; /* Pointer into the heap block, while encoding */ - size_t block_size; /* Total size of block needed */ - hsize_t tmp_nentries; /* Temp. variable for # of VDS entries */ - uint32_t chksum; /* Checksum for heap data */ - size_t i; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ + H5O_storage_virtual_t *virt = &layout->storage.u.virt; + uint8_t * heap_block = NULL; /* Block to add to heap */ + size_t * str_size = NULL; /* Array for VDS entry string lengths */ + uint8_t * heap_block_p; /* Pointer into the heap block, while encoding */ + size_t block_size; /* Total size of block needed */ + hsize_t tmp_nentries; /* Temp. variable for # of VDS entries */ + uint32_t chksum; /* Checksum for heap data */ + size_t i; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE /* Sanity checking */ HDassert(f); HDassert(layout); - HDassert(layout->storage.u.virt.serial_list_hobjid.addr == HADDR_UNDEF); + HDassert(virt->serial_list_hobjid.addr == HADDR_UNDEF); /* Create block if # of used entries > 0 */ - if (layout->storage.u.virt.list_nused > 0) { + if (virt->list_nused > 0) { /* Set the low/high bounds according to 'f' for the API context */ H5CX_set_libver_bounds(f); /* Allocate array for caching results of strlen */ - if (NULL == - (str_size = (size_t *)H5MM_malloc(2 * layout->storage.u.virt.list_nused * sizeof(size_t)))) + if (NULL == (str_size = (size_t *)H5MM_malloc(2 * virt->list_nused * sizeof(size_t)))) HGOTO_ERROR(H5E_OHDR, H5E_RESOURCE, FAIL, "unable to allocate string length array") /* @@ -445,31 +444,30 @@ H5D__virtual_store_layout(H5F_t *f, H5O_layout_t *layout) block_size = (size_t)1 + H5F_SIZEOF_SIZE(f); /* Calculate size of each entry */ - for (i = 0; i < layout->storage.u.virt.list_nused; i++) { - hssize_t select_serial_size; /* Size of serialized selection */ + for (i = 0; i < virt->list_nused; i++) { + H5O_storage_virtual_ent_t *ent = &virt->list[i]; + hssize_t select_serial_size; /* Size of serialized selection */ - HDassert(layout->storage.u.virt.list[i].source_file_name); - HDassert(layout->storage.u.virt.list[i].source_dset_name); - HDassert(layout->storage.u.virt.list[i].source_select); - HDassert(layout->storage.u.virt.list[i].source_dset.virtual_select); + HDassert(ent->source_file_name); + HDassert(ent->source_dset_name); + HDassert(ent->source_select); + HDassert(ent->source_dset.virtual_select); /* Source file name */ - str_size[2 * i] = HDstrlen(layout->storage.u.virt.list[i].source_file_name) + (size_t)1; + str_size[2 * i] = HDstrlen(ent->source_file_name) + (size_t)1; block_size += str_size[2 * i]; /* Source dset name */ - str_size[(2 * i) + 1] = HDstrlen(layout->storage.u.virt.list[i].source_dset_name) + (size_t)1; + str_size[(2 * i) + 1] = HDstrlen(ent->source_dset_name) + (size_t)1; block_size += str_size[(2 * i) + 1]; /* Source selection */ - if ((select_serial_size = H5S_SELECT_SERIAL_SIZE(layout->storage.u.virt.list[i].source_select)) < - 0) + if ((select_serial_size = H5S_SELECT_SERIAL_SIZE(ent->source_select)) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTENCODE, FAIL, "unable to check dataspace selection size") block_size += (size_t)select_serial_size; /* Virtual dataset selection */ - if ((select_serial_size = - H5S_SELECT_SERIAL_SIZE(layout->storage.u.virt.list[i].source_dset.virtual_select)) < 0) + if ((select_serial_size = H5S_SELECT_SERIAL_SIZE(ent->source_dset.virtual_select)) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTENCODE, FAIL, "unable to check dataspace selection size") block_size += (size_t)select_serial_size; } /* end for */ @@ -490,28 +488,26 @@ H5D__virtual_store_layout(H5F_t *f, H5O_layout_t *layout) *heap_block_p++ = (uint8_t)H5O_LAYOUT_VDS_GH_ENC_VERS; /* Number of entries */ - tmp_nentries = (hsize_t)layout->storage.u.virt.list_nused; + tmp_nentries = (hsize_t)virt->list_nused; H5F_ENCODE_LENGTH(f, heap_block_p, tmp_nentries) /* Encode each entry */ - for (i = 0; i < layout->storage.u.virt.list_nused; i++) { + for (i = 0; i < virt->list_nused; i++) { + H5O_storage_virtual_ent_t *ent = &virt->list[i]; /* Source file name */ - H5MM_memcpy((char *)heap_block_p, layout->storage.u.virt.list[i].source_file_name, - str_size[2 * i]); + H5MM_memcpy((char *)heap_block_p, ent->source_file_name, str_size[2 * i]); heap_block_p += str_size[2 * i]; /* Source dataset name */ - H5MM_memcpy((char *)heap_block_p, layout->storage.u.virt.list[i].source_dset_name, - str_size[(2 * i) + 1]); + H5MM_memcpy((char *)heap_block_p, ent->source_dset_name, str_size[(2 * i) + 1]); heap_block_p += str_size[(2 * i) + 1]; /* Source selection */ - if (H5S_SELECT_SERIALIZE(layout->storage.u.virt.list[i].source_select, &heap_block_p) < 0) + if (H5S_SELECT_SERIALIZE(ent->source_select, &heap_block_p) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to serialize source selection") /* Virtual selection */ - if (H5S_SELECT_SERIALIZE(layout->storage.u.virt.list[i].source_dset.virtual_select, - &heap_block_p) < 0) + if (H5S_SELECT_SERIALIZE(ent->source_dset.virtual_select, &heap_block_p) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to serialize virtual selection") } /* end for */ @@ -520,7 +516,7 @@ H5D__virtual_store_layout(H5F_t *f, H5O_layout_t *layout) UINT32ENCODE(heap_block_p, chksum) /* Insert block into global heap */ - if (H5HG_insert(f, block_size, heap_block, &(layout->storage.u.virt.serial_list_hobjid)) < + if (H5HG_insert(f, block_size, heap_block, &(virt->serial_list_hobjid)) < 0) /* Casting away const OK --NAF */ HGOTO_ERROR(H5E_OHDR, H5E_CANTINSERT, FAIL, "unable to insert virtual dataset heap block") } /* end if */ @@ -551,6 +547,7 @@ herr_t H5D__virtual_copy_layout(H5O_layout_t *layout) { H5O_storage_virtual_ent_t *orig_list = NULL; + H5O_storage_virtual_t * virt = &layout->storage.u.virt; hid_t orig_source_fapl; hid_t orig_source_dapl; H5P_genplist_t * plist; @@ -564,133 +561,125 @@ H5D__virtual_copy_layout(H5O_layout_t *layout) /* Save original entry list and top-level property lists and reset in layout * so the originals aren't closed on error */ - orig_source_fapl = layout->storage.u.virt.source_fapl; - layout->storage.u.virt.source_fapl = -1; - orig_source_dapl = layout->storage.u.virt.source_dapl; - layout->storage.u.virt.source_dapl = -1; - orig_list = layout->storage.u.virt.list; - layout->storage.u.virt.list = NULL; + orig_source_fapl = virt->source_fapl; + virt->source_fapl = -1; + orig_source_dapl = virt->source_dapl; + virt->source_dapl = -1; + orig_list = virt->list; + virt->list = NULL; /* Copy entry list */ - if (layout->storage.u.virt.list_nused > 0) { + if (virt->list_nused > 0) { HDassert(orig_list); /* Allocate memory for the list */ - if (NULL == (layout->storage.u.virt.list = (H5O_storage_virtual_ent_t *)H5MM_calloc( - layout->storage.u.virt.list_nused * sizeof(H5O_storage_virtual_ent_t)))) + if (NULL == (virt->list = H5MM_calloc(virt->list_nused * sizeof(virt->list[0])))) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate memory for virtual dataset entry list") - layout->storage.u.virt.list_nalloc = layout->storage.u.virt.list_nused; + virt->list_nalloc = virt->list_nused; /* Copy the list entries, though set source_dset.dset and sub_dset to * NULL */ - for (i = 0; i < layout->storage.u.virt.list_nused; i++) { + for (i = 0; i < virt->list_nused; i++) { + H5O_storage_virtual_ent_t *ent = &virt->list[i]; + /* Copy virtual selection */ - if (NULL == (layout->storage.u.virt.list[i].source_dset.virtual_select = + if (NULL == (ent->source_dset.virtual_select = H5S_copy(orig_list[i].source_dset.virtual_select, FALSE, TRUE))) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy virtual selection") /* Copy original source names */ - if (NULL == (layout->storage.u.virt.list[i].source_file_name = - H5MM_strdup(orig_list[i].source_file_name))) + if (NULL == (ent->source_file_name = H5MM_strdup(orig_list[i].source_file_name))) HGOTO_ERROR(H5E_DATASET, H5E_RESOURCE, FAIL, "unable to duplicate source file name") - if (NULL == (layout->storage.u.virt.list[i].source_dset_name = - H5MM_strdup(orig_list[i].source_dset_name))) + if (NULL == (ent->source_dset_name = H5MM_strdup(orig_list[i].source_dset_name))) HGOTO_ERROR(H5E_DATASET, H5E_RESOURCE, FAIL, "unable to duplicate source dataset name") /* Copy source selection */ - if (NULL == (layout->storage.u.virt.list[i].source_select = - H5S_copy(orig_list[i].source_select, FALSE, TRUE))) + if (NULL == (ent->source_select = H5S_copy(orig_list[i].source_select, FALSE, TRUE))) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy source selection") /* Initialize clipped selections */ if (orig_list[i].unlim_dim_virtual < 0) { - layout->storage.u.virt.list[i].source_dset.clipped_source_select = - layout->storage.u.virt.list[i].source_select; - layout->storage.u.virt.list[i].source_dset.clipped_virtual_select = - layout->storage.u.virt.list[i].source_dset.virtual_select; + ent->source_dset.clipped_source_select = ent->source_select; + ent->source_dset.clipped_virtual_select = ent->source_dset.virtual_select; } /* end if */ /* Copy parsed names */ - if (H5D__virtual_copy_parsed_name(&layout->storage.u.virt.list[i].parsed_source_file_name, + if (H5D__virtual_copy_parsed_name(&ent->parsed_source_file_name, orig_list[i].parsed_source_file_name) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy parsed source file name") - layout->storage.u.virt.list[i].psfn_static_strlen = orig_list[i].psfn_static_strlen; - layout->storage.u.virt.list[i].psfn_nsubs = orig_list[i].psfn_nsubs; - if (H5D__virtual_copy_parsed_name(&layout->storage.u.virt.list[i].parsed_source_dset_name, + ent->psfn_static_strlen = orig_list[i].psfn_static_strlen; + ent->psfn_nsubs = orig_list[i].psfn_nsubs; + if (H5D__virtual_copy_parsed_name(&ent->parsed_source_dset_name, orig_list[i].parsed_source_dset_name) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy parsed source dataset name") - layout->storage.u.virt.list[i].psdn_static_strlen = orig_list[i].psdn_static_strlen; - layout->storage.u.virt.list[i].psdn_nsubs = orig_list[i].psdn_nsubs; + ent->psdn_static_strlen = orig_list[i].psdn_static_strlen; + ent->psdn_nsubs = orig_list[i].psdn_nsubs; /* Copy source names in source dset or add reference as appropriate */ if (orig_list[i].source_dset.file_name) { if (orig_list[i].source_dset.file_name == orig_list[i].source_file_name) - layout->storage.u.virt.list[i].source_dset.file_name = - layout->storage.u.virt.list[i].source_file_name; + ent->source_dset.file_name = ent->source_file_name; else if (orig_list[i].parsed_source_file_name && (orig_list[i].source_dset.file_name != orig_list[i].parsed_source_file_name->name_segment)) { - HDassert(layout->storage.u.virt.list[i].parsed_source_file_name); - HDassert(layout->storage.u.virt.list[i].parsed_source_file_name->name_segment); - layout->storage.u.virt.list[i].source_dset.file_name = - layout->storage.u.virt.list[i].parsed_source_file_name->name_segment; + HDassert(ent->parsed_source_file_name); + HDassert(ent->parsed_source_file_name->name_segment); + ent->source_dset.file_name = ent->parsed_source_file_name->name_segment; } /* end if */ - else if (NULL == (layout->storage.u.virt.list[i].source_dset.file_name = - H5MM_strdup(orig_list[i].source_dset.file_name))) + else if (NULL == + (ent->source_dset.file_name = H5MM_strdup(orig_list[i].source_dset.file_name))) HGOTO_ERROR(H5E_DATASET, H5E_RESOURCE, FAIL, "unable to duplicate source file name") } /* end if */ if (orig_list[i].source_dset.dset_name) { if (orig_list[i].source_dset.dset_name == orig_list[i].source_dset_name) - layout->storage.u.virt.list[i].source_dset.dset_name = - layout->storage.u.virt.list[i].source_dset_name; + ent->source_dset.dset_name = ent->source_dset_name; else if (orig_list[i].parsed_source_dset_name && (orig_list[i].source_dset.dset_name != orig_list[i].parsed_source_dset_name->name_segment)) { - HDassert(layout->storage.u.virt.list[i].parsed_source_dset_name); - HDassert(layout->storage.u.virt.list[i].parsed_source_dset_name->name_segment); - layout->storage.u.virt.list[i].source_dset.dset_name = - layout->storage.u.virt.list[i].parsed_source_dset_name->name_segment; + HDassert(ent->parsed_source_dset_name); + HDassert(ent->parsed_source_dset_name->name_segment); + ent->source_dset.dset_name = ent->parsed_source_dset_name->name_segment; } /* end if */ - else if (NULL == (layout->storage.u.virt.list[i].source_dset.dset_name = - H5MM_strdup(orig_list[i].source_dset.dset_name))) + else if (NULL == + (ent->source_dset.dset_name = H5MM_strdup(orig_list[i].source_dset.dset_name))) HGOTO_ERROR(H5E_DATASET, H5E_RESOURCE, FAIL, "unable to duplicate source dataset name") } /* end if */ /* Copy other fields in entry */ - layout->storage.u.virt.list[i].unlim_dim_source = orig_list[i].unlim_dim_source; - layout->storage.u.virt.list[i].unlim_dim_virtual = orig_list[i].unlim_dim_virtual; - layout->storage.u.virt.list[i].unlim_extent_source = orig_list[i].unlim_extent_source; - layout->storage.u.virt.list[i].unlim_extent_virtual = orig_list[i].unlim_extent_virtual; - layout->storage.u.virt.list[i].clip_size_source = orig_list[i].clip_size_source; - layout->storage.u.virt.list[i].clip_size_virtual = orig_list[i].clip_size_virtual; - layout->storage.u.virt.list[i].source_space_status = orig_list[i].source_space_status; - layout->storage.u.virt.list[i].virtual_space_status = orig_list[i].virtual_space_status; + ent->unlim_dim_source = orig_list[i].unlim_dim_source; + ent->unlim_dim_virtual = orig_list[i].unlim_dim_virtual; + ent->unlim_extent_source = orig_list[i].unlim_extent_source; + ent->unlim_extent_virtual = orig_list[i].unlim_extent_virtual; + ent->clip_size_source = orig_list[i].clip_size_source; + ent->clip_size_virtual = orig_list[i].clip_size_virtual; + ent->source_space_status = orig_list[i].source_space_status; + ent->virtual_space_status = orig_list[i].virtual_space_status; } /* end for */ } /* end if */ else { /* Zero out other fields related to list, just to be sure */ - layout->storage.u.virt.list = NULL; - layout->storage.u.virt.list_nalloc = 0; + virt->list = NULL; + virt->list_nalloc = 0; } /* end else */ /* Copy property lists */ if (orig_source_fapl >= 0) { if (NULL == (plist = (H5P_genplist_t *)H5I_object_verify(orig_source_fapl, H5I_GENPROP_LST))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") - if ((layout->storage.u.virt.source_fapl = H5P_copy_plist(plist, FALSE)) < 0) + if ((virt->source_fapl = H5P_copy_plist(plist, FALSE)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy fapl") } /* end if */ if (orig_source_dapl >= 0) { if (NULL == (plist = (H5P_genplist_t *)H5I_object_verify(orig_source_dapl, H5I_GENPROP_LST))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") - if ((layout->storage.u.virt.source_dapl = H5P_copy_plist(plist, FALSE)) < 0) + if ((virt->source_dapl = H5P_copy_plist(plist, FALSE)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy dapl") } /* end if */ /* New layout is not fully initialized */ - layout->storage.u.virt.init = FALSE; + virt->init = FALSE; done: /* Release allocated resources on failure */ @@ -720,8 +709,9 @@ done: herr_t H5D__virtual_reset_layout(H5O_layout_t *layout) { - size_t i, j; - herr_t ret_value = SUCCEED; + size_t i, j; + H5O_storage_virtual_t *virt = &layout->storage.u.virt; + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE @@ -731,56 +721,54 @@ H5D__virtual_reset_layout(H5O_layout_t *layout) /* Free the list entries. Note we always attempt to free everything even in * the case of a failure. Because of this, and because we free the list * afterwards, we do not need to zero out the memory in the list. */ - for (i = 0; i < layout->storage.u.virt.list_nused; i++) { + for (i = 0; i < virt->list_nused; i++) { + H5O_storage_virtual_ent_t *ent = &virt->list[i]; /* Free source_dset */ - if (H5D__virtual_reset_source_dset(&layout->storage.u.virt.list[i], - &layout->storage.u.virt.list[i].source_dset) < 0) + if (H5D__virtual_reset_source_dset(ent, &ent->source_dset) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to reset source dataset") /* Free original source names */ - (void)H5MM_xfree(layout->storage.u.virt.list[i].source_file_name); - (void)H5MM_xfree(layout->storage.u.virt.list[i].source_dset_name); + (void)H5MM_xfree(ent->source_file_name); + (void)H5MM_xfree(ent->source_dset_name); /* Free sub_dset */ - for (j = 0; j < layout->storage.u.virt.list[i].sub_dset_nalloc; j++) - if (H5D__virtual_reset_source_dset(&layout->storage.u.virt.list[i], - &layout->storage.u.virt.list[i].sub_dset[j]) < 0) + for (j = 0; j < ent->sub_dset_nalloc; j++) + if (H5D__virtual_reset_source_dset(ent, &ent->sub_dset[j]) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to reset source dataset") - layout->storage.u.virt.list[i].sub_dset = - (H5O_storage_virtual_srcdset_t *)H5MM_xfree(layout->storage.u.virt.list[i].sub_dset); + ent->sub_dset = H5MM_xfree(ent->sub_dset); /* Free source_select */ - if (layout->storage.u.virt.list[i].source_select) - if (H5S_close(layout->storage.u.virt.list[i].source_select) < 0) + if (ent->source_select) + if (H5S_close(ent->source_select) < 0) HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release source selection") /* Free parsed_source_file_name */ - H5D_virtual_free_parsed_name(layout->storage.u.virt.list[i].parsed_source_file_name); + H5D_virtual_free_parsed_name(ent->parsed_source_file_name); /* Free parsed_source_dset_name */ - H5D_virtual_free_parsed_name(layout->storage.u.virt.list[i].parsed_source_dset_name); - } /* end for */ + H5D_virtual_free_parsed_name(ent->parsed_source_dset_name); + } /* Free the list */ - layout->storage.u.virt.list = (H5O_storage_virtual_ent_t *)H5MM_xfree(layout->storage.u.virt.list); - layout->storage.u.virt.list_nalloc = (size_t)0; - layout->storage.u.virt.list_nused = (size_t)0; - (void)HDmemset(layout->storage.u.virt.min_dims, 0, sizeof(layout->storage.u.virt.min_dims)); + virt->list = H5MM_xfree(virt->list); + virt->list_nalloc = (size_t)0; + virt->list_nused = (size_t)0; + (void)HDmemset(virt->min_dims, 0, sizeof(virt->min_dims)); /* Close access property lists */ - if (layout->storage.u.virt.source_fapl >= 0) { - if (H5I_dec_ref(layout->storage.u.virt.source_fapl) < 0) + if (virt->source_fapl >= 0) { + if (H5I_dec_ref(virt->source_fapl) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "can't close source fapl") - layout->storage.u.virt.source_fapl = -1; - } /* end if */ - if (layout->storage.u.virt.source_dapl >= 0) { - if (H5I_dec_ref(layout->storage.u.virt.source_dapl) < 0) + virt->source_fapl = -1; + } + if (virt->source_dapl >= 0) { + if (H5I_dec_ref(virt->source_dapl) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "can't close source dapl") - layout->storage.u.virt.source_dapl = -1; - } /* end if */ + virt->source_dapl = -1; + } /* The list is no longer initialized */ - layout->storage.u.virt.init = FALSE; + virt->init = FALSE; /* Note the lack of a done: label. This is because there are no HGOTO_ERROR * calls. If one is added, a done: label must also be added */ @@ -2883,8 +2871,9 @@ H5D__virtual_write_one(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, * extent in the unlimited dimension. -NAF */ /* Project intersection of file space and mapping virtual space onto * mapping source space */ - if (H5S_select_project_intersection(source_dset->virtual_select, source_dset->clipped_source_select, - file_space, &projected_src_space, TRUE) < 0) + if (H5S_select_project_intersection(source_dset->clipped_virtual_select, + source_dset->clipped_source_select, file_space, + &projected_src_space, TRUE) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto source space") diff --git a/src/H5Eprivate.h b/src/H5Eprivate.h index 8f89201..f3d3b6f 100644 --- a/src/H5Eprivate.h +++ b/src/H5Eprivate.h @@ -114,12 +114,18 @@ typedef struct H5E_t H5E_t; #define HSYS_DONE_ERROR(majorcode, minorcode, retcode, str) \ { \ int myerrno = errno; \ + /* Other projects may rely on the description format to get the errno and any changes should be \ + * considered as an API change \ + */ \ HDONE_ERROR(majorcode, minorcode, retcode, "%s, errno = %d, error message = '%s'", str, myerrno, \ HDstrerror(myerrno)); \ } #define HSYS_GOTO_ERROR(majorcode, minorcode, retcode, str) \ { \ int myerrno = errno; \ + /* Other projects may rely on the description format to get the errno and any changes should be \ + * considered as an API change \ + */ \ HGOTO_ERROR(majorcode, minorcode, retcode, "%s, errno = %d, error message = '%s'", str, myerrno, \ HDstrerror(myerrno)); \ } @@ -1542,7 +1542,7 @@ done: } /* end H5FDtruncate() */ /*------------------------------------------------------------------------- - * Function: H5FD_truncate + * Function: H5FD_truncate * * Purpose: Private version of H5FDtruncate() * @@ -1594,7 +1594,7 @@ H5FDlock(H5FD_t *file, hbool_t rw) /* Call private function */ if (H5FD_lock(file, rw) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "file lock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "file lock request failed") done: FUNC_LEAVE_API(ret_value) @@ -1622,7 +1622,7 @@ H5FD_lock(H5FD_t *file, hbool_t rw) /* Dispatch to driver */ if (file->cls->lock && (file->cls->lock)(file, rw) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "driver lock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "driver lock request failed") done: FUNC_LEAVE_NOAPI(ret_value) @@ -1653,7 +1653,7 @@ H5FDunlock(H5FD_t *file) /* Call private function */ if (H5FD_unlock(file) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "file unlock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "file unlock request failed") done: FUNC_LEAVE_API(ret_value) @@ -1681,7 +1681,7 @@ H5FD_unlock(H5FD_t *file) /* Dispatch to driver */ if (file->cls->unlock && (file->cls->unlock)(file) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "driver unlock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "driver unlock request failed") done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDcore.c b/src/H5FDcore.c index 791deb2..61f46c7 100644 --- a/src/H5FDcore.c +++ b/src/H5FDcore.c @@ -36,6 +36,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_CORE_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* The skip list node type. Represents a region in the file. */ typedef struct H5FD_core_region_t { haddr_t start; /* Start address of the region */ @@ -56,7 +59,8 @@ typedef struct H5FD_core_t { hbool_t backing_store; /* write to file name on flush */ hbool_t write_tracking; /* Whether to track writes */ size_t bstore_page_size; /* backing store page size */ - int fd; /* backing store file descriptor */ + hbool_t ignore_disabled_file_locks; + int fd; /* backing store file descriptor */ /* Information for determining uniqueness of a file with a backing store */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely @@ -413,10 +417,20 @@ done: static herr_t H5FD__init_package(void) { - herr_t ret_value = SUCCEED; + char * lock_env_var = NULL; /* Environment variable pointer */ + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if (H5FD_core_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize core VFD") @@ -794,6 +808,16 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr /* Save file image callbacks */ file->fi_callbacks = file_image_info.callbacks; + /* Check the file locking flags in the fapl */ + if (ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if (H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + if (fd >= 0) { /* Retrieve information for determining uniqueness of file */ #ifdef H5_HAVE_WIN32_API @@ -1633,10 +1657,12 @@ H5FD__core_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if (HDflock(file->fd, lock_flags | LOCK_NB) < 0) { - if (ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)") + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") } /* end if */ @@ -1669,10 +1695,12 @@ H5FD__core_unlock(H5FD_t *_file) if (file->fd >= 0) if (HDflock(file->fd, LOCK_UN) < 0) { - if (ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)") + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") } diff --git a/src/H5FDdirect.c b/src/H5FDdirect.c index 9637946..4c2d251 100644 --- a/src/H5FDdirect.c +++ b/src/H5FDdirect.c @@ -37,6 +37,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_DIRECT_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* File operations */ #define OP_UNKNOWN 0 #define OP_READ 1 @@ -70,6 +73,7 @@ typedef struct H5FD_direct_t { haddr_t pos; /*current file I/O position */ int op; /*last operation */ H5FD_direct_fapl_t fa; /*file access properties */ + hbool_t ignore_disabled_file_locks; #ifndef H5_HAVE_WIN32_API /* * On most systems the combination of device and i-node number uniquely @@ -187,10 +191,20 @@ DESCRIPTION static herr_t H5FD__init_package(void) { - herr_t ret_value = SUCCEED; + char * lock_env_var = NULL; /* Environment variable pointer */ + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if (H5FD_direct_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize direct VFD") @@ -370,14 +384,13 @@ static void * H5FD__direct_fapl_get(H5FD_t *_file) { H5FD_direct_t *file = (H5FD_direct_t *)_file; - void * ret_value; /* Return value */ + void *ret_value = NULL; /* Return value */ - FUNC_ENTER_STATIC + FUNC_ENTER_STATIC_NOERR /* Set return value */ ret_value = H5FD__direct_fapl_copy(&(file->fa)); -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD__direct_fapl_get() */ @@ -433,7 +446,7 @@ H5FD__direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxad int o_flags; int fd = (-1); H5FD_direct_t * file = NULL; - H5FD_direct_fapl_t *fa; + const H5FD_direct_fapl_t *fa; #ifdef H5_HAVE_WIN32_API HFILE filehandle; struct _BY_HANDLE_FILE_INFORMATION fileinfo; @@ -482,7 +495,7 @@ H5FD__direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxad /* Get the driver specific information */ if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list") - if (NULL == (fa = (H5FD_direct_fapl_t *)H5P_peek_driver_info(plist))) + if (NULL == (fa = (const H5FD_direct_fapl_t *)H5P_peek_driver_info(plist))) HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "bad VFL driver info") file->fd = fd; @@ -502,6 +515,16 @@ H5FD__direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxad file->fa.fbsize = fa->fbsize; file->fa.cbsize = fa->cbsize; + /* Check the file locking flags in the fapl */ + if (ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if (H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + /* Try to decide if data alignment is required. The reason to check it here * is to handle correctly the case that the file is in a different file system * than the one where the program is running. @@ -522,7 +545,8 @@ H5FD__direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxad } else { file->fa.must_align = FALSE; - HDftruncate(file->fd, (HDoff_t)0); + if (-1 == HDftruncate(file->fd, (HDoff_t)0)) + HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, NULL, "unable to truncate file") } } else { @@ -769,7 +793,7 @@ H5FD__direct_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) { const H5FD_direct_t *file = (const H5FD_direct_t *)_file; - FUNC_ENTER_STATIC + FUNC_ENTER_STATIC_NOERR FUNC_LEAVE_NOAPI(file->eof) } @@ -1293,17 +1317,28 @@ done: static herr_t H5FD__direct_lock(H5FD_t *_file, hbool_t rw) { - H5FD_direct_t *file = (H5FD_direct_t *)_file; /* VFD file struct */ - const int lock = rw ? LOCK_EX : LOCK_SH; - herr_t ret_value = SUCCEED; /* Return value */ + H5FD_direct_t *file = (H5FD_direct_t *)_file; /* VFD file struct */ + int lock_flags; /* file locking flags */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC HDassert(file); - /* Place the lock with non-blocking */ - if (HDflock(file->fd, lock | LOCK_NB) < 0) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to flock file") + /* Set exclusive or shared lock based on rw status */ + lock_flags = rw ? LOCK_EX : LOCK_SH; + + /* Place a non-blocking lock on the file */ + if (HDflock(file->fd, lock_flags | LOCK_NB) < 0) { + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } + else + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file") + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -1330,8 +1365,16 @@ H5FD__direct_unlock(H5FD_t *_file) HDassert(file); - if (HDflock(file->fd, LOCK_UN) < 0) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to flock (unlock) file") + if (HDflock(file->fd, LOCK_UN) < 0) { + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } + else + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDdirect.h b/src/H5FDdirect.h index 2d88a69..ea2b3f2 100644 --- a/src/H5FDdirect.h +++ b/src/H5FDdirect.h @@ -23,7 +23,7 @@ #ifdef H5_HAVE_DIRECT #define H5FD_DIRECT (H5FD_direct_init()) #else -#define H5FD_DIRECT (-1) +#define H5FD_DIRECT (H5I_INVALID_HID) #endif /* H5_HAVE_DIRECT */ #ifdef H5_HAVE_DIRECT diff --git a/src/H5FDfamily.c b/src/H5FDfamily.c index cf89f26..a0639a7 100644 --- a/src/H5FDfamily.c +++ b/src/H5FDfamily.c @@ -1305,9 +1305,9 @@ H5FD__family_lock(H5FD_t *_file, hbool_t rw) for (v = 0; v < u; v++) { if (H5FD_unlock(file->memb[v]) < 0) /* Push error, but keep going */ - HDONE_ERROR(H5E_IO, H5E_CANTUNLOCK, FAIL, "unable to unlock member files") + HDONE_ERROR(H5E_IO, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock member files") } /* end for */ - HGOTO_ERROR(H5E_IO, H5E_CANTLOCK, FAIL, "unable to lock member files") + HGOTO_ERROR(H5E_IO, H5E_CANTLOCKFILE, FAIL, "unable to lock member files") } /* end if */ done: @@ -1338,7 +1338,7 @@ H5FD__family_unlock(H5FD_t *_file) for (u = 0; u < file->nmembs; u++) if (file->memb[u]) if (H5FD_unlock(file->memb[u]) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTUNLOCK, FAIL, "unable to unlock member files") + HGOTO_ERROR(H5E_IO, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock member files") done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDhdfs.c b/src/H5FDhdfs.c index c3bfd12..286a8e6 100644 --- a/src/H5FDhdfs.c +++ b/src/H5FDhdfs.c @@ -373,7 +373,7 @@ done: hid_t H5FD_hdfs_init(void) { - hid_t ret_value = H5I_INVALID_HID; /* Return value */ + hid_t ret_value = H5I_INVALID_HID; #if HDFS_STATS unsigned int bin_i; #endif @@ -690,7 +690,7 @@ H5Pget_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa_out) HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info") /* Copy the hdfs fapl data out */ - HDmemcpy(fa_out, fa, sizeof(H5FD_hdfs_fapl_t)); + H5MM_memcpy(fa_out, fa, sizeof(H5FD_hdfs_fapl_t)); done: FUNC_LEAVE_API(ret_value) @@ -725,7 +725,7 @@ H5FD__hdfs_fapl_get(H5FD_t *_file) HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "memory allocation failed") /* Copy the fields of the structure */ - HDmemcpy(fa, &(file->fa), sizeof(H5FD_hdfs_fapl_t)); + H5MM_memcpy(fa, &(file->fa), sizeof(H5FD_hdfs_fapl_t)); ret_value = fa; @@ -763,7 +763,7 @@ H5FD__hdfs_fapl_copy(const void *_old_fa) if (new_fa == NULL) HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "memory allocation failed") - HDmemcpy(new_fa, old_fa, sizeof(H5FD_hdfs_fapl_t)); + H5MM_memcpy(new_fa, old_fa, sizeof(H5FD_hdfs_fapl_t)); ret_value = new_fa; done: @@ -920,7 +920,7 @@ H5FD__hdfs_open(const char *path, unsigned flags, hid_t fapl_id, haddr_t maxaddr if (file == NULL) HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate file struct") file->hdfs_handle = handle; - HDmemcpy(&(file->fa), &fa, sizeof(H5FD_hdfs_fapl_t)); + H5MM_memcpy(&(file->fa), &fa, sizeof(H5FD_hdfs_fapl_t)); #if HDFS_STATS if (FAIL == hdfs__reset_stats(file)) diff --git a/src/H5FDhdfs.h b/src/H5FDhdfs.h index 5fbc54b..9e46954 100644 --- a/src/H5FDhdfs.h +++ b/src/H5FDhdfs.h @@ -25,7 +25,7 @@ #ifdef H5_HAVE_LIBHDFS #define H5FD_HDFS (H5FD_hdfs_init()) #else /* H5_HAVE_LIBHDFS */ -#define H5FD_HDFS (-1) +#define H5FD_HDFS (H5I_INVALID_HID) #endif /* H5_HAVE_LIBHDFS */ /**************************************************************************** diff --git a/src/H5FDlog.c b/src/H5FDlog.c index 929cd9c..fc2cb12 100644 --- a/src/H5FDlog.c +++ b/src/H5FDlog.c @@ -39,6 +39,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_LOG_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* Driver-specific file access properties */ typedef struct H5FD_log_fapl_t { char * logfile; /* Allocated log file name */ @@ -68,12 +71,13 @@ static const char *flavors[] = { * occurs), and `op' will be set to H5F_OP_UNKNOWN. */ typedef struct H5FD_log_t { - H5FD_t pub; /* public stuff, must be first */ - int fd; /* the unix file */ - haddr_t eoa; /* end of allocated region */ - haddr_t eof; /* end of file; current file size */ - haddr_t pos; /* current file I/O position */ - H5FD_file_op_t op; /* last operation */ + H5FD_t pub; /* public stuff, must be first */ + int fd; /* the unix file */ + haddr_t eoa; /* end of allocated region */ + haddr_t eof; /* end of file; current file size */ + haddr_t pos; /* current file I/O position */ + H5FD_file_op_t op; /* last operation */ + hbool_t ignore_disabled_file_locks; char filename[H5FD_MAX_FILENAME_LEN]; /* Copy of file name from open operation */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely @@ -224,10 +228,20 @@ H5FL_DEFINE_STATIC(H5FD_log_t); static herr_t H5FD__init_package(void) { - herr_t ret_value = SUCCEED; + char * lock_env_var = NULL; /* Environment variable pointer */ + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if (H5FD_log_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize log VFD") @@ -611,6 +625,16 @@ H5FD__log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) } /* end if */ } /* end if */ + /* Check the file locking flags in the fapl */ + if (ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if (H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + /* Check for non-default FAPL */ if (H5P_FILE_ACCESS_DEFAULT != fapl_id) { /* This step is for h5repart tool only. If user wants to change file driver from @@ -1699,13 +1723,15 @@ H5FD__log_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if (HDflock(file->fd, lock_flags | LOCK_NB) < 0) { - if (ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)") + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file") + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -1733,13 +1759,15 @@ H5FD__log_unlock(H5FD_t *_file) HDassert(file); if (HDflock(file->fd, LOCK_UN) < 0) { - if (ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)") + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDmirror.c b/src/H5FDmirror.c index 3895b42..7ea9609 100644 --- a/src/H5FDmirror.c +++ b/src/H5FDmirror.c @@ -242,7 +242,7 @@ H5FD_mirror_init(void) { hid_t ret_value = H5I_INVALID_HID; - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI(H5I_INVALID_HID) LOG_OP_CALL(FUNC); @@ -298,7 +298,7 @@ H5FD__mirror_xmit_decode_uint16(uint16_t *out, const unsigned char *_buf) HDassert(_buf && out); - HDmemcpy(&n, _buf, sizeof(n)); + H5MM_memcpy(&n, _buf, sizeof(n)); *out = (uint16_t)HDntohs(n); return 2; /* number of bytes eaten */ @@ -326,7 +326,7 @@ H5FD__mirror_xmit_decode_uint32(uint32_t *out, const unsigned char *_buf) HDassert(_buf && out); - HDmemcpy(&n, _buf, sizeof(n)); + H5MM_memcpy(&n, _buf, sizeof(n)); *out = (uint32_t)HDntohl(n); return 4; /* number of bytes eaten */ @@ -385,7 +385,7 @@ H5FD__mirror_xmit_decode_uint64(uint64_t *out, const unsigned char *_buf) HDassert(_buf && out); - HDmemcpy(&n, _buf, sizeof(n)); + H5MM_memcpy(&n, _buf, sizeof(n)); if (TRUE == is_host_little_endian()) *out = BSWAP_64(n); else @@ -412,7 +412,7 @@ H5FD__mirror_xmit_decode_uint8(uint8_t *out, const unsigned char *_buf) HDassert(_buf && out); - HDmemcpy(out, _buf, sizeof(uint8_t)); + H5MM_memcpy(out, _buf, sizeof(uint8_t)); return 1; /* number of bytes eaten */ } /* end H5FD__mirror_xmit_decode_uint8() */ @@ -439,7 +439,7 @@ H5FD__mirror_xmit_encode_uint16(unsigned char *_dest, uint16_t v) HDassert(_dest); n = (uint16_t)HDhtons(v); - HDmemcpy(_dest, &n, sizeof(n)); + H5MM_memcpy(_dest, &n, sizeof(n)); return 2; } /* end H5FD__mirror_xmit_encode_uint16() */ @@ -466,7 +466,7 @@ H5FD__mirror_xmit_encode_uint32(unsigned char *_dest, uint32_t v) HDassert(_dest); n = (uint32_t)HDhtonl(v); - HDmemcpy(_dest, &n, sizeof(n)); + H5MM_memcpy(_dest, &n, sizeof(n)); return 4; } /* end H5FD__mirror_xmit_encode_uint32() */ @@ -494,7 +494,7 @@ H5FD__mirror_xmit_encode_uint64(unsigned char *_dest, uint64_t v) if (TRUE == is_host_little_endian()) n = BSWAP_64(v); - HDmemcpy(_dest, &n, sizeof(n)); + H5MM_memcpy(_dest, &n, sizeof(n)); return 8; } /* H5FD__mirror_xmit_encode_uint64() */ @@ -519,7 +519,7 @@ H5FD__mirror_xmit_encode_uint8(unsigned char *dest, uint8_t v) HDassert(dest); - HDmemcpy(dest, &v, sizeof(v)); + H5MM_memcpy(dest, &v, sizeof(v)); return 1; } /* end H5FD__mirror_xmit_encode_uint8() */ @@ -1188,7 +1188,7 @@ H5FD__mirror_fapl_get(H5FD_t *_file) if (NULL == fa) HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "calloc failed"); - HDmemcpy(fa, &(file->fa), sizeof(H5FD_mirror_fapl_t)); + H5MM_memcpy(fa, &(file->fa), sizeof(H5FD_mirror_fapl_t)); ret_value = fa; @@ -1224,7 +1224,7 @@ H5FD__mirror_fapl_copy(const void *_old_fa) if (new_fa == NULL) HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "memory allocation failed"); - HDmemcpy(new_fa, old_fa, sizeof(H5FD_mirror_fapl_t)); + H5MM_memcpy(new_fa, old_fa, sizeof(H5FD_mirror_fapl_t)); ret_value = new_fa; done: @@ -1298,7 +1298,7 @@ H5Pget_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa_out) HDassert(fa->magic == H5FD_MIRROR_FAPL_MAGIC); /* sanity check */ - HDmemcpy(fa_out, fa, sizeof(H5FD_mirror_fapl_t)); + H5MM_memcpy(fa_out, fa, sizeof(H5FD_mirror_fapl_t)); done: FUNC_LEAVE_API(ret_value); diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index 6042776..c16e01e 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -1746,33 +1746,4 @@ H5FD__mpio_communicator(const H5FD_t *_file) FUNC_LEAVE_NOAPI(file->comm) } /* end H5FD__mpio_communicator() */ -/*------------------------------------------------------------------------- - * Function: H5FD__mpio_get_info - * - * Purpose: Returns the file info of MPIO file driver. - * - * Returns: Non-negative if succeed or negative if fails. - * - * Programmer: John Mainzer - * April 4, 2017 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5FD__mpio_get_info(H5FD_t *_file, void **mpi_info) -{ - H5FD_mpio_t *file = (H5FD_mpio_t *)_file; - herr_t ret_value = SUCCEED; - - FUNC_ENTER_STATIC - - if (!mpi_info) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "mpi info not valid") - - *mpi_info = &(file->info); - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD__mpio_get_info() */ - #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5FDmpio.h b/src/H5FDmpio.h index c9280c7..0773d9e 100644 --- a/src/H5FDmpio.h +++ b/src/H5FDmpio.h @@ -25,7 +25,7 @@ #ifdef H5_HAVE_PARALLEL #define H5FD_MPIO (H5FD_mpio_init()) #else -#define H5FD_MPIO (-1) +#define H5FD_MPIO (H5I_INVALID_HID) #endif /* H5_HAVE_PARALLEL */ #ifdef H5_HAVE_PARALLEL diff --git a/src/H5FDmulti.c b/src/H5FDmulti.c index 8f9257d..d60ddad 100644 --- a/src/H5FDmulti.c +++ b/src/H5FDmulti.c @@ -1860,7 +1860,7 @@ H5FD_multi_lock(H5FD_t *_file, hbool_t rw) } /* end if */ if (nerrors) - H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "error locking member files", -1) return 0; + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTLOCKFILE, "error locking member files", -1) return 0; } /* H5FD_multi_lock() */ @@ -1897,7 +1897,7 @@ H5FD_multi_unlock(H5FD_t *_file) END_MEMBERS; if (nerrors) - H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "error unlocking member files", -1) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTUNLOCKFILE, "error unlocking member files", -1) return 0; } /* H5FD_multi_unlock() */ diff --git a/src/H5FDros3.c b/src/H5FDros3.c index fa41b6f..a557f1d 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -320,7 +320,7 @@ H5FD_ros3_init(void) unsigned int bin_i; #endif - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI(H5I_INVALID_HID) #if ROS3_DEBUG HDfprintf(stdout, "H5FD_ros3_init() called.\n"); @@ -340,7 +340,6 @@ H5FD_ros3_init(void) } #endif - /* Set return value */ ret_value = H5FD_ROS3_g; done: @@ -501,7 +500,7 @@ H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_out) HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info") /* Copy the ros3 fapl data out */ - HDmemcpy(fa_out, fa, sizeof(H5FD_ros3_fapl_t)); + H5MM_memcpy(fa_out, fa, sizeof(H5FD_ros3_fapl_t)); done: FUNC_LEAVE_API(ret_value) @@ -536,7 +535,7 @@ H5FD__ros3_fapl_get(H5FD_t *_file) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") /* Copy the fields of the structure */ - HDmemcpy(fa, &(file->fa), sizeof(H5FD_ros3_fapl_t)); + H5MM_memcpy(fa, &(file->fa), sizeof(H5FD_ros3_fapl_t)); /* Set return value */ ret_value = fa; @@ -576,7 +575,7 @@ H5FD__ros3_fapl_copy(const void *_old_fa) if (new_fa == NULL) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed"); - HDmemcpy(new_fa, old_fa, sizeof(H5FD_ros3_fapl_t)); + H5MM_memcpy(new_fa, old_fa, sizeof(H5FD_ros3_fapl_t)); ret_value = new_fa; done: @@ -769,7 +768,7 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate file struct") file->s3r_handle = handle; - HDmemcpy(&(file->fa), &fa, sizeof(H5FD_ros3_fapl_t)); + H5MM_memcpy(&(file->fa), &fa, sizeof(H5FD_ros3_fapl_t)); #if ROS3_STATS if (FAIL == ros3_reset_stats(file)) diff --git a/src/H5FDs3comms.c b/src/H5FDs3comms.c index 2ac27d0..ac58f67 100644 --- a/src/H5FDs3comms.c +++ b/src/H5FDs3comms.c @@ -144,7 +144,7 @@ curlwritecallback(char *ptr, size_t size, size_t nmemb, void *userdata) return written; if (size > 0) { - HDmemcpy(&(sds->data[sds->size]), ptr, product); + H5MM_memcpy(&(sds->data[sds->size]), ptr, product); sds->size += product; written = product; } @@ -263,12 +263,12 @@ H5FD_s3comms_hrb_node_set(hrb_node_t **L, const char *name, const char *value) namecpy = (char *)H5MM_malloc(sizeof(char) * (namelen + 1)); if (namecpy == NULL) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for name copy."); - HDmemcpy(namecpy, name, (namelen + 1)); + H5MM_memcpy(namecpy, name, (namelen + 1)); valuecpy = (char *)H5MM_malloc(sizeof(char) * (valuelen + 1)); if (valuecpy == NULL) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for value copy."); - HDmemcpy(valuecpy, value, (valuelen + 1)); + H5MM_memcpy(valuecpy, value, (valuelen + 1)); nvcat = (char *)H5MM_malloc(sizeof(char) * catwrite); if (nvcat == NULL) @@ -701,14 +701,14 @@ H5FD_s3comms_hrb_init_request(const char *_verb, const char *_resource, const ch res = (char *)H5MM_malloc(sizeof(char) * (reslen + 1)); if (res == NULL) HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, "no space for resource string"); - HDmemcpy(res, _resource, (reslen + 1)); + H5MM_memcpy(res, _resource, (reslen + 1)); } else { res = (char *)H5MM_malloc(sizeof(char) * (reslen + 2)); if (res == NULL) HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, "no space for resource string"); *res = '/'; - HDmemcpy((&res[1]), _resource, (reslen + 1)); + H5MM_memcpy((&res[1]), _resource, (reslen + 1)); HDassert((reslen + 1) == HDstrlen(res)); } /* end if (else resource string not starting with '/') */ @@ -910,7 +910,7 @@ H5FD_s3comms_s3r_getsize(s3r_t *handle) handle->httpverb = (char *)H5MM_malloc(sizeof(char) * 16); if (handle->httpverb == NULL) HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, "unable to allocate space for S3 request HTTP verb"); - HDmemcpy(handle->httpverb, "HEAD", 5); + H5MM_memcpy(handle->httpverb, "HEAD", 5); headerresponse = (char *)H5MM_malloc(sizeof(char) * CURL_MAX_HTTP_HEADER); if (headerresponse == NULL) @@ -1078,19 +1078,19 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const handle->region = (char *)H5MM_malloc(sizeof(char) * tmplen); if (handle->region == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle region copy."); - HDmemcpy(handle->region, region, tmplen); + H5MM_memcpy(handle->region, region, tmplen); tmplen = HDstrlen(id) + 1; handle->secret_id = (char *)H5MM_malloc(sizeof(char) * tmplen); if (handle->secret_id == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle ID copy."); - HDmemcpy(handle->secret_id, id, tmplen); + H5MM_memcpy(handle->secret_id, id, tmplen); tmplen = SHA256_DIGEST_LENGTH; handle->signing_key = (unsigned char *)H5MM_malloc(sizeof(unsigned char) * tmplen); if (handle->signing_key == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle key copy."); - HDmemcpy(handle->signing_key, signing_key, tmplen); + H5MM_memcpy(handle->signing_key, signing_key, tmplen); } /* if authentication information provided */ /************************ @@ -1138,7 +1138,7 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const *********************/ HDassert(handle->httpverb != NULL); - HDmemcpy(handle->httpverb, "GET", 4); + H5MM_memcpy(handle->httpverb, "GET", 4); ret_value = handle; @@ -1951,7 +1951,7 @@ H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, cha for (buffer_i = 0; buffer_i < 128; buffer_i++) buffer[buffer_i] = 0; - line_buffer = fgets(line_buffer, 128, file); + line_buffer = HDfgets(line_buffer, 128, file); if (line_buffer == NULL) /* reached end of file */ goto done; } while (HDstrncmp(line_buffer, profile_line, HDstrlen(profile_line))); @@ -1963,7 +1963,7 @@ H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, cha buffer[buffer_i] = 0; /* collect a line from file */ - line_buffer = fgets(line_buffer, 128, file); + line_buffer = HDfgets(line_buffer, 128, file); if (line_buffer == NULL) goto done; /* end of file */ @@ -2062,9 +2062,9 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char * #endif #ifdef H5_HAVE_WIN32_API - ret = HDsnprintf(awspath, 117, "%s/.aws/", getenv("USERPROFILE")); + ret = HDsnprintf(awspath, 117, "%s/.aws/", HDgetenv("USERPROFILE")); #else - ret = HDsnprintf(awspath, 117, "%s/.aws/", getenv("HOME")); + ret = HDsnprintf(awspath, 117, "%s/.aws/", HDgetenv("HOME")); #endif if (ret < 0 || (size_t)ret >= 117) HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, "unable to format home-aws path") @@ -2150,7 +2150,7 @@ H5FD_s3comms_nlowercase(char *dest, const char *s, size_t len) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "destination cannot be null."); if (len > 0) { - HDmemcpy(dest, s, len); + H5MM_memcpy(dest, s, len); do { len--; dest[len] = (char)HDtolower((int)dest[len]); @@ -2686,14 +2686,14 @@ H5FD_s3comms_tostringtosign(char *dest, const char *req, const char *now, const if (ret <= 0 || ret >= 127) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem adding day and region to string") - HDmemcpy((dest + d), "AWS4-HMAC-SHA256\n", 17); + H5MM_memcpy((dest + d), "AWS4-HMAC-SHA256\n", 17); d = 17; - HDmemcpy((dest + d), now, HDstrlen(now)); + H5MM_memcpy((dest + d), now, HDstrlen(now)); d += HDstrlen(now); dest[d++] = '\n'; - HDmemcpy((dest + d), tmp, HDstrlen(tmp)); + H5MM_memcpy((dest + d), tmp, HDstrlen(tmp)); d += HDstrlen(tmp); dest[d++] = '\n'; @@ -2777,7 +2777,7 @@ H5FD_s3comms_trim(char *dest, char *s, size_t s_len, size_t *n_written) s_len++; /* write output into dest */ - HDmemcpy(dest, s, s_len); + H5MM_memcpy(dest, s, s_len); } } diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c index 15de017..7789d39 100644 --- a/src/H5FDsec2.c +++ b/src/H5FDsec2.c @@ -38,6 +38,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_SEC2_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* The description of a file belonging to this driver. The 'eoa' and 'eof' * determine the amount of hdf5 address space in use and the high-water mark * of the file (the current size of the underlying filesystem file). The @@ -50,12 +53,13 @@ static hid_t H5FD_SEC2_g = 0; * occurs), and 'op' will be set to H5F_OP_UNKNOWN. */ typedef struct H5FD_sec2_t { - H5FD_t pub; /* public stuff, must be first */ - int fd; /* the filesystem file descriptor */ - haddr_t eoa; /* end of allocated region */ - haddr_t eof; /* end of file; current file size */ - haddr_t pos; /* current file I/O position */ - H5FD_file_op_t op; /* last operation */ + H5FD_t pub; /* public stuff, must be first */ + int fd; /* the filesystem file descriptor */ + haddr_t eoa; /* end of allocated region */ + haddr_t eof; /* end of file; current file size */ + haddr_t pos; /* current file I/O position */ + H5FD_file_op_t op; /* last operation */ + hbool_t ignore_disabled_file_locks; char filename[H5FD_MAX_FILENAME_LEN]; /* Copy of file name from open operation */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely @@ -185,10 +189,20 @@ H5FL_DEFINE_STATIC(H5FD_sec2_t); static herr_t H5FD__init_package(void) { - herr_t ret_value = SUCCEED; + char * lock_env_var = NULL; /* Environment variable pointer */ + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if (H5FD_sec2_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize sec2 VFD") @@ -306,8 +320,9 @@ H5FD__sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr #ifdef H5_HAVE_WIN32_API struct _BY_HANDLE_FILE_INFORMATION fileinfo; #endif - h5_stat_t sb; - H5FD_t * ret_value = NULL; /* Return value */ + h5_stat_t sb; + H5P_genplist_t *plist; /* Property list pointer */ + H5FD_t * ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC @@ -367,17 +382,26 @@ H5FD__sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr file->inode = sb.st_ino; #endif /* H5_HAVE_WIN32_API */ + /* Get the FAPL */ + if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, NULL, "not a file access property list") + + /* Check the file locking flags in the fapl */ + if (ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if (H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + /* Retain a copy of the name used to open the file, for possible error reporting */ HDstrncpy(file->filename, name, sizeof(file->filename)); file->filename[sizeof(file->filename) - 1] = '\0'; /* Check for non-default FAPL */ if (H5P_FILE_ACCESS_DEFAULT != fapl_id) { - H5P_genplist_t *plist; /* Property list pointer */ - - /* Get the FAPL */ - if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) - HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, NULL, "not a file access property list") /* This step is for h5repart tool only. If user wants to change file driver from * family to one that uses single files (sec2, etc.) while using h5repart, this @@ -968,13 +992,15 @@ H5FD__sec2_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if (HDflock(file->fd, lock_flags | LOCK_NB) < 0) { - if (ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)") + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file") + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -1002,13 +1028,15 @@ H5FD__sec2_unlock(H5FD_t *_file) HDassert(file); if (HDflock(file->fd, LOCK_UN) < 0) { - if (ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)") + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDsplitter.c b/src/H5FDsplitter.c index 6af5c74..6ecafe0 100644 --- a/src/H5FDsplitter.c +++ b/src/H5FDsplitter.c @@ -210,7 +210,7 @@ H5FD_splitter_init(void) { hid_t ret_value = H5I_INVALID_HID; - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI(H5I_INVALID_HID) H5FD_SPLITTER_LOG_CALL(FUNC); @@ -585,7 +585,7 @@ H5FD__splitter_fapl_copy(const void *_old_fa) if (NULL == new_fa_ptr) HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate log file FAPL") - HDmemcpy(new_fa_ptr, old_fa_ptr, sizeof(H5FD_splitter_fapl_t)); + H5MM_memcpy(new_fa_ptr, old_fa_ptr, sizeof(H5FD_splitter_fapl_t)); HDstrncpy(new_fa_ptr->wo_path, old_fa_ptr->wo_path, H5FD_SPLITTER_PATH_MAX); HDstrncpy(new_fa_ptr->log_file_path, old_fa_ptr->log_file_path, H5FD_SPLITTER_PATH_MAX); @@ -1096,10 +1096,11 @@ H5FD__splitter_lock(H5FD_t *_file, hbool_t rw) /* Place the lock on each file */ if (H5FD_lock(file->rw_file, rw) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTLOCK, FAIL, "unable to lock R/W file") + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock R/W file") + if (file->wo_file != NULL) if (H5FD_lock(file->wo_file, rw) < 0) - H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_CANTLOCK, FAIL, "unable to lock W/O file") + H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock W/O file") done: FUNC_LEAVE_NOAPI(ret_value) @@ -1129,10 +1130,11 @@ H5FD__splitter_unlock(H5FD_t *_file) /* Remove the lock on each file */ if (H5FD_unlock(file->rw_file) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCK, FAIL, "unable to unlock R/W file") + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock R/W file") + if (file->wo_file != NULL) if (H5FD_unlock(file->wo_file) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCK, FAIL, "unable to unlock W/O file") + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock W/O file") done: FUNC_LEAVE_NOAPI(ret_value) @@ -1298,14 +1300,14 @@ H5FD__splitter_log_error(const H5FD_splitter_t *file, const char *atfunc, const char * s; size = HDstrlen(atfunc) + HDstrlen(msg) + 3; /* ':', ' ', '\n' */ - s = (char *)HDmalloc(sizeof(char) * (size + 1)); + s = (char *)H5MM_malloc(sizeof(char) * (size + 1)); if (NULL == s) ret_value = FAIL; else if (size < (size_t)HDsnprintf(s, size + 1, "%s: %s\n", atfunc, msg)) ret_value = FAIL; else if (size != HDfwrite(s, 1, size, file->logfp)) ret_value = FAIL; - HDfree(s); + H5MM_free(s); } FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDstdio.c b/src/H5FDstdio.c index 45ea503..4650c39 100644 --- a/src/H5FDstdio.c +++ b/src/H5FDstdio.c @@ -52,6 +52,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_STDIO_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = -1; + /* The maximum number of bytes which can be written in a single I/O operation */ static size_t H5_STDIO_MAX_IO_BYTES_g = (size_t)-1; @@ -82,7 +85,8 @@ typedef struct H5FD_stdio_t { haddr_t eof; /* end of file; current file size */ haddr_t pos; /* current file I/O position */ unsigned write_access; /* Flag to indicate the file was opened with write access */ - H5FD_stdio_file_op op; /* last operation */ + hbool_t ignore_disabled_file_locks; + H5FD_stdio_file_op op; /* last operation */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely * identify a file. Note that Cygwin, MinGW and other Windows POSIX @@ -230,9 +234,20 @@ static const H5FD_class_t H5FD_stdio_g = { hid_t H5FD_stdio_init(void) { + char *lock_env_var = NULL; /* Environment variable pointer */ + /* Clear the error stack */ H5Eclear2(H5E_DEFAULT); + /* Check the use disabled file locks environment variable */ + lock_env_var = getenv("HDF5_USE_FILE_LOCKING"); + if (lock_env_var && !strcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = 1; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!strcmp(lock_env_var, "TRUE") || !strcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = 0; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = -1; /* Environment variable not set, or not set correctly */ + if (H5I_VFL != H5Iget_type(H5FD_STDIO_g)) H5FD_STDIO_g = H5FDregister(&H5FD_stdio_g); @@ -397,6 +412,22 @@ H5FD_stdio_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr file->eof = (haddr_t)x; } + /* Check the file locking flags in the fapl */ + if (ignore_disabled_file_locks_s != -1) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + hbool_t unused; + + /* Use the value in the property list */ + if (H5Pget_file_locking(fapl_id, &unused, &file->ignore_disabled_file_locks) < 0) { + free(file); + fclose(f); + H5Epush_ret(func, H5E_ERR_CLS, H5E_FILE, H5E_CANTGET, + "unable to get use disabled file locks property", NULL); + } + } + /* Get the file descriptor (needed for truncate and some Windows information) */ #ifdef H5_HAVE_WIN32_API file->fd = _fileno(file->fp); @@ -1104,11 +1135,13 @@ H5FD_stdio_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if (flock(file->fd, lock_flags | LOCK_NB) < 0) { - if (ENOSYS == errno) - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment " - "variable to override)", - -1) else H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file lock failed", -1) + if (file->ignore_disabled_file_locks && ENOSYS == errno) + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + else + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTLOCKFILE, "file lock failed", -1) } /* end if */ /* Flush the stream */ @@ -1154,12 +1187,13 @@ H5FD_stdio_unlock(H5FD_t *_file) /* Place a non-blocking lock on the file */ if (flock(file->fd, LOCK_UN) < 0) { - if (ENOSYS == errno) - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)", - -1) else H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file unlock failed", - -1) + if (file->ignore_disabled_file_locks && ENOSYS == errno) + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + else + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTUNLOCKFILE, "file unlock failed", -1) } /* end if */ #endif /* H5_HAVE_FLOCK */ diff --git a/src/H5Fint.c b/src/H5Fint.c index d364a04..1d7a86f 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -80,6 +80,7 @@ static int H5F__get_objects_cb(void *obj_ptr, hid_t obj_id, void *key); static herr_t H5F__build_name(const char *prefix, const char *file_name, char **full_name /*out*/); static char * H5F__getenv_prefix_name(char **env_prefix /*in,out*/); static H5F_t *H5F__new(H5F_shared_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t *lf); +static herr_t H5F__check_if_using_file_locks(H5P_genplist_t *fapl, hbool_t *use_file_locking); static herr_t H5F__build_actual_name(const H5F_t *f, const H5P_genplist_t *fapl, const char *name, char ** /*out*/ actual_name); static herr_t H5F__flush_phase1(H5F_t *f); @@ -92,6 +93,12 @@ static herr_t H5F__flush_phase2(H5F_t *f, hbool_t closing); /* Package initialization variable */ hbool_t H5_PKG_INIT_VAR = FALSE; +/* Based on the value of the HDF5_USE_FILE_LOCKING environment variable. + * TRUE/FALSE have obvious meanings. FAIL means the environment variable was + * not set, so the code should ignore it and use the fapl value instead. + */ +htri_t use_locks_env_g = FAIL; + /*****************************/ /* Library Private Variables */ /*****************************/ @@ -160,6 +167,10 @@ H5F__init_package(void) if (H5I_register_type(H5I_FILE_CLS) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to initialize interface") + /* Check the file locking environment variable */ + if (H5F__parse_file_lock_env_var(&use_locks_env_g) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to parse file locking environment variable") + done: FUNC_LEAVE_NOAPI(ret_value) } /* H5F__init_package() */ @@ -247,6 +258,38 @@ done: } /* end H5F__close_cb() */ /*------------------------------------------------------------------------- + * Function: H5F__parse_file_lock_env_var + * + * Purpose: Parses the HDF5_USE_FILE_LOCKING environment variable. + * + * NOTE: This is done in a separate function so we can call it from + * the test code. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5F__parse_file_lock_env_var(htri_t *use_locks) +{ + char *lock_env_var = NULL; /* Environment variable pointer */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Check the file locking environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if (lock_env_var && (!HDstrcmp(lock_env_var, "FALSE") || !HDstrcmp(lock_env_var, "0"))) + *use_locks = FALSE; /* Override: Never use locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "BEST_EFFORT") || + !HDstrcmp(lock_env_var, "1"))) + *use_locks = TRUE; /* Override: Always use locks */ + else + *use_locks = FAIL; /* Environment variable not set, or not set correctly */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5F__parse_file_lock_env_var() */ + +/*------------------------------------------------------------------------- * Function: H5F__set_vol_conn * * Purpose: Set the VOL connector ID and info for a file. @@ -1573,6 +1616,51 @@ H5F__dest(H5F_t *f, hbool_t flush) } /* end H5F__dest() */ /*------------------------------------------------------------------------- + * Function: H5F__check_if_using_file_locks + * + * Purpose: Determines if this file will use file locks. + * + * There are three ways that file locking can be controlled: + * + * 1) The configure/cmake option that sets the H5_USE_FILE_LOCKING + * symbol (which is used as the default fapl value). + * + * 2) The H5Pset_file_locking() API call, which will override + * the configuration default. + * + * 3) The HDF5_USE_FILE_LOCKING environment variable, which overrides + * everything above. + * + * The main reason to disable file locking is to prevent errors on file + * systems where locking is not supported or has been disabled (as is + * often the case in parallel file systems). + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5F__check_if_using_file_locks(H5P_genplist_t *fapl, hbool_t *use_file_locking) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Make sure the out parameter has a value */ + *use_file_locking = TRUE; + + /* Check the fapl property */ + if (H5P_get(fapl, H5F_ACS_USE_FILE_LOCKING_NAME, use_file_locking) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get use file locking flag") + + /* Check the environment variable */ + if (use_locks_env_g != FAIL) + *use_file_locking = (use_locks_env_g == TRUE) ? TRUE : FALSE; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__check_if_using_file_locks() */ + +/*------------------------------------------------------------------------- * Function: H5F_open * * Purpose: Opens (or creates) a file. This function understands the @@ -1661,11 +1749,10 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) hbool_t set_flag = FALSE; /*set the status_flags in the superblock */ hbool_t clear = FALSE; /*clear the status_flags */ hbool_t evict_on_close; /* evict on close value from plist */ - char * lock_env_var = NULL; /*env var pointer */ - hbool_t use_file_locking; /*read from env var */ - hbool_t ci_load = FALSE; /* whether MDC ci load requested */ - hbool_t ci_write = FALSE; /* whether MDC CI write requested */ - H5F_t * ret_value = NULL; /*actual return value */ + hbool_t use_file_locking = TRUE; /* Using file locks? */ + hbool_t ci_load = FALSE; /* whether MDC ci load requested */ + hbool_t ci_write = FALSE; /* whether MDC CI write requested */ + H5F_t * ret_value = NULL; /*actual return value */ FUNC_ENTER_NOAPI(NULL) @@ -1680,15 +1767,13 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) if (NULL == (drvr = H5FD_get_class(fapl_id))) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to retrieve VFL class") - /* Check the environment variable that determines if we care - * about file locking. File locking should be used unless explicitly - * disabled. - */ - lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); - if (lock_env_var && !HDstrcmp(lock_env_var, "FALSE")) - use_file_locking = FALSE; - else - use_file_locking = TRUE; + /* Get the file access property list, for future queries */ + if (NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list") + + /* Check if we are using file locking */ + if (H5F__check_if_using_file_locks(a_plist, &use_file_locking) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to get file locking flag") /* * Opening a file is a two step process. First we try to open the @@ -1771,8 +1856,8 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) if (H5FD_lock(lf, (hbool_t)((flags & H5F_ACC_RDWR) ? TRUE : FALSE)) < 0) { /* Locking failed - Closing will remove the lock */ if (H5FD_close(lf) < 0) - HDONE_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to close low-level file info") - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to lock the file") + HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, NULL, "unable to close low-level file info") + HGOTO_ERROR(H5E_FILE, H5E_CANTLOCKFILE, NULL, "unable to lock the file") } /* end if */ /* Create the 'top' file structure */ @@ -1804,9 +1889,14 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) shared = file->shared; lf = shared->lf; - /* Get the file access property list, for future queries */ - if (NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list") + /* Set the file locking flag. If the file is already open, the file + * requested file locking flag must match that of the open file. + */ + if (shared->nrefs == 1) + file->shared->use_file_locking = use_file_locking; + else if (shared->nrefs > 1) + if (file->shared->use_file_locking != use_file_locking) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "file locking flag values don't match") /* Check if page buffering is enabled */ if (H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &page_buf_size) < 0) @@ -1952,7 +2042,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) /* Remove the file lock for SWMR_WRITE */ if (use_file_locking && (H5F_INTENT(file) & H5F_ACC_SWMR_WRITE)) { if (H5FD_unlock(file->shared->lf) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to unlock the file") + HGOTO_ERROR(H5E_FILE, H5E_CANTUNLOCKFILE, NULL, "unable to unlock the file") } /* end if */ } /* end if */ else { /* H5F_ACC_RDONLY: check consistency of status_flags */ @@ -2070,6 +2160,11 @@ H5F__flush_phase2(H5F_t *f, hbool_t closing) /* Sanity check arguments */ HDassert(f); + /* Inform the metadata cache that we are about to flush */ + if (H5AC_prep_for_file_flush(f) < 0) + /* Push error, but keep going*/ + HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "prep for MDC flush failed") + /* Flush the entire metadata cache */ if (H5AC_flush(f) < 0) /* Push error, but keep going*/ @@ -2102,6 +2197,11 @@ H5F__flush_phase2(H5F_t *f, hbool_t closing) H5CX_set_mpi_file_flushing(FALSE); #endif /* H5_HAVE_PARALLEL */ + /* Inform the metadata cache that we are done with the flush */ + if (H5AC_secure_from_file_flush(f) < 0) + /* Push error, but keep going*/ + HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "secure from MDC flush failed") + /* Flush out the metadata accumulator */ if (H5F__accum_flush(f->shared) < 0) /* Push error, but keep going*/ diff --git a/src/H5Fmodule.h b/src/H5Fmodule.h index 953b4da..d57926f 100644 --- a/src/H5Fmodule.h +++ b/src/H5Fmodule.h @@ -29,4 +29,17 @@ #define H5_MY_PKG_ERR H5E_FILE #define H5_MY_PKG_INIT YES +/** + * \defgroup H5F H5F + * \brief File Interface + * \todo Describe concisely what the functions in this module are about. + * + * \defgroup MDC Metadata Cache + * \ingroup H5F + * \defgroup PH5F Parallel + * \ingroup H5F + * \defgroup SWMR Single Writer Multiple Readers + * \ingroup H5F + */ + #endif /* _H5Fmodule_H */ diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 9f4eae6..94096af 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -304,6 +304,7 @@ struct H5F_shared_t { struct H5G_t * root_grp; /* Open root group */ H5FO_t * open_objs; /* Open objects in file */ H5UC_t * grp_btree_shared; /* Ref-counted group B-tree node info */ + hbool_t use_file_locking; /* Whether or not to use file locking */ hbool_t closing; /* File is in the process of being closed */ /* Cached VOL connector ID & info */ @@ -392,6 +393,11 @@ H5FL_EXTERN(H5F_t); /* Declare a free list to manage the H5F_shared_t struct */ H5FL_EXTERN(H5F_shared_t); +/* Whether or not to use file locking (based on the environment variable) + * FAIL means ignore the environment variable. + */ +H5_DLLVAR htri_t use_locks_env_g; + /******************************/ /* Package Private Prototypes */ /******************************/ @@ -409,6 +415,7 @@ H5_DLL herr_t H5F__start_swmr_write(H5F_t *f); H5_DLL herr_t H5F__close(H5F_t *f); H5_DLL herr_t H5F__set_libver_bounds(H5F_t *f, H5F_libver_t low, H5F_libver_t high); H5_DLL herr_t H5F__get_cont_info(const H5F_t *f, H5VL_file_cont_info_t *info); +H5_DLL herr_t H5F__parse_file_lock_env_var(htri_t *use_locks); /* File mount related routines */ H5_DLL herr_t H5F__mount(H5G_loc_t *loc, const char *name, H5F_t *child, hid_t plist_id); @@ -471,6 +478,7 @@ H5_DLL herr_t H5F__check_cached_stab_test(hid_t file_id); H5_DLL herr_t H5F__get_maxaddr_test(hid_t file_id, haddr_t *maxaddr); H5_DLL herr_t H5F__get_sbe_addr_test(hid_t file_id, haddr_t *sbe_addr); H5_DLL htri_t H5F__same_file_test(hid_t file_id1, hid_t file_id2); +H5_DLL herr_t H5F__reparse_file_lock_variable_test(void); #endif /* H5F_TESTING */ #endif /* _H5Fpkg_H */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 9b93469..ba6d67f 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -381,6 +381,7 @@ typedef struct H5F_t H5F_t; #define H5F_SET_MIN_DSET_OHDR(F, V) ((F)->shared->crt_dset_min_ohdr_flag = (V)) #define H5F_VOL_CLS(F) ((F)->shared->vol_cls) #define H5F_VOL_OBJ(F) ((F)->vol_obj) +#define H5F_USE_FILE_LOCKING(F) ((F)->shared->use_file_locking) #else /* H5F_MODULE */ #define H5F_LOW_BOUND(F) (H5F_get_low_bound(F)) #define H5F_HIGH_BOUND(F) (H5F_get_high_bound(F)) @@ -443,6 +444,7 @@ typedef struct H5F_t H5F_t; #define H5F_SET_MIN_DSET_OHDR(F, V) (H5F_set_min_dset_ohdr((F), (V))) #define H5F_VOL_CLS(F) (H5F_get_vol_cls(F)) #define H5F_VOL_OBJ(F) (H5F_get_vol_obj(F)) +#define H5F_USE_FILE_LOCKING(F) (H5F_get_use_file_locking(F)) #endif /* H5F_MODULE */ /* Macros to encode/decode offset/length's for storing in the file */ @@ -608,6 +610,11 @@ typedef struct H5F_t H5F_t; "page_buffer_min_meta_perc" /* the min metadata percentage for the page buffer cache */ #define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME \ "page_buffer_min_raw_perc" /* the min raw data percentage for the page buffer cache */ +#define H5F_ACS_USE_FILE_LOCKING_NAME \ + "use_file_locking" /* whether or not we use file locks for SWMR control and to prevent multiple writers \ + */ +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME \ + "ignore_disabled_file_locks" /* whether or not we ignore "locks disabled" errors */ #ifdef H5_HAVE_PARALLEL #define H5F_ACS_MPI_PARAMS_COMM_NAME "mpi_params_comm" /* the MPI communicator */ #define H5F_ACS_MPI_PARAMS_INFO_NAME "mpi_params_info" /* the MPI info struct */ @@ -853,6 +860,7 @@ H5_DLL hbool_t H5F_get_min_dset_ohdr(const H5F_t *f); H5_DLL herr_t H5F_set_min_dset_ohdr(H5F_t *f, hbool_t minimize); H5_DLL const H5VL_class_t *H5F_get_vol_cls(const H5F_t *f); H5_DLL H5VL_object_t *H5F_get_vol_obj(const H5F_t *f); +H5_DLL hbool_t H5F_get_file_locking(const H5F_t *f); /* Functions than retrieve values set/cached from the superblock/FCPL */ H5_DLL haddr_t H5F_get_base_addr(const H5F_t *f); diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index c6a5b76..fbee9c5 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -47,24 +47,20 @@ * We're assuming that these constants are used rather early in the hdf5 * session. */ -#define H5F_ACC_RDONLY (H5CHECK H5OPEN 0x0000u) /*absence of rdwr => rd-only */ -#define H5F_ACC_RDWR (H5CHECK H5OPEN 0x0001u) /*open for read and write */ -#define H5F_ACC_TRUNC (H5CHECK H5OPEN 0x0002u) /*overwrite existing files */ -#define H5F_ACC_EXCL (H5CHECK H5OPEN 0x0004u) /*fail if file already exists*/ +#define H5F_ACC_RDONLY (H5CHECK H5OPEN 0x0000u) /**< absence of rdwr => rd-only */ +#define H5F_ACC_RDWR (H5CHECK H5OPEN 0x0001u) /**< open for read and write */ +#define H5F_ACC_TRUNC (H5CHECK H5OPEN 0x0002u) /**< overwrite existing files */ +#define H5F_ACC_EXCL (H5CHECK H5OPEN 0x0004u) /**< fail if file already exists*/ /* NOTE: 0x0008u was H5F_ACC_DEBUG, now deprecated */ -#define H5F_ACC_CREAT (H5CHECK H5OPEN 0x0010u) /*create non-existing files */ +#define H5F_ACC_CREAT (H5CHECK H5OPEN 0x0010u) /**< create non-existing files */ #define H5F_ACC_SWMR_WRITE \ - (H5CHECK 0x0020u) /*indicate that this file is \ - * open for writing in a \ - * single-writer/multi-reader (SWMR) \ - * scenario. Note that the \ - * process(es) opening the file \ - * for reading must open the file \ - * with RDONLY access, and use \ - * the special "SWMR_READ" access \ - * flag. */ + (H5CHECK 0x0020u) /**< indicate that this file is open for writing in a \ + single-writer/multi-reader (SWMR) scenario. \ + Note that the process(es) opening the file for reading must \ + open the file with RDONLY access, and use the special "SWMR_READ" \ + access flag. */ #define H5F_ACC_SWMR_READ \ - (H5CHECK 0x0040u) /*indicate that this file is \ + (H5CHECK 0x0040u) /**< indicate that this file is \ * open for reading in a \ * single-writer/multi-reader (SWMR) \ * scenario. Note that the \ @@ -73,19 +69,23 @@ * open the file with the RDONLY \ * flag. */ -/* Value passed to H5Pset_elink_acc_flags to cause flags to be taken from the - * parent file. */ -#define H5F_ACC_DEFAULT (H5CHECK H5OPEN 0xffffu) /*ignore setting on lapl */ +/** + * Default property list identifier + * + * \internal Value passed to H5Pset_elink_acc_flags to cause flags to be taken from the parent file. + * \internal ignore setting on lapl + */ +#define H5F_ACC_DEFAULT (H5CHECK H5OPEN 0xffffu) /* Flags for H5Fget_obj_count() & H5Fget_obj_ids() calls */ -#define H5F_OBJ_FILE (0x0001u) /* File objects */ -#define H5F_OBJ_DATASET (0x0002u) /* Dataset objects */ -#define H5F_OBJ_GROUP (0x0004u) /* Group objects */ -#define H5F_OBJ_DATATYPE (0x0008u) /* Named datatype objects */ -#define H5F_OBJ_ATTR (0x0010u) /* Attribute objects */ +#define H5F_OBJ_FILE (0x0001u) /**< File objects */ +#define H5F_OBJ_DATASET (0x0002u) /**< Dataset objects */ +#define H5F_OBJ_GROUP (0x0004u) /**< Group objects */ +#define H5F_OBJ_DATATYPE (0x0008u) /**< Named datatype objects */ +#define H5F_OBJ_ATTR (0x0010u) /**< Attribute objects */ #define H5F_OBJ_ALL (H5F_OBJ_FILE | H5F_OBJ_DATASET | H5F_OBJ_GROUP | H5F_OBJ_DATATYPE | H5F_OBJ_ATTR) -#define H5F_OBJ_LOCAL (0x0020u) /* Restrict search to objects opened through current file ID */ -/* (as opposed to objects opened through any file ID accessing this file) */ +#define H5F_OBJ_LOCAL (0x0020u) /**< Restrict search to objects opened through current file ID + (as opposed to objects opened through any file ID accessing this file) */ #define H5F_FAMILY_DEFAULT (hsize_t)0 @@ -99,127 +99,146 @@ #define H5F_MPIO_DEBUG_KEY "H5F_mpio_debug_key" #endif /* H5_HAVE_PARALLEL */ -/* The difference between a single file and a set of mounted files */ +/** + * The difference between a single file and a set of mounted files + */ typedef enum H5F_scope_t { - H5F_SCOPE_LOCAL = 0, /*specified file handle only */ - H5F_SCOPE_GLOBAL = 1 /*entire virtual file */ + H5F_SCOPE_LOCAL = 0, /**< specified file handle only */ + H5F_SCOPE_GLOBAL = 1 /**< entire virtual file */ } H5F_scope_t; -/* Unlimited file size for H5Pset_external() */ +/** + * Unlimited file size for H5Pset_external() + */ #define H5F_UNLIMITED ((hsize_t)(-1L)) -/* How does file close behave? - * H5F_CLOSE_DEFAULT - Use the degree pre-defined by underlining VFL - * H5F_CLOSE_WEAK - file closes only after all opened objects are closed - * H5F_CLOSE_SEMI - if no opened objects, file is close; otherwise, file - close fails - * H5F_CLOSE_STRONG - if there are opened objects, close them first, then - close file +/** + * How does file close behave? */ typedef enum H5F_close_degree_t { - H5F_CLOSE_DEFAULT = 0, - H5F_CLOSE_WEAK = 1, - H5F_CLOSE_SEMI = 2, - H5F_CLOSE_STRONG = 3 + H5F_CLOSE_DEFAULT = 0, /**< Use the degree pre-defined by underlining VFL */ + H5F_CLOSE_WEAK = 1, /**< File closes only after all opened objects are closed */ + H5F_CLOSE_SEMI = 2, /**< If no opened objects, file is close; otherwise, file close fails */ + H5F_CLOSE_STRONG = 3 /**< If there are opened objects, close them first, then close file */ } H5F_close_degree_t; -/* Current "global" information about file */ +/** + * Current "global" information about file + */ +//! [H5F_info2_t_snip] typedef struct H5F_info2_t { struct { - unsigned version; /* Superblock version # */ - hsize_t super_size; /* Superblock size */ - hsize_t super_ext_size; /* Superblock extension size */ + unsigned version; /**< Superblock version # */ + hsize_t super_size; /**< Superblock size */ + hsize_t super_ext_size; /**< Superblock extension size */ } super; struct { - unsigned version; /* Version # of file free space management */ - hsize_t meta_size; /* Free space manager metadata size */ - hsize_t tot_space; /* Amount of free space in the file */ + unsigned version; /**< Version # of file free space management */ + hsize_t meta_size; /**< Free space manager metadata size */ + hsize_t tot_space; /**< Amount of free space in the file */ } free; struct { - unsigned version; /* Version # of shared object header info */ - hsize_t hdr_size; /* Shared object header message header size */ - H5_ih_info_t msgs_info; /* Shared object header message index & heap size */ + unsigned version; /**< Version # of shared object header info */ + hsize_t hdr_size; /**< Shared object header message header size */ + H5_ih_info_t msgs_info; /**< Shared object header message index & heap size */ } sohm; } H5F_info2_t; +//! [H5F_info2_t_snip] -/* - * Types of allocation requests. The values larger than H5FD_MEM_DEFAULT +/** + * Types of allocation requests. The values larger than #H5FD_MEM_DEFAULT * should not change other than adding new types to the end. These numbers * might appear in files. * - * Note: please change the log VFD flavors array if you change this - * enumeration. + * \internal Please change the log VFD flavors array if you change this + * enumeration. */ typedef enum H5F_mem_t { - H5FD_MEM_NOLIST = -1, /* Data should not appear in the free list. + H5FD_MEM_NOLIST = -1, /**< Data should not appear in the free list. * Must be negative. */ - H5FD_MEM_DEFAULT = 0, /* Value not yet set. Can also be the + H5FD_MEM_DEFAULT = 0, /**< Value not yet set. Can also be the * datatype set in a larger allocation * that will be suballocated by the library. * Must be zero. */ - H5FD_MEM_SUPER = 1, /* Superblock data */ - H5FD_MEM_BTREE = 2, /* B-tree data */ - H5FD_MEM_DRAW = 3, /* Raw data (content of datasets, etc.) */ - H5FD_MEM_GHEAP = 4, /* Global heap data */ - H5FD_MEM_LHEAP = 5, /* Local heap data */ - H5FD_MEM_OHDR = 6, /* Object header data */ - - H5FD_MEM_NTYPES /* Sentinel value - must be last */ + H5FD_MEM_SUPER = 1, /**< Superblock data */ + H5FD_MEM_BTREE = 2, /**< B-tree data */ + H5FD_MEM_DRAW = 3, /**< Raw data (content of datasets, etc.) */ + H5FD_MEM_GHEAP = 4, /**< Global heap data */ + H5FD_MEM_LHEAP = 5, /**< Local heap data */ + H5FD_MEM_OHDR = 6, /**< Object header data */ + + H5FD_MEM_NTYPES /**< Sentinel value - must be last */ } H5F_mem_t; -/* Free space section information */ +/** + * Free space section information + */ +//! [H5F_sect_info_t_snip] typedef struct H5F_sect_info_t { - haddr_t addr; /* Address of free space section */ - hsize_t size; /* Size of free space section */ + haddr_t addr; /**< Address of free space section */ + hsize_t size; /**< Size of free space section */ } H5F_sect_info_t; +//! [H5F_sect_info_t_snip] -/* Library's format versions */ +/** + * Library's format versions + */ typedef enum H5F_libver_t { H5F_LIBVER_ERROR = -1, - H5F_LIBVER_EARLIEST = 0, /* Use the earliest possible format for storing objects */ - H5F_LIBVER_V18 = 1, /* Use the latest v18 format for storing objects */ - H5F_LIBVER_V110 = 2, /* Use the latest v110 format for storing objects */ - H5F_LIBVER_V112 = 3, /* Use the latest v112 format for storing objects */ + H5F_LIBVER_EARLIEST = 0, /**< Use the earliest possible format for storing objects */ + H5F_LIBVER_V18 = 1, /**< Use the latest v18 format for storing objects */ + H5F_LIBVER_V110 = 2, /**< Use the latest v110 format for storing objects */ + H5F_LIBVER_V112 = 3, /**< Use the latest v112 format for storing objects */ H5F_LIBVER_NBOUNDS } H5F_libver_t; #define H5F_LIBVER_LATEST H5F_LIBVER_V112 -/* File space handling strategy */ +/** + * File space handling strategy + */ typedef enum H5F_fspace_strategy_t { H5F_FSPACE_STRATEGY_FSM_AGGR = - 0, /* Mechanisms: free-space managers, aggregators, and virtual file drivers */ - /* This is the library default when not set */ + 0, /**< Mechanisms: free-space managers, aggregators, and virtual file drivers + This is the library default when not set */ H5F_FSPACE_STRATEGY_PAGE = - 1, /* Mechanisms: free-space managers with embedded paged aggregation and virtual file drivers */ - H5F_FSPACE_STRATEGY_AGGR = 2, /* Mechanisms: aggregators and virtual file drivers */ - H5F_FSPACE_STRATEGY_NONE = 3, /* Mechanisms: virtual file drivers */ - H5F_FSPACE_STRATEGY_NTYPES /* must be last */ + 1, /**< Mechanisms: free-space managers with embedded paged aggregation and virtual file drivers */ + H5F_FSPACE_STRATEGY_AGGR = 2, /**< Mechanisms: aggregators and virtual file drivers */ + H5F_FSPACE_STRATEGY_NONE = 3, /**< Mechanisms: virtual file drivers */ + H5F_FSPACE_STRATEGY_NTYPES /**< Sentinel */ } H5F_fspace_strategy_t; -/* Deprecated: File space handling strategy for release 1.10.0 */ -/* They are mapped to H5F_fspace_strategy_t as defined above from release 1.10.1 onwards */ +/** + * File space handling strategy for release 1.10.0 + * + * \deprecated 1.10.1 + */ typedef enum H5F_file_space_type_t { - H5F_FILE_SPACE_DEFAULT = 0, /* Default (or current) free space strategy setting */ - H5F_FILE_SPACE_ALL_PERSIST = 1, /* Persistent free space managers, aggregators, virtual file driver */ - H5F_FILE_SPACE_ALL = 2, /* Non-persistent free space managers, aggregators, virtual file driver */ - /* This is the library default */ - H5F_FILE_SPACE_AGGR_VFD = 3, /* Aggregators, Virtual file driver */ - H5F_FILE_SPACE_VFD = 4, /* Virtual file driver */ - H5F_FILE_SPACE_NTYPES /* must be last */ + H5F_FILE_SPACE_DEFAULT = 0, /**< Default (or current) free space strategy setting */ + H5F_FILE_SPACE_ALL_PERSIST = 1, /**< Persistent free space managers, aggregators, virtual file driver */ + H5F_FILE_SPACE_ALL = 2, /**< Non-persistent free space managers, aggregators, virtual file driver + This is the library default */ + H5F_FILE_SPACE_AGGR_VFD = 3, /**< Aggregators, Virtual file driver */ + H5F_FILE_SPACE_VFD = 4, /**< Virtual file driver */ + H5F_FILE_SPACE_NTYPES /**< Sentinel */ } H5F_file_space_type_t; -/* Data structure to report the collection of read retries for metadata items with checksum */ -/* Used by public routine H5Fget_metadata_read_retry_info() */ #define H5F_NUM_METADATA_READ_RETRY_TYPES 21 + +/** + * Data structure to report the collection of read retries for metadata items with checksum as + * used by H5Fget_metadata_read_retry_info() + */ typedef struct H5F_retry_info_t { unsigned nbins; uint32_t *retries[H5F_NUM_METADATA_READ_RETRY_TYPES]; } H5F_retry_info_t; -/* Callback for H5Pset_object_flush_cb() in a file access property list */ +/** + * Callback for H5Pset_object_flush_cb() in a file access property list + */ typedef herr_t (*H5F_flush_cb_t)(hid_t object_id, void *udata); /*********************/ @@ -229,56 +248,1073 @@ typedef herr_t (*H5F_flush_cb_t)(hid_t object_id, void *udata); extern "C" { #endif +/** + * \ingroup H5F + * + * \brief Checks if a file can be opened with a given file access property + * list + * + * \param[in] container_name Name of a file + * \fapl_id + * + * \return \htri_t + * + * \details H5Fis_accessible() checks if the file specified by \p + * container_name can be opened with the file access property list + * \p fapl_id. + * + * \note The H5Fis_accessible() function enables files to be checked with a + * given file access property list, unlike H5Fis_hdf5(), which only uses + * the default file driver when opening a file. + * + * \since 1.12.0 + * + */ H5_DLL htri_t H5Fis_accessible(const char *container_name, hid_t fapl_id); -H5_DLL hid_t H5Fcreate(const char *filename, unsigned flags, hid_t create_plist, hid_t access_plist); -H5_DLL hid_t H5Fopen(const char *filename, unsigned flags, hid_t access_plist); +/** + * \example H5Fcreate.c + * After creating an HDF5 file with H5Fcreate(), we close it with + * H5Fclose(). + */ +/** + * \ingroup H5F + * + * \brief Creates an HDF5 file + * + * \param[in] filename Name of the file to create + * \param[in] flags File access flags. Allowable values are: + * - #H5F_ACC_TRUNC: Truncate file, if it already exists, + * erasing all data previously stored in the file + * - #H5F_ACC_EXCL: Fail if file already exists + * \fcpl_id + * \fapl_id + * \return \hid_t{file} + * + * \details H5Fcreate() is the primary function for creating HDF5 files; it + * creates a new HDF5 file with the specified name and property lists. + * + * The \p filename parameter specifies the name of the new file. + * + * The \p flags parameter specifies whether an existing file is to be + * overwritten. It should be set to either #H5F_ACC_TRUNC to overwrite + * an existing file or #H5F_ACC_EXCL, instructing the function to fail + * if the file already exists. + * + * New files are always created in read-write mode, so the read-write + * and read-only flags, #H5F_ACC_RDWR and #H5F_ACC_RDONLY, + * respectively, are not relevant in this function. Further note that + * a specification of #H5F_ACC_RDONLY will be ignored; the file will + * be created in read-write mode, regardless. + * + * More complex behaviors of file creation and access are controlled + * through the file creation and file access property lists, + * \p fcpl_id and \p fapl_id, respectively. The value of #H5P_DEFAULT + * for any property list value indicates that the library should use + * the default values for that appropriate property list. + * + * The return value is a file identifier for the newly-created file; + * this file identifier should be closed by calling H5Fclose() when + * it is no longer needed. + * + * \include H5Fcreate.c + * + * \note #H5F_ACC_TRUNC and #H5F_ACC_EXCL are mutually exclusive; use + * exactly one. + * + * \note An additional flag, #H5F_ACC_DEBUG, prints debug information. This + * flag can be combined with one of the above values using the bit-wise + * OR operator (\c |), but it is used only by HDF5 library developers; + * \Emph{it is neither tested nor supported for use in applications}. + * + * \attention \Bold{Special case — File creation in the case of an already-open file:} + * If a file being created is already opened, by either a previous + * H5Fopen() or H5Fcreate() call, the HDF5 library may or may not + * detect that the open file and the new file are the same physical + * file. (See H5Fopen() regarding the limitations in detecting the + * re-opening of an already-open file.)\n + * If the library detects that the file is already opened, + * H5Fcreate() will return a failure, regardless of the use of + * #H5F_ACC_TRUNC.\n + * If the library does not detect that the file is already opened + * and #H5F_ACC_TRUNC is not used, H5Fcreate() will return a failure + * because the file already exists. Note that this is correct + * behavior.\n + * But if the library does not detect that the file is already + * opened and #H5F_ACC_TRUNC is used, H5Fcreate() will truncate the + * existing file and return a valid file identifier. Such a + * truncation of a currently-opened file will almost certainly + * result in errors. While unlikely, the HDF5 library may not be + * able to detect, and thus report, such errors.\n + * Applications should avoid calling H5Fcreate() with an already + * opened file. + * + * \since 1.0.0 + * + * \see H5Fopen(), H5Fclose() + * + */ +H5_DLL hid_t H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id); +/** + * \ingroup H5F + * + * \brief Opens an existing HDF5 file + * + * \param[in] filename Name of the file to be opened + * \param[in] flags File access flags. Allowable values are: + * - #H5F_ACC_RDWR: Allows read and write access to file + * - #H5F_ACC_RDONLY: Allows read-only access to file + * - #H5F_ACC_RDWR \c | #H5F_ACC_SWMR_WRITE: Indicates that + * the file is open for writing in a + * single-writer/multi-writer (SWMR) scenario. + * - #H5F_ACC_RDONLY \c | #H5F_ACC_SWMR_READ: Indicates + * that the file is open for reading in a + * single-writer/multi-reader (SWMR) scenario. + * - An additional flag, #H5F_ACC_DEBUG, prints debug + * information. This flag can be combined with one of the + * above values using the bit-wise OR operator (\c |), but + * it is used only by HDF5 library developers; + * \Emph{it is neither tested nor supported} for use in + * applications. + * \fapl_id + * \return \hid_t{file} + * + * \details H5Fopen() is the primary function for accessing existing HDF5 files. + * This function opens the named file in the specified access mode and + * with the specified access property list. + * + * Note that H5Fopen() does not create a file if it does not already + * exist; see H5Fcreate(). + * + * The \p filename parameter specifies the name of the file to be + * opened. + * + * The \p fapl_id parameter specifies the file access property list. + * Use of #H5P_DEFAULT specifies that default I/O access properties + * are to be used. + * + * The \p flags parameter specifies whether the file will be opened in + * read-write or read-only mode, #H5F_ACC_RDWR or #H5F_ACC_RDONLY, + * respectively. More complex behaviors of file access are controlled + * through the file-access property list. + * + * The return value is a file identifier for the open file; this file + * identifier should be closed by calling H5Fclose() when it is no + * longer needed. + * + * \note #H5F_ACC_RDWR and #H5F_ACC_RDONLY are mutually exclusive; use + * exactly one. + * + * \attention \Bold{Special cases — Multiple opens:} A file can often be opened + * with a new H5Fopen() call without closing an already-open + * identifier established in a previous H5Fopen() or H5Fcreate() + * call. Each such H5Fopen() call will return a unique identifier + * and the file can be accessed through any of these identifiers as + * long as the identifier remains valid. In such multiply-opened + * cases, the open calls must use the same flags argument and the + * file access property lists must use the same file close degree + * property setting (see the external link discussion below and + * H5Pset_fclose_degree()).\n + * In some cases, such as files on a local Unix file system, the + * HDF5 library can detect that a file is multiply opened and will + * maintain coherent access among the file identifiers.\n + * But in many other cases, such as parallel file systems or + * networked file systems, it is not always possible to detect + * multiple opens of the same physical file. In such cases, HDF5 + * will treat the file identifiers as though they are accessing + * different files and will be unable to maintain coherent access. + * Errors are likely to result in these cases. While unlikely, the + * HDF5 library may not be able to detect, and thus report, + * such errors.\n + * It is generally recommended that applications avoid multiple + * opens of the same file. + * + * \attention \Bold{Special restriction on multiple opens of a file first + * opened by means of an external link:} When an external link is + * followed, the external file is always opened with the weak file + * close degree property setting, #H5F_CLOSE_WEAK (see + * H5Lcreate_external() and H5Pset_fclose_degree()). If the file is + * reopened with H5Fopen while it remains held open from such an + * external link call, the file access property list used in the + * open call must include the file close degree setting + * #H5F_CLOSE_WEAK or the open will fail. + * + * \version 1.10.0 The #H5F_ACC_SWMR_WRITE and #H5F_ACC_SWMR_READ flags were added. + * + * \see H5Fclose() + * + */ +H5_DLL hid_t H5Fopen(const char *filename, unsigned flags, hid_t fapl_id); +/** + * \ingroup H5F + * + * \brief Returns a new identifier for a previously-opened HDF5 file + * + * \param[in] file_id Identifier of a file for which an additional identifier + * is required + * + * \return \hid_t{file} + * + * \details H5Freopen() returns a new file identifier for an already-open HDF5 + * file, as specified by \p file_id. Both identifiers share caches and + * other information. The only difference between the identifiers is + * that the new identifier is not mounted anywhere and no files are + * mounted on it. + * + * The new file identifier should be closed by calling H5Fclose() when + * it is no longer needed. + * + * \note Note that there is no circumstance under which H5Freopen() can + * actually open a closed file; the file must already be open and have an + * active \p file_id. E.g., one cannot close a file with H5Fclose() on + * \p file_id then use H5Freopen() on \p file_id to reopen it. + * + */ H5_DLL hid_t H5Freopen(hid_t file_id); +/** + * \ingroup H5F + * + * \brief Flushes all buffers associated with a file to storage + * + * \loc_id{object_id} + * \param[in] scope The scope of the flush action + * + * \return \herr_t + * + * \details H5Fflush() causes all buffers associated with a file to be + * immediately flushed to storage without removing the data from the + * cache. + * + * \p object_id can be any object associated with the file, including + * the file itself, a dataset, a group, an attribute, or a named + * datatype. + * + * \p scope specifies whether the scope of the flush action is + * global or local. Valid values are as follows: + * \scopes + * + * \attention HDF5 does not possess full control over buffering. H5Fflush() + * flushes the internal HDF5 buffers then asks the operating system + * (the OS) to flush the system buffers for the open files. After + * that, the OS is responsible for ensuring that the data is + * actually flushed to disk. + * + */ H5_DLL herr_t H5Fflush(hid_t object_id, H5F_scope_t scope); +/** + * \example H5Fclose.c + * After creating an HDF5 file with H5Fcreate(), we close it with + * H5Fclose(). + */ +/** + * \ingroup H5F + * + * \brief Terminates access to an HDF5 file + * + * \file_id + * \return \herr_t + * + * \details H5Fclose() terminates access to an HDF5 file (specified by + * \p file_id) by flushing all data to storage. + * + * If this is the last file identifier open for the file and no other + * access identifier is open (e.g., a dataset identifier, group + * identifier, or shared datatype identifier), the file will be fully + * closed and access will end. + * + * Use H5Fclose() as shown in the following example: + * \include H5Fclose.c + * + * \note \Bold{Delayed close:} Note the following deviation from the + * above-described behavior. If H5Fclose() is called for a file but one + * or more objects within the file remain open, those objects will remain + * accessible until they are individually closed. Thus, if the dataset + * \c data_sample is open when H5Fclose() is called for the file + * containing it, \c data_sample will remain open and accessible + * (including writable) until it is explicitly closed. The file will be + * automatically closed once all objects in the file have been closed.\n + * Be warned, however, that there are circumstances where it is not + * possible to delay closing a file. For example, an MPI-IO file close is + * a collective call; all of the processes that opened the file must + * close it collectively. The file cannot be closed at some time in the + * future by each process in an independent fashion. Another example is + * that an application using an AFS token-based file access privilege may + * destroy its AFS token after H5Fclose() has returned successfully. This + * would make any future access to the file, or any object within it, + * illegal.\n + * In such situations, applications must close all open objects in a file + * before calling H5Fclose. It is generally recommended to do so in all + * cases. + * + * \see H5Fopen() + * + */ H5_DLL herr_t H5Fclose(hid_t file_id); +/** + * \ingroup H5F + * + * \brief Deletes an HDF5 file + * + * \param[in] filename Name of the file to delete + * \fapl_id + * + * \return \herr_t + * + * \details H5Fdelete() deletes an HDF5 file \p filename with a file access + * property list \p fapl_id. The \p fapl_id should be configured with + * the same VOL connector or VFD that was used to open the file. + * + * This API was introduced for use with the Virtual Object Layer + * (VOL). With the VOL, HDF5 "files" can map to arbitrary storage + * schemes such as object stores and relational database tables. The + * data created by these implementations may be inconvenient for a + * user to remove without a detailed knowledge of the storage scheme. + * H5Fdelete() gives VOL connector authors the ability to add + * connector-specific delete code to their connectors so that users + * can remove these "files" without detailed knowledge of the storage + * scheme. + * + * For a VOL connector, H5Fdelete() deletes the file in a way that + * makes sense for the specified VOL connector. + * + * For the native HDF5 connector, HDF5 files will be deleted via the + * VFDs, each of which will have to be modified to delete the files it + * creates. + * + * For all implementations, H5Fdelete() will first check if the file + * is an HDF5 file via H5Fis_accessible(). This is done to ensure that + * H5Fdelete() cannot be used as an arbitrary file deletion call. + * + * \since 1.12.0 + * + */ H5_DLL herr_t H5Fdelete(const char *filename, hid_t fapl_id); +/** + * \ingroup H5F + * + * \brief Returns a file creation property list identifier + * + * \file_id + * \return \hid_t{file creation property list} + * + * \details H5Fget_create_plist() returns the file creation property list + * identifier identifying the creation properties used to create this + * file. This function is useful for duplicating properties when + * creating another file. + * + * The creation property list identifier should be released with + * H5Pclose(). + * + */ H5_DLL hid_t H5Fget_create_plist(hid_t file_id); +/** + * \ingroup H5F + * + * \brief Returns a file access property list identifier + * + * \file_id + * \return \hid_t{file access property list} + * + * \details H5Fget_access_plist() returns the file access property list + * identifier of the specified file. + * + */ H5_DLL hid_t H5Fget_access_plist(hid_t file_id); +/** + * \ingroup H5F + * + * \brief Determines the read/write or read-only status of a file + * + * \file_id + * \param[out] intent Access mode flag as originally passed with H5Fopen() + * + * \return \herr_t + * + * \details Given the identifier of an open file, \p file_id, H5Fget_intent() + * retrieves the intended access mode" flag passed with H5Fopen() when + * the file was opened. + * + * The value of the flag is returned in \p intent. Valid values are as + * follows: + * \file_access + * + * \note The function will not return an error if intent is NULL; it will + * simply do nothing. + * + * \version 1.10.0 C function enhanced to work with SWMR functionality. + * + * \since 1.8.0 + * + */ H5_DLL herr_t H5Fget_intent(hid_t file_id, unsigned *intent); +/** + * \ingroup H5F + * + * \brief Retrieves a file's file number that uniquely identifies an open file + * + * \file_id + * \param[out] fileno A buffer to hold the file number + * + * \return \herr_t + * + * \details H5Fget_fileno() retrieves a file number for a file specified by the + * file identifier \p file_id and the pointer \p fnumber to the file + * number. + * + * \since 1.12.0 + * + */ H5_DLL herr_t H5Fget_fileno(hid_t file_id, unsigned long *fileno); +/** + * \ingroup H5F + * + * \brief Returns the number of open object identifiers for an open file + * + * \file_id or #H5F_OBJ_ALL for all currently-open HDF5 files + * \param[in] types Type of object for which identifiers are to be returned + * + * \return Returns the number of open objects if successful; otherwise returns + * a negative value. + * + * \details Given the identifier of an open file, file_id, and the desired + * object types, types, H5Fget_obj_count() returns the number of open + * object identifiers for the file. + * + * To retrieve a count of open identifiers for open objects in all + * HDF5 application files that are currently open, pass the value + * #H5F_OBJ_ALL in \p file_id. + * + * The types of objects to be counted are specified in types as + * follows: + * \obj_types + * + * Multiple object types can be combined with the + * logical \c OR operator (|). For example, the expression + * \c (#H5F_OBJ_DATASET|#H5F_OBJ_GROUP) would call for datasets and + * groups. + * + * \version 1.6.8, 1.8.2 C function return type changed to \c ssize_t. + * \version 1.6.5 #H5F_OBJ_LOCAL has been added as a qualifier on the types + * of objects to be counted. #H5F_OBJ_LOCAL restricts the + * search to objects opened through current file identifier. + * + */ H5_DLL ssize_t H5Fget_obj_count(hid_t file_id, unsigned types); +/** + *------------------------------------------------------------------------- + * \ingroup H5F + * + * \brief Returns a list of open object identifiers + * + * \file_id or #H5F_OBJ_ALL for all currently-open HDF5 files + * \param[in] types Type of object for which identifiers are to be returned + * \param[in] max_objs Maximum number of object identifiers to place into + * \p obj_id_list + * \param[out] obj_id_list Pointer to the returned buffer of open object + * identifiers + * + * \return Returns number of objects placed into \p obj_id_list if successful; + * otherwise returns a negative value. + * + * \details Given the file identifier \p file_id and the type of objects to be + * identified, types, H5Fget_obj_ids() returns the list of identifiers + * for all open HDF5 objects fitting the specified criteria. + * + * To retrieve identifiers for open objects in all HDF5 application + * files that are currently open, pass the value #H5F_OBJ_ALL in + * \p file_id. + * + * The types of object identifiers to be retrieved are specified in + * types using the codes listed for the same parameter in + * H5Fget_obj_count(). + * + * To retrieve a count of open objects, use the H5Fget_obj_count() + * function. This count can be used to set the \p max_objs parameter. + * + * \version 1.8.2 C function return type changed to \c ssize_t and \p + * max_objs parameter datatype changed to \c size_t. + * \version 1.6.8 C function return type changed to \c ssize_t and \p + * max_objs parameter datatype changed to \c size_t. + * \since 1.6.0 + * + */ H5_DLL ssize_t H5Fget_obj_ids(hid_t file_id, unsigned types, size_t max_objs, hid_t *obj_id_list); +/** + * \ingroup H5F + * + * \brief Returns pointer to the file handle from the virtual file driver + * + * \file_id + * \fapl_id{fapl} + * \param[out] file_handle Pointer to the file handle being used by the + * low-level virtual file driver + * + * \return \herr_t + * + * \details Given the file identifier \p file_id and the file access property + * list \p fapl_id, H5Fget_vfd_handle() returns a pointer to the file + * handle from the low-level file driver currently being used by the + * HDF5 library for file I/O. + * + * \note For most drivers, the value of \p fapl_id will be #H5P_DEFAULT. For + * the \c FAMILY or \c MULTI drivers, this value should be defined + * through the property list functions: H5Pset_family_offset() for the + * \c FAMILY driver and H5Pset_multi_type() for the \c MULTI driver + * + * \since 1.6.0 + * + */ H5_DLL herr_t H5Fget_vfd_handle(hid_t file_id, hid_t fapl, void **file_handle); +/** + * \ingroup H5F + * + * \brief Mounts an HDF5 file + * + * \loc_id{loc} + * \param[in] name Name of the group onto which the file specified by \p child + * is to be mounted + * \file_id{child} + * \param[in] plist File mount property list identifier. Pass #H5P_DEFAULT! + * + * \return \herr_t + * + * \details H5Fmount() mounts the file specified by \p child onto the object + * specified by \p loc and \p name using the mount properties \p plist + * If the object specified by \p loc is a dataset, named datatype or + * attribute, then the file will be mounted at the location where the + * attribute, dataset, or named datatype is attached. + * + * \note To date, no file mount properties have been defined in HDF5. The + * proper value to pass for \p plist is #H5P_DEFAULT, indicating the + * default file mount property list. + * + */ H5_DLL herr_t H5Fmount(hid_t loc, const char *name, hid_t child, hid_t plist); +/** + * \ingroup H5F + * + * \brief Unounts an HDF5 file + * + * \loc_id{loc} + * \param[in] name Name of the mount point + * + * \return \herr_t + * + * \details Given a mount point, H5Funmount() dissociates the mount point's + * file from the file mounted there. This function does not close + * either file. + * + * The mount point can be either the group in the parent or the root + * group of the mounted file (both groups have the same name). If the + * mount point was opened before the mount then it is the group in the + * parent; if it was opened after the mount then it is the root group + * of the child. + * + */ H5_DLL herr_t H5Funmount(hid_t loc, const char *name); +/** + * \ingroup H5F + * + * \brief Returns the amount of free space in a file (in bytes) + * + * \file_id + * + * \return Returns the amount of free space in the file if successful; + * otherwise returns a negative value. + * + * \details Given the identifier of an open file, \p file_id, + * H5Fget_freespace() returns the amount of space that is unused by + * any objects in the file. + * + * The interpretation of this number depends on the configured free space + * management strategy. For example, if the HDF5 library only tracks free + * space in a file from a file open or create until that file is closed, + * then this routine will report the free space that has been created + * during that interval. + * + * \since 1.6.1 + * + */ H5_DLL hssize_t H5Fget_freespace(hid_t file_id); +/** + * \ingroup H5F + * + * \brief Returns the size of an HDF5 file (in bytes) + * + * \file_id + * \param[out] size Size of the file, in bytes + * + * \return \herr_t + * + * \details H5Fget_filesize() returns the size of the HDF5 file specified by + * \p file_id. + * + * The returned size is that of the entire file, as opposed to only + * the HDF5 portion of the file. I.e., size includes the user block, + * if any, the HDF5 portion of the file, and any data that may have + * been appended beyond the data written through the HDF5 library. + * + * \version 1.6.3 Fortran subroutine introduced in this release. + * + * \since 1.6.3 + * + */ H5_DLL herr_t H5Fget_filesize(hid_t file_id, hsize_t *size); +/** + * \ingroup H5F + * + * \brief Retrieves the file's end-of-allocation (EOA) + * + * \file_id + * \param[out] eoa The file's EOA + * + * \return \herr_t + * + * \details H5Fget_eoa() retrieves the file's EOA and returns it in the + * parameter eoa. + * + * \since 1.10.2 + * + */ H5_DLL herr_t H5Fget_eoa(hid_t file_id, haddr_t *eoa); +/** + * \ingroup H5F + * + * \brief Sets the file' EOA to the maximum of (EOA, EOF) + increment + * + * \file_id + * \param[in] increment The number of bytes to be added to the maximum of + * (EOA, EOF) + * + * \return \herr_t + * + * \details H5Fincrement_filesize() sets the file's EOA to the maximum of (EOA, + * EOF) + \p increment. The EOA is the end-of-file address stored in + * the file's superblock while EOF is the file's actual end-of-file. + * + * \since 1.10.2 + * + */ H5_DLL herr_t H5Fincrement_filesize(hid_t file_id, hsize_t increment); +/** + * \ingroup H5F + * + * \brief Retrieves a copy of the image of an existing, open file + * + * \file_id + * \param[out] buf_ptr Pointer to the buffer into which the image of the + * HDF5 file is to be copied. If \p buf_ptr is NULL, + * no data will be copied but the function’s return value + * will still indicate the buffer size required (or a + * negative value on error). + * \param[out] buf_len Size of the supplied buffer + * + * \return ssize_t + * + * \details H5Fget_file_image() retrieves a copy of the image of an existing, + * open file. This routine can be used with files opened using the + * SEC2 (or POSIX), STDIO, and Core (or Memory) virtual file drivers + * (VFDs). + * + * If the return value of H5Fget_file_image() is a positive value, it + * will be the length in bytes of the buffer required to store the + * file image. So if the file size is unknown, it can be safely + * determined with an initial H5Fget_file_image() call with buf_ptr + * set to NULL. The file image can then be retrieved with a second + * H5Fget_file_image() call with \p buf_len set to the initial call’s + * return value. + * + * While the current file size can also be retrieved with + * H5Fget_filesize(), that call may produce a larger value than is + * needed. The value returned by H5Fget_filesize() includes the user + * block, if it exists, and any unallocated space at the end of the + * file. It is safe in all situations to get the file size with + * H5Fget_file_image() and it often produces a value that is more + * appropriate for the size of a file image buffer. + * + * \note \Bold{Recommended Reading:} This function is part of the file image + * operations feature set. It is highly recommended to study the guide + * "HDF5 File Image Operations" before using this feature set.\n See the + * "See Also" section below for links to other elements of HDF5 file + * image operations. \todo Fix the references. + * + * \attention H5Pget_file_image() will fail, returning a negative value, if the + * file is too large for the supplied buffer. + * + * \see H5LTopen_file_image(), H5Pset_file_image(), H5Pget_file_image(), + * H5Pset_file_image_callbacks(), H5Pget_file_image_callbacks() + * + * \version 1.8.13 Fortran subroutine added in this release. + * + * \since 1.8.0 + * + */ H5_DLL ssize_t H5Fget_file_image(hid_t file_id, void *buf_ptr, size_t buf_len); +/** + * \ingroup MDC + * + * \todo Finish this! + */ H5_DLL herr_t H5Fget_mdc_config(hid_t file_id, H5AC_cache_config_t *config_ptr); +/** + * \ingroup MDC + * + * \todo Finish this! + */ H5_DLL herr_t H5Fset_mdc_config(hid_t file_id, H5AC_cache_config_t *config_ptr); +/** + * \ingroup MDC + * + * \todo Finish this! + */ H5_DLL herr_t H5Fget_mdc_hit_rate(hid_t file_id, double *hit_rate_ptr); +/** + * \ingroup MDC + * + * \todo Finish this! + */ H5_DLL herr_t H5Fget_mdc_size(hid_t file_id, size_t *max_size_ptr, size_t *min_clean_size_ptr, size_t *cur_size_ptr, int *cur_num_entries_ptr); +/** + * \ingroup MDC + * + * \todo Finish this! + */ H5_DLL herr_t H5Freset_mdc_hit_rate_stats(hid_t file_id); +/** + * \ingroup H5F + * + * \brief Retrieves name of file to which object belongs + * + * \obj_id + * \param[out] name Buffer for the file name + * \param[in] size Size, in bytes, of the \p name buffer + * + * \return Returns the length of the file name if successful; otherwise returns + * a negative value. + * + * \details H5Fget_name() retrieves the name of the file to which the object \p + * obj_id belongs. The object can be a file, group, dataset, + * attribute, or named datatype. + * + * Up to \p size characters of the file name are returned in \p name; + * additional characters, if any, are not returned to the user + * application. + * + * If the length of the name, which determines the required value of + * size, is unknown, a preliminary H5Fget_name() call can be made by + * setting \p name to NULL. The return value of this call will be the + * size of the file name; that value plus one (1) can then be assigned + * to size for a second H5Fget_name() call, which will retrieve the + * actual name. (The value passed in with the parameter \p size must + * be one greater than size in bytes of the actual name in order to + * accommodate the null terminator; if \p size is set to the exact + * size of the name, the last byte passed back will contain the null + * terminator and the last character will be missing from the name + * passed back to the calling application.) + * + * If an error occurs, the buffer pointed to by \p name is unchanged + * and the function returns a negative value. + * + * \since 1.6.3 + * + */ H5_DLL ssize_t H5Fget_name(hid_t obj_id, char *name, size_t size); -H5_DLL herr_t H5Fget_info2(hid_t obj_id, H5F_info2_t *finfo); +/** + * \ingroup H5F + * + * \brief Retrieves name of file to which object belongs + * + * \fgdta_obj_id + * \param[out] file_info Buffer for global file information + * + * \return \herr_t + * + * \details H5Fget_info2() returns global information for the file associated + * with the object identifier \p obj_id in the H5F_info2_t \c struct + * named \p file_info. + * + * \p obj_id is an identifier for any object in the file of interest. + * + * H5F_info2_t struct is defined in H5Fpublic.h as follows: + * \snippet this H5F_info2_t_snip + * + * The \c super sub-struct contains the following information: + * \li \c vers is the version number of the superblock. + * \li \c super_size is the size of the superblock. + * \li \c super_ext_size is the size of the superblock extension. + * + * The \c free sub-struct contains the following information: + * \li vers is the version number of the free-space manager. + * \li \c hdr_size is the size of the free-space manager header. + * \li \c tot_space is the total amount of free space in the file. + * + * The \c sohm sub-struct contains shared object header message + * information as follows: + * \li \c vers is the version number of the shared object header information. + * \li \c hdr_size is the size of the shared object header message. + * \li \c msgs_info is an H5_ih_info_t struct defined in H5public.h as + * follows: \snippet H5public.h H5_ih_info_t_snip + * \li \p index_size is the summed size of all the shared object + * header indexes. Each index might be either a B-tree or + * a list. + * \li \p heap_size is the size of the heap. + * + * + * \since 1.10.0 + * + */ +H5_DLL herr_t H5Fget_info2(hid_t obj_id, H5F_info2_t *file_info); +/** + * \ingroup SWMR + * + * \todo Finish this! + */ H5_DLL herr_t H5Fget_metadata_read_retry_info(hid_t file_id, H5F_retry_info_t *info); +/** + * \ingroup SWMR + * + * \todo Finish this! + */ H5_DLL herr_t H5Fstart_swmr_write(hid_t file_id); +/** + * \ingroup H5F + * + * \brief Retrieves free-space section information for a file + * + * \file_id + * \param[in] type The file memory allocation type + * \param[in] nsects The number of free-space sections + * \param[out] sect_info Array of instances of H5F_sect_info_t in which + * the free-space section information is to be returned + * + * \return Returns the number of free-space sections for the specified + * free-space manager in the file; otherwise returns a negative value. + * + * \details H5Fget_free_sections() retrieves free-space section information for + * the free-space manager with type that is associated with file + * \p file_id. If type is #H5FD_MEM_DEFAULT, this routine retrieves + * free-space section information for all the free-space managers in + * the file. + * + * Valid values for \p type are the following: + * \mem_types + * + * H5F_sect_info_t is defined as follows (in H5Fpublic.h): + * \snippet this H5F_sect_info_t_snip + * + * This routine retrieves free-space section information for \p nsects + * sections or at most the maximum number of sections in the specified + * free-space manager. If the number of sections is not known, a + * preliminary H5Fget_free_sections() call can be made by setting \p + * sect_info to NULL and the total number of free-space sections for + * the specified free-space manager will be returned. Users can then + * allocate space for entries in \p sect_info, each of which is + * defined as an H5F_sect_info_t \c struct. + * + * \attention \Bold{Failure Modes:} This routine will fail when the following + * is true: + * \li The library fails to retrieve the file creation property list + * associated with \p file_id. + * \li If the parameter \p sect_info is non-null, but the parameter + * \p nsects is equal to 0. + * \li The library fails to retrieve free-space section information + * for the file. + * + * \since 1.10.0 + * + */ H5_DLL ssize_t H5Fget_free_sections(hid_t file_id, H5F_mem_t type, size_t nsects, H5F_sect_info_t *sect_info /*out*/); +/** + * \ingroup H5F + * + * \brief Clears the external link open file cache + * + * \file_id + * \return \herr_t + * + * \details H5Fclear_elink_file_cache() evicts all the cached child files in + * the specified file’s external file cache, causing them to be closed + * if there is nothing else holding them open. + * + * H5Fclear_elink_file_cache() does not close the cache itself; + * subsequent external link traversals from the parent file will again + * cache the target file. See H5Pset_elink_file_cache_size() for + * information on closing the file cache. + * + * \see H5Pset_elink_file_cache_size(), H5Pget_elink_file_cache_size() + * + * \since 1.8.7 + * + */ H5_DLL herr_t H5Fclear_elink_file_cache(hid_t file_id); +/** + * \ingroup H5F + * + * \brief Enables the switch of version bounds setting for a file + * + * \file_id + * \param[in] low The earliest version of the library that will be used for + * writing objects + * \param[in] high The latest version of the library that will be used for + * writing objects + * + * \return \herr_t + * + * \details H5Fset_libver_bounds() enables the switch of version bounds setting + * for an open file associated with \p file_id. + * + * For the parameters \p low and \p high, see the description for + * H5Pset_libver_bounds(). + * + * \since 1.10.2 + * + */ H5_DLL herr_t H5Fset_libver_bounds(hid_t file_id, H5F_libver_t low, H5F_libver_t high); +/** + * \ingroup MDC + * + * \todo Finish this! + */ H5_DLL herr_t H5Fstart_mdc_logging(hid_t file_id); +/** + * \ingroup MDC + * + * \todo Finish this! + */ H5_DLL herr_t H5Fstop_mdc_logging(hid_t file_id); +/** + * \ingroup MDC + * + * \todo Finish this! + */ H5_DLL herr_t H5Fget_mdc_logging_status(hid_t file_id, /*OUT*/ hbool_t *is_enabled, /*OUT*/ hbool_t *is_currently_logging); +/** + * \ingroup SWMR + * + * \todo Finish this! + */ H5_DLL herr_t H5Fformat_convert(hid_t fid); +/** + * \ingroup H5F + * + * \brief Resets the page buffer statistics + * + * \file_id + * + * \return \herr_t + * + * \details H5Freset_page_buffering_stats() resets the page buffer statistics + * for a specified file identifier \p file_id. + * + * \since 1.10.1 + * + */ H5_DLL herr_t H5Freset_page_buffering_stats(hid_t file_id); +/** + * \ingroup H5F + * + * \brief Retrieves statistics about page access when it is enabled + * + * \file_id + * \param[out] accesses Two integer array for the number of metadata and raw + * data accesses to the page buffer + * \param[out] hits Two integer array for the number of metadata and raw data + * hits in the page buffer + * \param[out] misses Two integer array for the number of metadata and raw data + * misses in the page buffer + * \param[out] evictions Two integer array for the number of metadata and raw + * data evictions from the page buffer + * \param[out] bypasses Two integer array for the number of metadata and raw + * data accesses that bypass the page buffer + * + * \return \herr_t + * + * \details H5Fget_page_buffering_stats() retrieves page buffering statistics + * such as the number of metadata and raw data accesses (\p accesses), + * hits (\p hits), misses (\p misses), evictions (\p evictions), and + * accesses that bypass the page buffer (\p bypasses). + * + * \since 1.10.1 + * + */ H5_DLL herr_t H5Fget_page_buffering_stats(hid_t file_id, unsigned accesses[2], unsigned hits[2], unsigned misses[2], unsigned evictions[2], unsigned bypasses[2]); +/** + * \ingroup MDC + * + * \todo Finish this! + */ H5_DLL herr_t H5Fget_mdc_image_info(hid_t file_id, haddr_t *image_addr, hsize_t *image_size); +/** + * \ingroup H5F + * + * \brief Retrieves the setting for whether or not a file will create minimized + * dataset object headers + * + * \file_id + * \param[out] minimize Flag indicating whether the library will or will not + * create minimized dataset object headers + * + * \return \herr_t + * + * \details H5Fget_dset_no_attrs_hint() retrieves the no dataset attributes + * hint setting for the file specified by the file identifier \p + * file_id. This setting is used to inform the library to create + * minimized dataset object headers when \c TRUE. + * + * The setting's value is returned in the boolean pointer minimize. + * + * \since 1.10.5 + * + */ H5_DLL herr_t H5Fget_dset_no_attrs_hint(hid_t file_id, hbool_t *minimize); +/** + * \ingroup H5F + * + * \brief Sets the flag to create minimized dataset object headers + * + * \file_id + * \param[in] minimize Flag indicating whether the library will or will not + * create minimized dataset object headers + * + * \return \herr_t + * + * \details H5Fset_dset_no_attrs_hint() sets the no dataset attributes hint + * setting for the file specified by the file identifier \p file_id. + * If the boolean flag \p minimize is set to \c TRUE, then the library + * will create minimized dataset object headers in the file. + * \Bold{All} files that refer to the same file-on-disk will be + * affected by the most recent setting, regardless of the file + * identifier/handle (e.g., as returned by H5Fopen()). By setting the + * \p minimize flag to \c TRUE, the library expects that no attributes + * will be added to the dataset - attributes can be added, but they + * are appended with a continuation message, which can reduce + * performance. + * + * \attention This setting interacts with H5Pset_dset_no_attrs_hint(): if + * either is set to \c TRUE, then the created dataset's object header + * will be minimized. + * + * \since 1.10.5 + * + */ H5_DLL herr_t H5Fset_dset_no_attrs_hint(hid_t file_id, hbool_t minimize); #ifdef H5_HAVE_PARALLEL +/** + * \ingroup PH5F + * + * \todo Finish this! + */ H5_DLL herr_t H5Fset_mpi_atomicity(hid_t file_id, hbool_t flag); +/** + * \ingroup PH5F + * + * \todo Finish this! + */ H5_DLL herr_t H5Fget_mpi_atomicity(hid_t file_id, hbool_t *flag); #endif /* H5_HAVE_PARALLEL */ @@ -293,19 +1329,95 @@ H5_DLL herr_t H5Fget_mpi_atomicity(hid_t file_id, hbool_t *flag); /* Typedefs */ -/* Current "global" information about file */ +/** + * Current "global" information about file + */ +//! [H5F_info1_t_snip] typedef struct H5F_info1_t { - hsize_t super_ext_size; /* Superblock extension size */ + hsize_t super_ext_size; /**< Superblock extension size */ struct { - hsize_t hdr_size; /* Shared object header message header size */ - H5_ih_info_t msgs_info; /* Shared object header message index & heap size */ + hsize_t hdr_size; /**< Shared object header message header size */ + H5_ih_info_t msgs_info; /**< Shared object header message index & heap size */ } sohm; } H5F_info1_t; +//! [H5F_info1_t_snip] /* Function prototypes */ -H5_DLL herr_t H5Fget_info1(hid_t obj_id, H5F_info1_t *finfo); +/** + * \ingroup H5F + * + * \brief Retrieves name of file to which object belongs + * + * \fgdta_obj_id + * \param[out] file_info Buffer for global file information + * + * \return \herr_t + * + * \deprecated This function has been renamed from H5Fget_info() and is + * deprecated in favor of the macro #H5Fget_info or the function + * H5Fget_info2(). + * + * \details H5Fget_info1() returns global information for the file associated + * with the object identifier \p obj_id in the H5F_info1_t \c struct + * named \p file_info. + * + * \p obj_id is an identifier for any object in the file of interest. + * + * H5F_info1_t struct is defined in H5Fpublic.h as follows: + * \snippet this H5F_info1_t_snip + * + * \c super_ext_size is the size of the superblock extension. + * + * The \c sohm sub-struct contains shared object header message + * information as follows: + * \li \c hdr_size is the size of the shared object header message. + * \li \c msgs_info is an H5_ih_info_t struct defined in H5public.h as + * follows: \snippet H5public.h H5_ih_info_t_snip + * + * \li \p index_size is the summed size of all the shared object + * header indexes. Each index might be either a B-tree or + * a list. + * + * \version 1.10.0 C function H5Fget_info() renamed to H5Fget_info1() and + * deprecated in this release. + * + * \since 1.8.0 + * + */ +H5_DLL herr_t H5Fget_info1(hid_t obj_id, H5F_info1_t *file_info); +/** + * \ingroup H5F + * + * \brief Sets thelatest version of the library to be used for writing objects + * + * \file_id + * \param[in] latest_format Latest format flag + * + * \return \herr_t + * + * \deprecated When? + * + * \todo In which version was this function deprecated? + * + */ H5_DLL herr_t H5Fset_latest_format(hid_t file_id, hbool_t latest_format); -H5_DLL htri_t H5Fis_hdf5(const char *filename); +/** + * \ingroup H5F + * + * \brief Determines whether a file is in the HDF5 format + * + * \param[in] file_name File name + * + * \return \htri_t + * + * \deprecated When? + * + * \details H5Fis_hdf5() determines whether a file is in the HDF5 format. + * + * \todo In which version was this function deprecated? + * + */ +H5_DLL htri_t H5Fis_hdf5(const char *file_name); #endif /* H5_NO_DEPRECATED_SYMBOLS */ diff --git a/src/H5Fquery.c b/src/H5Fquery.c index ed435a8..dac6e66 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -1299,3 +1299,23 @@ H5F__get_cont_info(const H5F_t *f, H5VL_file_cont_info_t *info) done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F_get_cont_info */ + +/*------------------------------------------------------------------------- + * Function: H5F_get_file_locking + * + * Purpose: Get the file locking flag for the file + * + * Return: TRUE/FALSE + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_get_file_locking(const H5F_t *f) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->use_file_locking) +} /* end H5F_get_file_locking */ diff --git a/src/H5Ftest.c b/src/H5Ftest.c index cc7ed1b..c2ce16e 100644 --- a/src/H5Ftest.c +++ b/src/H5Ftest.c @@ -253,3 +253,35 @@ H5F__same_file_test(hid_t file_id1, hid_t file_id2) done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F__same_file_test() */ + +/*------------------------------------------------------------------------- + * Function: H5F__reparse_file_lock_variable_test + * + * Purpose: Re-parse the file locking environment variable. + * + * Since getenv(3) is fairly expensive, we only parse it once, + * when the library opens. This test function is used to + * re-parse the environment variable after we've changed it + * with setnev(3). + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Summer 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F__reparse_file_lock_variable_test(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check the file locking environment variable */ + if (H5F__parse_file_lock_env_var(&use_locks_env_g) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to parse file locking environment variable") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__reparse_file_lock_variable_test() */ diff --git a/src/H5Osdspace.c b/src/H5Osdspace.c index b34eb76..83702eb 100644 --- a/src/H5Osdspace.c +++ b/src/H5Osdspace.c @@ -138,8 +138,11 @@ H5O__sdspace_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UN flags = *p++; /* Get or determine the type of the extent */ - if (version >= H5O_SDSPACE_VERSION_2) + if (version >= H5O_SDSPACE_VERSION_2) { sdim->type = (H5S_class_t)*p++; + if (sdim->type != H5S_SIMPLE && sdim->rank > 0) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "invalid rank for scalar or NULL dataspace") + } /* end if */ else { /* Set the dataspace type to be simple or scalar as appropriate */ if (sdim->rank > 0) @@ -248,13 +251,15 @@ H5O__sdspace_encode(H5F_t *f, uint8_t *p, const void *_mesg) *p++ = 0; /*reserved*/ } /* end else */ - /* Current & maximum dimensions */ - if (sdim->rank > 0) { - for (u = 0; u < sdim->rank; u++) - H5F_ENCODE_LENGTH(f, p, sdim->size[u]); - if (flags & H5S_VALID_MAX) { + /* Encode dataspace dimensions for simple dataspaces */ + if (H5S_SIMPLE == sdim->type) { + /* Encode current & maximum dimensions */ + if (sdim->rank > 0) { for (u = 0; u < sdim->rank; u++) - H5F_ENCODE_LENGTH(f, p, sdim->max[u]); + H5F_ENCODE_LENGTH(f, p, sdim->size[u]); + if (flags & H5S_VALID_MAX) + for (u = 0; u < sdim->rank; u++) + H5F_ENCODE_LENGTH(f, p, sdim->max[u]); } /* end if */ } /* end if */ diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 7588b4a..f9a730d 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -288,6 +288,29 @@ #define H5F_ACS_VOL_CONN_COPY H5P__facc_vol_copy #define H5F_ACS_VOL_CONN_CMP H5P__facc_vol_cmp #define H5F_ACS_VOL_CONN_CLOSE H5P__facc_vol_close +/* Definition for using file locking or not. The default is set + * via the configure step. + */ +#define H5F_ACS_USE_FILE_LOCKING_SIZE sizeof(hbool_t) +#if defined H5_USE_FILE_LOCKING && H5_USE_FILE_LOCKING +#define H5F_ACS_USE_FILE_LOCKING_DEF TRUE +#else +#define H5F_ACS_USE_FILE_LOCKING_DEF FALSE +#endif +#define H5F_ACS_USE_FILE_LOCKING_ENC H5P__encode_hbool_t +#define H5F_ACS_USE_FILE_LOCKING_DEC H5P__decode_hbool_t +/* Definition for whether we ignore file locking errors when we can + * tell that file locking has been disabled on the file system. + * The default is set via the configure step. + */ +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_SIZE sizeof(hbool_t) +#if defined H5_IGNORE_DISABLED_FILE_LOCKS && H5_IGNORE_DISABLED_FILE_LOCKS +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF TRUE +#else +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF FALSE +#endif +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_ENC H5P__encode_hbool_t +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEC H5P__decode_hbool_t /******************/ /* Local Typedefs */ @@ -478,6 +501,10 @@ static const unsigned H5F_def_page_buf_min_meta_perc_g = H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEF; /* Default page buffer minimum metadata size */ static const unsigned H5F_def_page_buf_min_raw_perc_g = H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF; /* Default page buffer mininum raw data size */ +static const hbool_t H5F_def_use_file_locking_g = + H5F_ACS_USE_FILE_LOCKING_DEF; /* Default use file locking flag */ +static const hbool_t H5F_def_ignore_disabled_file_locks_g = + H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF; /* Default ignore disabled file locks flag */ /*------------------------------------------------------------------------- * Function: H5P__facc_reg_prop @@ -764,6 +791,19 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) H5F_ACS_VOL_CONN_CLOSE) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the use file locking flag */ + if (H5P__register_real(pclass, H5F_ACS_USE_FILE_LOCKING_NAME, H5F_ACS_USE_FILE_LOCKING_SIZE, + &H5F_def_use_file_locking_g, NULL, NULL, NULL, H5F_ACS_USE_FILE_LOCKING_ENC, + H5F_ACS_USE_FILE_LOCKING_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the ignore disabled file locks flag */ + if (H5P__register_real(pclass, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, + H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_SIZE, &H5F_def_ignore_disabled_file_locks_g, + NULL, NULL, NULL, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_ENC, + H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__facc_reg_prop() */ @@ -4566,6 +4606,96 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Pget_evict_on_close() */ +/*------------------------------------------------------------------------- + * Function: H5Pset_file_locking + * + * Purpose: Sets the file locking property values. + * + * Overrides the default file locking flag setting that was + * set when the library was configured. + * + * Can be overridden by the HDF5_USE_FILE_LOCKING environment + * variable. + * + * File locking is used when creating/opening a file to prevent + * problematic file accesses. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Spring 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_file_locking(hid_t fapl_id, hbool_t use_file_locking, hbool_t ignore_when_disabled) +{ + H5P_genplist_t *plist; /* property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "ibb", fapl_id, use_file_locking, ignore_when_disabled); + + /* Make sure this is a fapl */ + if (TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not a file access plist") + + /* Get the plist structure */ + if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Set values */ + if (H5P_set(plist, H5F_ACS_USE_FILE_LOCKING_NAME, &use_file_locking) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set use file locking property") + if (H5P_set(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &ignore_when_disabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set ignore disabled file locks property") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_file_locking() */ + +/*------------------------------------------------------------------------- + * Function: H5Pget_file_locking + * + * Purpose: Gets the file locking property values. + * + * File locking is used when creating/opening a file to prevent + * problematic file accesses. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Spring 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_file_locking(hid_t fapl_id, hbool_t *use_file_locking, hbool_t *ignore_when_disabled) +{ + H5P_genplist_t *plist; /* property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "i*b*b", fapl_id, use_file_locking, ignore_when_disabled); + + /* Make sure this is a fapl */ + if (TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not an access plist") + + /* Get the plist structure */ + if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get values */ + if (H5P_get(plist, H5F_ACS_USE_FILE_LOCKING_NAME, use_file_locking) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get use file locking property") + if (H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, ignore_when_disabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get ignore disabled file locks property") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_file_locking() */ + #ifdef H5_HAVE_PARALLEL /*------------------------------------------------------------------------- diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 68519f8..ffcdd99 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -361,6 +361,8 @@ H5_DLL herr_t H5Pget_mdc_log_options(hid_t plist_id, hbool_t *is_enabled, c size_t *location_size, hbool_t *start_on_access); H5_DLL herr_t H5Pset_evict_on_close(hid_t fapl_id, hbool_t evict_on_close); H5_DLL herr_t H5Pget_evict_on_close(hid_t fapl_id, hbool_t *evict_on_close); +H5_DLL herr_t H5Pset_file_locking(hid_t fapl_id, hbool_t use_file_locking, hbool_t ignore_when_disabled); +H5_DLL herr_t H5Pget_file_locking(hid_t fapl_id, hbool_t *use_file_locking, hbool_t *ignore_when_disabled); #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5Pset_all_coll_metadata_ops(hid_t plist_id, hbool_t is_collective); H5_DLL herr_t H5Pget_all_coll_metadata_ops(hid_t plist_id, hbool_t *is_collective); diff --git a/src/H5Rint.c b/src/H5Rint.c index 8f3163a..60f8131 100644 --- a/src/H5Rint.c +++ b/src/H5Rint.c @@ -746,8 +746,10 @@ H5R__copy(const H5R_ref_priv_t *src_ref, H5R_ref_priv_t *dst_ref) dst_ref->info.obj.filename = NULL; /* Set location ID and hold reference to it */ - if (H5R__set_loc_id(dst_ref, src_ref->loc_id, TRUE, TRUE) < 0) - HGOTO_ERROR(H5E_REFERENCE, H5E_CANTSET, FAIL, "cannot set reference location ID") + dst_ref->loc_id = src_ref->loc_id; + if (H5I_inc_ref(dst_ref->loc_id, TRUE) < 0) + HGOTO_ERROR(H5E_REFERENCE, H5E_CANTINC, FAIL, "incrementing location ID failed") + dst_ref->app_ref = TRUE; } done: @@ -266,7 +266,7 @@ H5SM__type_to_flag(unsigned type_id, unsigned *type_flag) switch (type_id) { case H5O_FILL_ID: type_id = H5O_FILL_NEW_ID; - /* Fall through... */ + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case H5O_SDSPACE_ID: diff --git a/src/H5Shyper.c b/src/H5Shyper.c index 56a8b14..c527cd5 100644 --- a/src/H5Shyper.c +++ b/src/H5Shyper.c @@ -2137,24 +2137,31 @@ H5S__hyper_iter_get_seq_list_opt(H5S_sel_iter_t *iter, size_t maxseq, size_t max case 0: do { DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 7: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 6: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 5: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 4: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 3: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 2: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 1: DUFF_GUTS @@ -10100,7 +10107,7 @@ H5S_select_hyperslab(H5S_t *space, H5S_seloper_t op, const hsize_t start[], cons case H5S_SEL_POINTS: /* Can't combine hyperslab operations and point selections currently */ if (op == H5S_SELECT_SET) /* Allow only "set" operation to proceed */ break; - /* Else fall through to error */ + /* FALLTHROUGH (to error) */ H5_ATTR_FALLTHROUGH case H5S_SEL_ERROR: diff --git a/src/H5Spublic.h b/src/H5Spublic.h index ad0bbd5..f0aa6dc 100644 --- a/src/H5Spublic.h +++ b/src/H5Spublic.h @@ -172,6 +172,7 @@ H5_DLL hid_t H5Sselect_project_intersection(hid_t src_space_id, hid_t dst_spa H5_DLL hid_t H5Ssel_iter_create(hid_t spaceid, size_t elmt_size, unsigned flags); H5_DLL herr_t H5Ssel_iter_get_seq_list(hid_t sel_iter_id, size_t maxseq, size_t maxbytes, size_t *nseq, size_t *nbytes, hsize_t *off, size_t *len); +H5_DLL herr_t H5Ssel_iter_reset(hid_t sel_iter_id, hid_t space_id); H5_DLL herr_t H5Ssel_iter_close(hid_t sel_iter_id); /* Symbols defined for compatibility with previous versions of the HDF5 API. diff --git a/src/H5Sselect.c b/src/H5Sselect.c index 37a065a..023ab80 100644 --- a/src/H5Sselect.c +++ b/src/H5Sselect.c @@ -3089,6 +3089,56 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Ssel_iter_get_seq_list() */ +/*-------------------------------------------------------------------------- + NAME + H5Ssel_iter_reset + PURPOSE + Resets a dataspace selection iterator back to an initial state. + USAGE + herr_t H5Ssel_iter_reset(sel_iter_id) + hid_t sel_iter_id; IN: ID of the dataspace selection iterator to + reset + hid_t space_id; IN: ID of the dataspace with selection to + iterate over + RETURNS + Non-negative on success / Negative on failure + DESCRIPTION + Resets a dataspace selection iterator back to an initial state so that + the iterator may be used for iteration once again. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5Ssel_iter_reset(hid_t sel_iter_id, hid_t space_id) +{ + H5S_sel_iter_t *sel_iter; + H5S_t * space; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ii", sel_iter_id, space_id); + + /* Check args */ + if (NULL == (sel_iter = (H5S_sel_iter_t *)H5I_object_verify(sel_iter_id, H5I_SPACE_SEL_ITER))) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "not a dataspace selection iterator") + if (NULL == (space = (H5S_t *)H5I_object_verify(space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "not a dataspace") + + /* Call selection type-specific release routine */ + if (H5S_SELECT_ITER_RELEASE(sel_iter) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, FAIL, + "problem releasing a selection iterator's type-specific info") + + /* Simply re-initialize iterator */ + if (H5S_select_iter_init(sel_iter, space, sel_iter->elmt_size, sel_iter->flags) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to re-initialize selection iterator") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Ssel_iter_reset() */ + /*------------------------------------------------------------------------- * Function: H5S_sel_iter_close * diff --git a/src/H5Tdbg.c b/src/H5Tdbg.c index 5d0c5cd..758430b 100644 --- a/src/H5Tdbg.c +++ b/src/H5Tdbg.c @@ -120,9 +120,9 @@ H5T__print_stats(H5T_path_t H5_ATTR_UNUSED *path, int H5_ATTR_UNUSED *nprint /*i HDfprintf(H5DEBUG(T), " %-16s %10" PRIdHSIZE " %10u %8s %8s %8s %10s\n", path->name, path->stats.nelmts, path->stats.ncalls, timestrs.user, timestrs.system, timestrs.elapsed, bandwidth); - free(timestrs.user); - free(timestrs.system); - free(timestrs.elapsed); + HDfree(timestrs.user); + HDfree(timestrs.system); + HDfree(timestrs.elapsed); } #endif diff --git a/src/H5Tref.c b/src/H5Tref.c index 7092a71..c71724f 100644 --- a/src/H5Tref.c +++ b/src/H5Tref.c @@ -513,7 +513,7 @@ H5T__ref_mem_read(H5VL_object_t H5_ATTR_UNUSED *src_file, const void *src_buf, s /* Memory-to-memory conversion to support vlen conversion */ if (NULL == dst_file) { - HDmemcpy(dst_buf, src_buf, dst_size); + H5MM_memcpy(dst_buf, src_buf, dst_size); HGOTO_DONE(ret_value); } @@ -602,7 +602,7 @@ H5T__ref_mem_write(H5VL_object_t *src_file, const void *src_buf, size_t src_size /* Memory-to-memory conversion to support vlen conversion */ if (NULL == src_file) { - HDmemcpy(dst_buf, src_buf, src_size); + H5MM_memcpy(dst_buf, src_buf, src_size); HGOTO_DONE(ret_value); } @@ -650,6 +650,7 @@ H5T__ref_mem_write(H5VL_object_t *src_file, const void *src_buf, size_t src_size /* Pass the correct encoding version for the selection depending on the * file libver bounds, this is later retrieved in H5S hyper decode */ H5CX_set_libver_bounds(src_f); + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case H5R_OBJECT2: case H5R_ATTR: diff --git a/src/H5VLnative_object.c b/src/H5VLnative_object.c index ac717b0..81ff5ea 100644 --- a/src/H5VLnative_object.c +++ b/src/H5VLnative_object.c @@ -338,7 +338,7 @@ H5VL__native_object_specific(void *obj, const H5VL_loc_params_t *loc_params, /* Lookup object */ case H5VL_OBJECT_LOOKUP: { - H5O_token_t *token = va_arg(arguments, H5O_token_t *); + H5O_token_t *token = HDva_arg(arguments, H5O_token_t *); HDassert(token); @@ -175,9 +175,9 @@ H5Z_term_package(void) H5Z_stat_table_g[i].stats[dir].errors, timestrs.user, timestrs.system, timestrs.elapsed, bandwidth); next: - free(timestrs.user); - free(timestrs.system); - free(timestrs.elapsed); + HDfree(timestrs.user); + HDfree(timestrs.system); + HDfree(timestrs.elapsed); } /* end for */ } /* end for */ } /* end if */ @@ -990,6 +990,70 @@ done: } /* end H5Z_set_local_direct() */ /*------------------------------------------------------------------------- + * Function: H5Z_ignore_filters + * + * Purpose: Determine whether filters can be ignored. + * + * Description: + * When the filters are optional (i.e., H5Z_FLAG_OPTIONAL is provided,) + * if any of the following conditions is met, the filters will be ignored: + * - dataspace is either H5S_NULL or H5S_SCALAR + * - datatype is variable-length (string or non-string) + * However, if any of these conditions exists and a filter is not + * optional, the function will produce an error. + * + * Return: Non-negative(TRUE/FALSE) on success + * Negative on failure + * + *------------------------------------------------------------------------- + */ +htri_t +H5Z_ignore_filters(hid_t dcpl_id, const H5T_t *type, const H5S_t *space) +{ + H5P_genplist_t *dc_plist; /* Dataset creation property list object */ + H5O_pline_t pline; /* Object's I/O pipeline information */ + H5S_class_t space_class; /* To check class of space */ + H5T_class_t type_class; /* To check if type is VL */ + bool bad_for_filters = FALSE; /* Suitable to have filters */ + htri_t ret_value = FALSE; /* TRUE for ignoring filters */ + + FUNC_ENTER_NOAPI(FAIL) + + if (NULL == (dc_plist = (H5P_genplist_t *)H5I_object(dcpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get dataset creation property list") + + /* Get pipeline information */ + if (H5P_peek(dc_plist, H5O_CRT_PIPELINE_NAME, &pline) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't retrieve pipeline filter") + + /* Get datatype and dataspace classes for quick access */ + space_class = H5S_GET_EXTENT_TYPE(space); + type_class = H5T_get_class(type, FALSE); + + /* These conditions are not suitable for filters */ + bad_for_filters = (H5S_NULL == space_class || H5S_SCALAR == space_class || H5T_VLEN == type_class || + (H5T_STRING == type_class && TRUE == H5T_is_variable_str(type))); + + /* When these conditions occur, if there are required filters in pline, + then report a failure, otherwise, set flag that they can be ignored */ + if (bad_for_filters) { + size_t ii; + if (pline.nused > 0) { + for (ii = 0; ii < pline.nused; ii++) { + if (!(pline.filter[ii].flags & H5Z_FLAG_OPTIONAL)) + HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "not suitable for filters") + } + + /* All filters are optional, we can ignore them */ + ret_value = TRUE; + } + } /* bad for filters */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_ignore_filters() */ + +/*------------------------------------------------------------------------- * Function: H5Z_modify * * Purpose: Modify filter parameters for specified pipeline. diff --git a/src/H5Zprivate.h b/src/H5Zprivate.h index 66054bf..21d05b0 100644 --- a/src/H5Zprivate.h +++ b/src/H5Zprivate.h @@ -26,6 +26,7 @@ typedef struct H5Z_filter_info_t H5Z_filter_info_t; /* Private headers needed by this file */ #include "H5Tprivate.h" /* Datatypes */ +#include "H5Sprivate.h" /* Dataspace */ /**************************/ /* Library Private Macros */ @@ -86,6 +87,7 @@ H5_DLL herr_t H5Z_can_apply(hid_t dcpl_id, hid_t type_id); H5_DLL herr_t H5Z_set_local(hid_t dcpl_id, hid_t type_id); H5_DLL herr_t H5Z_can_apply_direct(const struct H5O_pline_t *pline); H5_DLL herr_t H5Z_set_local_direct(const struct H5O_pline_t *pline); +H5_DLL htri_t H5Z_ignore_filters(hid_t dcpl_id, const H5T_t *type, const H5S_t *space); H5_DLL H5Z_filter_info_t *H5Z_filter_info(const struct H5O_pline_t *pline, H5Z_filter_t filter); H5_DLL htri_t H5Z_filter_in_pline(const struct H5O_pline_t *pline, H5Z_filter_t filter); H5_DLL htri_t H5Z_all_filters_avail(const struct H5O_pline_t *pline); diff --git a/src/H5Zshuffle.c b/src/H5Zshuffle.c index bd28f84..2e3ff97 100644 --- a/src/H5Zshuffle.c +++ b/src/H5Zshuffle.c @@ -177,24 +177,31 @@ H5Z__filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[] case 0: do { DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 7: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 6: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 5: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 4: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 3: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 2: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 1: DUFF_GUTS @@ -241,24 +248,31 @@ H5Z__filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[] case 0: do { DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 7: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 6: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 5: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 4: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 3: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 2: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 1: DUFF_GUTS diff --git a/src/H5checksum.c b/src/H5checksum.c index 7ae588e..bc36bb9 100644 --- a/src/H5checksum.c +++ b/src/H5checksum.c @@ -414,36 +414,47 @@ H5_checksum_lookup3(const void *key, size_t length, uint32_t initval) { case 12: c += ((uint32_t)k[11]) << 24; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 11: c += ((uint32_t)k[10]) << 16; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 10: c += ((uint32_t)k[9]) << 8; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 9: c += k[8]; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 8: b += ((uint32_t)k[7]) << 24; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 7: b += ((uint32_t)k[6]) << 16; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 6: b += ((uint32_t)k[5]) << 8; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 5: b += k[4]; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 4: a += ((uint32_t)k[3]) << 24; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 3: a += ((uint32_t)k[2]) << 16; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 2: a += ((uint32_t)k[1]) << 8; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 1: a += k[0]; diff --git a/src/H5err.txt b/src/H5err.txt index 9fec521..24ac2ac 100644 --- a/src/H5err.txt +++ b/src/H5err.txt @@ -136,6 +136,8 @@ MINOR, FILEACC, H5E_BADFILE, Bad file ID accessed MINOR, FILEACC, H5E_TRUNCATED, File has been truncated MINOR, FILEACC, H5E_MOUNT, File mount error MINOR, FILEACC, H5E_CANTDELETEFILE, Unable to delete file +MINOR, FILEACC, H5E_CANTLOCKFILE, Unable to lock file +MINOR, FILEACC, H5E_CANTUNLOCKFILE, Unable to unlock file # Generic low-level file I/O errors MINOR, FILE, H5E_SEEKERROR, Seek failed diff --git a/src/H5mpi.c b/src/H5mpi.c index c1f9132..613a4bf 100644 --- a/src/H5mpi.c +++ b/src/H5mpi.c @@ -25,7 +25,7 @@ /****************/ /* Local Macros */ /****************/ -#define TWO_GIG_LIMIT (1 << 31) +#define TWO_GIG_LIMIT INT32_MAX #ifndef H5_MAX_MPI_COUNT #define H5_MAX_MPI_COUNT (1 << 30) #endif @@ -33,7 +33,7 @@ /*******************/ /* Local Variables */ /*******************/ -static hsize_t bigio_count = H5_MAX_MPI_COUNT; +static hsize_t bigio_count_g = H5_MAX_MPI_COUNT; /*------------------------------------------------------------------------- * Function: H5_mpi_set_bigio_count @@ -42,7 +42,7 @@ static hsize_t bigio_count = H5_MAX_MPI_COUNT; * when we utilize derived datatypes. This is of * particular interest for allowing nightly testing * - * Return: The current/previous value of bigio_count. + * Return: The current/previous value of bigio_count_g. * * Programmer: Richard Warren, March 10, 2017 * @@ -51,10 +51,10 @@ static hsize_t bigio_count = H5_MAX_MPI_COUNT; hsize_t H5_mpi_set_bigio_count(hsize_t new_count) { - hsize_t orig_count = bigio_count; + hsize_t orig_count = bigio_count_g; if ((new_count > 0) && (new_count < (hsize_t)TWO_GIG_LIMIT)) { - bigio_count = new_count; + bigio_count_g = new_count; } return orig_count; } /* end H5_mpi_set_bigio_count() */ @@ -63,9 +63,9 @@ H5_mpi_set_bigio_count(hsize_t new_count) * Function: H5_mpi_get_bigio_count * * Purpose: Allow other HDF5 library functions to access - * the current value for bigio_count. + * the current value for bigio_count_g. * - * Return: The current/previous value of bigio_count. + * Return: The current/previous value of bigio_count_g. * * Programmer: Richard Warren, October 7, 2019 * @@ -74,7 +74,7 @@ H5_mpi_set_bigio_count(hsize_t new_count) hsize_t H5_mpi_get_bigio_count(void) { - return bigio_count; + return bigio_count_g; } /*------------------------------------------------------------------------- @@ -471,8 +471,8 @@ H5_mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes, MPI_Datat FUNC_ENTER_NOAPI(FAIL) /* Calculate how many Big MPI datatypes are needed to represent the buffer */ - num_big_types = (int)(num_elements / bigio_count); - leftover = (hsize_t)num_elements - (hsize_t)num_big_types * bigio_count; + num_big_types = (int)(num_elements / bigio_count_g); + leftover = (hsize_t)num_elements - (hsize_t)num_big_types * bigio_count_g; H5_CHECKED_ASSIGN(remaining_bytes, int, leftover, hsize_t); /* Create a contiguous datatype of size equal to the largest @@ -481,11 +481,11 @@ H5_mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes, MPI_Datat * use type_hvector to create the type with the displacement provided */ if (0 == stride_bytes) { - if (MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)bigio_count, old_type, &inner_type))) + if (MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)bigio_count_g, old_type, &inner_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) } /* end if */ else if (MPI_SUCCESS != - (mpi_code = MPI_Type_create_hvector((int)bigio_count, 1, stride_bytes, old_type, &inner_type))) + (mpi_code = MPI_Type_create_hvector((int)bigio_count_g, 1, stride_bytes, old_type, &inner_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code) /* Create a contiguous datatype of the buffer (minus the remaining < 2GB part) @@ -510,7 +510,7 @@ H5_mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes, MPI_Datat HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) } /* end if */ else if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hvector( - (int)(num_elements - (hsize_t)num_big_types * bigio_count), 1, + (int)(num_elements - (hsize_t)num_big_types * bigio_count_g), 1, stride_bytes, old_type, &leftover_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code) @@ -529,7 +529,7 @@ H5_mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes, MPI_Datat block_len[0] = 1; block_len[1] = 1; disp[0] = 0; - disp[1] = (old_extent + stride_bytes) * num_big_types * (MPI_Aint)bigio_count; + disp[1] = (old_extent + stride_bytes) * num_big_types * (MPI_Aint)bigio_count_g; if (MPI_SUCCESS != (mpi_code = MPI_Type_create_struct(2, block_len, disp, type, new_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) diff --git a/src/H5private.h b/src/H5private.h index ca005f6..1390b3f 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -298,6 +298,10 @@ * Note that Solaris Studio supports attribute, but does not support the * attributes we use. * + * When using H5_ATTR_FALLTHROUGH, you should also include a comment that + * says FALLTHROUGH to reduce warnings on compilers that don't use + * attributes but do respect fall-through comments. + * * H5_ATTR_CONST is redefined in tools/h5repack/dynlib_rpk.c to quiet * gcc warnings (it has to use the public API and can't include this * file). Be sure to update that file if the #ifdefs change here. diff --git a/src/H5public.h b/src/H5public.h index 2de4459..29a78c6 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -92,11 +92,11 @@ extern "C" { #endif /* Version numbers */ -#define H5_VERS_MAJOR 1 /* For major interface/format changes */ -#define H5_VERS_MINOR 12 /* For minor interface/format changes */ -#define H5_VERS_RELEASE 1 /* For tweaks, bug-fixes, or development */ -#define H5_VERS_SUBRELEASE "3" /* For pre-releases like snap0 */ -/* Empty string for real releases. */ +#define H5_VERS_MAJOR 1 /* For major interface/format changes */ +#define H5_VERS_MINOR 12 /* For minor interface/format changes */ +#define H5_VERS_RELEASE 1 /* For tweaks, bug-fixes, or development */ +#define H5_VERS_SUBRELEASE "3" /* For pre-releases like snap0 */ + /* Empty string for real releases. */ #define H5_VERS_INFO "HDF5 library version: 1.12.1-3" /* Full version string */ #define H5check() H5check_version(H5_VERS_MAJOR, H5_VERS_MINOR, H5_VERS_RELEASE) @@ -333,13 +333,15 @@ typedef enum H5_index_t { H5_INDEX_N /* Number of indices defined */ } H5_index_t; -/* +/** * Storage info struct used by H5O_info_t and H5F_info_t */ +//! [H5_ih_info_t_snip] typedef struct H5_ih_info_t { - hsize_t index_size; /* btree and/or list */ + hsize_t index_size; /**< btree and/or list */ hsize_t heap_size; } H5_ih_info_t; +//! [H5_ih_info_t_snip] /* Tokens are unique and permanent identifiers that are * used to reference HDF5 objects in a container. */ diff --git a/src/Makefile.am b/src/Makefile.am index 118c0a3..4798627 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -41,7 +41,7 @@ DISTCLEANFILES=H5pubconf.h # library sources libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5lib_settings.c H5system.c \ - H5timer.c H5trace.c \ + H5timer.c H5trace.c \ H5A.c H5Abtree2.c H5Adense.c H5Adeprec.c H5Aint.c H5Atest.c \ H5AC.c H5ACdbg.c H5ACproxy_entry.c \ H5B.c H5Bcache.c H5Bdbg.c \ diff --git a/src/libhdf5.settings.in b/src/libhdf5.settings.in index fb9deec..98390b8 100644 --- a/src/libhdf5.settings.in +++ b/src/libhdf5.settings.in @@ -89,5 +89,6 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ Using memory checker: @USINGMEMCHECKER@ Memory allocation sanity checks: @MEMORYALLOCSANITYCHECK@ Function stack tracing: @CODESTACK@ + Use file locking: @DESIRED_FILE_LOCKING@ Strict file format checks: @STRICT_FORMAT_CHECKS@ Optimization instrumentation: @INSTRUMENT_LIBRARY@ |