From afd8f64c29f7cffde895232406a3a388c6d77e77 Mon Sep 17 00:00:00 2001 From: mainzer Date: Tue, 28 Mar 2017 11:21:40 -0500 Subject: Omnibus checkin for several relatively minor modifications: 1) Added code test/page_buffer.c to verify that page buffering is disabled in parallel builds. 2) Added code to test/cache_image.c to verify correct interaction between evict on close and cache image -- in particular management of a file containing a cache image containing dirty metadata that has been opened R/O. Also fix for the bug exposed. 3) Added code to testpar/t_cache_image.c to verify expected procedure for reading cache images, and also supporting stats collection code needed for the test. 4) Repair of an overactive sanity check in H5C__reconstruct_cache_contents(). 5) Other minor tidies in passing. Tested serial and parallel, debug and production on Jelly. --- src/H5Cdbg.c | 4 +- src/H5Cimage.c | 73 +++++--- src/H5Cpkg.h | 21 +++ src/H5Ctag.c | 47 ++++- test/cache_image.c | 467 +++++++++++++++++++++++++++++++++++++++++++++++- test/page_buffer.c | 135 +++++++++++++- testpar/t_cache_image.c | 217 ++++++++++++++++++---- 7 files changed, 886 insertions(+), 78 deletions(-) diff --git a/src/H5Cdbg.c b/src/H5Cdbg.c index a955eaf..0a98406 100644 --- a/src/H5Cdbg.c +++ b/src/H5Cdbg.c @@ -738,9 +738,10 @@ H5C_stats(H5C_t * cache_ptr, (long long)(cache_ptr->index_scan_restarts)); HDfprintf(stdout, - "%s cache image creations/loads/size = %d / %d / %Hu\n", + "%s cache image creations/reads/loads/size = %d / %d /%d / %Hu\n", cache_ptr->prefix, cache_ptr->images_created, + cache_ptr->images_read, cache_ptr->images_loaded, cache_ptr->last_image_size); @@ -993,6 +994,7 @@ H5C_stats__reset(H5C_t H5_ATTR_UNUSED * cache_ptr) cache_ptr->index_scan_restarts = 0; cache_ptr->images_created = 0; + cache_ptr->images_read = 0; cache_ptr->images_loaded = 0; cache_ptr->last_image_size = (hsize_t)0; diff --git a/src/H5Cimage.c b/src/H5Cimage.c index fc58dac..d60ee05 100644 --- a/src/H5Cimage.c +++ b/src/H5Cimage.c @@ -122,7 +122,7 @@ static H5C_cache_entry_t *H5C__reconstruct_cache_entry(const H5F_t *f, H5C_t *cache_ptr, const uint8_t **buf); static herr_t H5C__write_cache_image_superblock_msg(H5F_t *f, hid_t dxpl_id, hbool_t create); -static herr_t H5C__read_cache_image(H5F_t * f, hid_t dxpl_id, const H5C_t *cache_ptr); +static herr_t H5C__read_cache_image(H5F_t * f, hid_t dxpl_id, H5C_t *cache_ptr); static herr_t H5C__write_cache_image(H5F_t *f, hid_t dxpl_id, const H5C_t *cache_ptr); static herr_t H5C__construct_cache_image_buffer(H5F_t *f, H5C_t *cache_ptr); static herr_t H5C__free_image_entries_array(H5C_t *cache_ptr); @@ -1035,7 +1035,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5C__read_cache_image(H5F_t *f, hid_t dxpl_id, const H5C_t *cache_ptr) +H5C__read_cache_image(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr) { herr_t ret_value = SUCCEED; /* Return value */ @@ -1053,31 +1053,53 @@ H5C__read_cache_image(H5F_t *f, hid_t dxpl_id, const H5C_t *cache_ptr) H5AC_aux_t *aux_ptr = (H5AC_aux_t *)cache_ptr->aux_ptr; int mpi_result; - if((NULL == aux_ptr) || (aux_ptr->mpi_rank == 0)) { - HDassert((NULL == aux_ptr) || (aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC)); + if ( ( NULL == aux_ptr ) || ( aux_ptr->mpi_rank == 0 ) ) { + + HDassert((NULL == aux_ptr) || + (aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC)); #endif /* H5_HAVE_PARALLEL */ /* Read the buffer (if serial access, or rank 0 of parallel access) */ - if(H5F_block_read(f, H5FD_MEM_SUPER, cache_ptr->image_addr, cache_ptr->image_len, dxpl_id, cache_ptr->image_buffer) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_READERROR, FAIL, "Can't read metadata cache image block") + if ( H5F_block_read(f, H5FD_MEM_SUPER, cache_ptr->image_addr, + cache_ptr->image_len, dxpl_id, + cache_ptr->image_buffer) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_READERROR, FAIL, \ + "Can't read metadata cache image block") + + H5C__UPDATE_STATS_FOR_CACHE_IMAGE_READ(cache_ptr) #ifdef H5_HAVE_PARALLEL - if(aux_ptr) { + if ( aux_ptr ) { + /* Broadcast cache image */ - if(MPI_SUCCESS != (mpi_result = MPI_Bcast(cache_ptr->image_buffer, (int)cache_ptr->image_len, MPI_BYTE, 0, aux_ptr->mpi_comm))) + if ( MPI_SUCCESS != + (mpi_result = MPI_Bcast(cache_ptr->image_buffer, + (int)cache_ptr->image_len, MPI_BYTE, + 0, aux_ptr->mpi_comm)) ) + HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_result) + } /* end if */ } /* end if */ - else if(aux_ptr) { + else if ( aux_ptr ) { + /* Retrieve the contents of the metadata cache image from process 0 */ - if(MPI_SUCCESS != (mpi_result = MPI_Bcast(cache_ptr->image_buffer, (int)cache_ptr->image_len, MPI_BYTE, 0, aux_ptr->mpi_comm))) - HMPI_GOTO_ERROR(FAIL, "can't receive cache image MPI_Bcast", mpi_result) + if ( MPI_SUCCESS != + (mpi_result = MPI_Bcast(cache_ptr->image_buffer, + (int)cache_ptr->image_len, MPI_BYTE, + 0, aux_ptr->mpi_comm)) ) + + HMPI_GOTO_ERROR(FAIL, "can't receive cache image MPI_Bcast", \ + mpi_result) } /* end else-if */ } /* end block */ #endif /* H5_HAVE_PARALLEL */ done: + FUNC_LEAVE_NOAPI(ret_value) + } /* H5C__read_cache_image() */ @@ -3203,26 +3225,31 @@ H5C__reconstruct_cache_contents(H5F_t *f, hid_t dxpl_id, H5C_t *cache_ptr) i = -1; entry_ptr = cache_ptr->LRU_head_ptr; + while(entry_ptr != NULL) { - HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + + HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(entry_ptr->type != NULL); - if(entry_ptr->prefetched) { - HDassert(i <= entry_ptr->lru_rank); - HDassert((entry_ptr->lru_rank <= 2) || - (entry_ptr->lru_rank == i + 1) || - (entry_ptr->lru_rank == i + 2)); + if ( entry_ptr->prefetched ) { - if((entry_ptr->lru_rank <= 2) && (entry_ptr->lru_rank == i + 2)) - lru_rank_holes++; + HDassert(entry_ptr->lru_rank != 0); + HDassert((entry_ptr->lru_rank == -1) || + (entry_ptr->lru_rank > i)); - i = entry_ptr->lru_rank; - } /* end if */ + if ( ( entry_ptr->lru_rank > 1 ) && + ( entry_ptr->lru_rank > i + 1 ) ) + + lru_rank_holes += entry_ptr->lru_rank - (i + 1); + + i = entry_ptr->lru_rank; + + } /* end if */ - entry_ptr = entry_ptr->next; + entry_ptr = entry_ptr->next; } /* end while */ - /* Holes of size 1 appear in the LRU ranking due to epoch + /* Holes in the sequences of LRU ranks can appear due to epoch * markers. They are left in to allow re-insertion of the * epoch markers on reconstruction of the cache -- thus * the following sanity check will have to be revised when diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h index 69e8145..90a84c4 100644 --- a/src/H5Cpkg.h +++ b/src/H5Cpkg.h @@ -686,6 +686,13 @@ if ( ( ( ( (head_ptr) == NULL ) || ( (tail_ptr) == NULL ) ) && \ (cache_ptr)->images_created++; \ } +#define H5C__UPDATE_STATS_FOR_CACHE_IMAGE_READ(cache_ptr) \ +{ \ + /* make sure image len is still good */ \ + HDassert((cache_ptr)->image_len > 0); \ + (cache_ptr)->images_read++; \ +} + #define H5C__UPDATE_STATS_FOR_CACHE_IMAGE_LOAD(cache_ptr) \ { \ /* make sure image len is still good */ \ @@ -931,6 +938,7 @@ if ( ( ( ( (head_ptr) == NULL ) || ( (tail_ptr) == NULL ) ) && \ #define H5C__UPDATE_STATS_FOR_LRU_SCAN_RESTART(cache_ptr) #define H5C__UPDATE_STATS_FOR_INDEX_SCAN_RESTART(cache_ptr) #define H5C__UPDATE_STATS_FOR_CACHE_IMAGE_CREATE(cache_ptr) +#define H5C__UPDATE_STATS_FOR_CACHE_IMAGE_READ(cache_ptr) #define H5C__UPDATE_STATS_FOR_CACHE_IMAGE_LOAD(cache_ptr) #define H5C__UPDATE_STATS_FOR_PREFETCH(cache_ptr, dirty) #define H5C__UPDATE_STATS_FOR_PREFETCH_HIT(cache_ptr) @@ -4549,6 +4557,18 @@ typedef struct H5C_tag_info_t { * Further, since cache images are only created at file * close, this field should only be set at that time. * + * images_read: Integer field containing the number of cache images + * read from file. Note that reading an image is different + * from loading it -- reading the image means just that, + * while loading the image refers to decoding it and loading + * it into the metadata cache. + * + * In the serial case, image_read should always equal + * images_loaded. However, in the parallel case, the + * image should only be read by process 0. All other + * processes should receive the cache image via a broadcast + * from process 0. + * * images_loaded: Integer field containing the number of cache images * loaded since the last time statistics were reset. * @@ -4864,6 +4884,7 @@ struct H5C_t { /* Fields for tracking cache image operations */ int32_t images_created; + int32_t images_read; int32_t images_loaded; hsize_t last_image_size; diff --git a/src/H5Ctag.c b/src/H5Ctag.c index a9bcca1..0170ce9 100644 --- a/src/H5Ctag.c +++ b/src/H5Ctag.c @@ -58,8 +58,18 @@ typedef struct { H5F_t * f; /* File pointer for evicting entry */ hid_t dxpl_id; /* DXPL for evicting entry */ - hbool_t evicted_entries_last_pass; /* Flag to indicate that an entry was evicted when iterating over cache */ - hbool_t pinned_entries_need_evicted; /* Flag to indicate that a pinned entry was attempted to be evicted */ + hbool_t evicted_entries_last_pass; /* Flag to indicate that an entry + * was evicted when iterating over + * cache + */ + hbool_t pinned_entries_need_evicted;/* Flag to indicate that a pinned + * entry was attempted to be evicted + */ + hbool_t skipped_pf_dirty_entries; /* Flag indicating that one or more + * entries marked prefetched_dirty + * were encountered and not + * evicted. + */ } H5C_tag_iter_evict_ctx_t; /* Typedef for tagged entry iterator callback context - expunge tag type metadata */ @@ -470,7 +480,9 @@ H5C__evict_tagged_entries_cb(H5C_cache_entry_t *entry, void *_ctx) if(H5C__flush_single_entry(ctx->f, ctx->dxpl_id, entry, H5C__FLUSH_INVALIDATE_FLAG | H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, H5_ITER_ERROR, "Entry eviction failed.") ctx->evicted_entries_last_pass = TRUE; - } /* end else */ + } else { + ctx->skipped_pf_dirty_entries = TRUE; + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -516,6 +528,7 @@ H5C_evict_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag, hbool_t match_gl /* Reset pinned/evicted tracking flags */ ctx.pinned_entries_need_evicted = FALSE; ctx.evicted_entries_last_pass = FALSE; + ctx.skipped_pf_dirty_entries = FALSE; /* Iterate through entries in the cache */ if(H5C__iter_tagged_entries(cache, tag, match_global, H5C__evict_tagged_entries_cb, &ctx) < 0) @@ -524,8 +537,32 @@ H5C_evict_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag, hbool_t match_gl /* Keep doing this until we have stopped evicted entries */ } while(TRUE == ctx.evicted_entries_last_pass); - /* Fail if we have finished evicting entries and pinned entries still need evicted */ - if(ctx.pinned_entries_need_evicted) + /* In most cases, fail if we have finished evicting entries and pinned + * entries still need evicted + * + * However, things can get strange if the file was opened R/O and + * the file contains a cache image and the cache image contains dirty + * entries. + * + * Since the file was opened read only, dirty entries in the cache + * image were marked as clean when they were inserted into the metadata + * cache. This is necessary, as if they are marked dirty, the metadata + * cache will attempt to write them on file close, which is frowned + * upon when the file is opened R/O. + * + * On the other hand, such entries (marked prefetched_dirty) must not + * be evicted, as should the cache be asked to re-load them, the cache + * will attempt to read them from the file, and at best load an outdated + * version. + * + * To avoid this, H5C__evict_tagged_entries_cb has been modified to + * skip such entries. However, by doing so, it may prevent pinned + * entries from becoming unpinned. + * + * Thus we must ignore ctx.pinned_entries_need_evicted if + * ctx.skipped_pf_dirty_entries is TRUE. + */ + if((!ctx.skipped_pf_dirty_entries) && (ctx.pinned_entries_need_evicted)) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Pinned entries still need evicted?!") done: diff --git a/test/cache_image.c b/test/cache_image.c index f49f7b3..dfbd02e 100644 --- a/test/cache_image.c +++ b/test/cache_image.c @@ -35,7 +35,7 @@ static void create_datasets(hid_t file_id, int min_dset, int max_dset); static void delete_datasets(hid_t file_id, int min_dset, int max_dset); static void open_hdf5_file(hbool_t create_file, hbool_t mdci_sbem_expected, hbool_t read_only, hbool_t set_mdci_fapl, hbool_t config_fsm, - const char *hdf_file_name, unsigned cache_image_flags, + hbool_t set_eoc, const char *hdf_file_name, unsigned cache_image_flags, hid_t *file_id_ptr, H5F_t **file_ptr_ptr, H5C_t **cache_ptr_ptr); static void attempt_swmr_open_hdf5_file(hbool_t create_file, hbool_t set_mdci_fapl, const char *hdf_file_name); @@ -62,6 +62,7 @@ static unsigned cache_image_api_error_check_3(void); static unsigned cache_image_api_error_check_4(void); static unsigned get_free_sections_test(void); +static unsigned evict_on_close_test(void); /****************************************************************************/ @@ -535,7 +536,7 @@ delete_datasets(hid_t file_id, int min_dset, int max_dset) static void open_hdf5_file(hbool_t create_file, hbool_t mdci_sbem_expected, hbool_t read_only, hbool_t set_mdci_fapl, hbool_t config_fsm, - const char *hdf_file_name, unsigned cache_image_flags, + hbool_t set_eoc, const char *hdf_file_name, unsigned cache_image_flags, hid_t *file_id_ptr, H5F_t ** file_ptr_ptr, H5C_t ** cache_ptr_ptr) { const char * fcn_name = "open_hdf5_file()"; @@ -601,7 +602,8 @@ open_hdf5_file(hbool_t create_file, hbool_t mdci_sbem_expected, /* call H5Pset_libver_bounds() on the fapl_id */ if ( pass ) { - if ( H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0 ) { + if ( H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, + H5F_LIBVER_LATEST) < 0 ) { pass = FALSE; failure_mssg = "H5Pset_libver_bounds() failed.\n"; @@ -667,7 +669,8 @@ open_hdf5_file(hbool_t create_file, hbool_t mdci_sbem_expected, } if ( ( pass ) && ( config_fsm ) ) { - if(H5Pset_file_space_strategy(fcpl_id, H5F_FSPACE_STRATEGY_PAGE, TRUE, (hsize_t)1) < 0) { + if(H5Pset_file_space_strategy(fcpl_id, H5F_FSPACE_STRATEGY_PAGE, + TRUE, (hsize_t)1) < 0) { pass = FALSE; failure_mssg = "H5Pset_file_space_strategy() failed."; } @@ -675,6 +678,18 @@ open_hdf5_file(hbool_t create_file, hbool_t mdci_sbem_expected, if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + /* set evict on close if indicated */ + if ( ( pass ) && ( set_eoc ) ) { + + if ( H5Pset_evict_on_close(fapl_id, TRUE) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Pset_evict_on_close() failed."; + } + } + + if ( show_progress ) HDfprintf(stdout, "%s: cp = %d.\n", fcn_name, cp++); + /* open the file */ if ( pass ) { @@ -682,10 +697,12 @@ open_hdf5_file(hbool_t create_file, hbool_t mdci_sbem_expected, if ( fcpl_id != -1 ) - file_id = H5Fcreate(hdf_file_name, H5F_ACC_TRUNC, fcpl_id, fapl_id); + file_id = H5Fcreate(hdf_file_name, H5F_ACC_TRUNC, + fcpl_id, fapl_id); else - file_id = H5Fcreate(hdf_file_name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + file_id = H5Fcreate(hdf_file_name, H5F_ACC_TRUNC, + H5P_DEFAULT, fapl_id); } else { @@ -1362,6 +1379,7 @@ check_cache_image_ctl_flow_1(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__GEN_MDCI_SBE_MESG, /* file_id_ptr */ &file_id, @@ -1415,6 +1433,7 @@ check_cache_image_ctl_flow_1(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -1476,6 +1495,7 @@ check_cache_image_ctl_flow_1(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -1637,6 +1657,7 @@ check_cache_image_ctl_flow_2(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__GEN_MDCI_SBE_MESG, /* file_id_ptr */ &file_id, @@ -1679,6 +1700,7 @@ check_cache_image_ctl_flow_2(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -1718,6 +1740,7 @@ check_cache_image_ctl_flow_2(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -1894,6 +1917,7 @@ check_cache_image_ctl_flow_3(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -1935,6 +1959,7 @@ check_cache_image_ctl_flow_3(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__GEN_MDCI_SBE_MESG, /* file_id_ptr */ &file_id, @@ -1988,6 +2013,7 @@ check_cache_image_ctl_flow_3(void) /* read_only */ TRUE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -2041,6 +2067,7 @@ check_cache_image_ctl_flow_3(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -2091,6 +2118,7 @@ check_cache_image_ctl_flow_3(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -2260,6 +2288,7 @@ check_cache_image_ctl_flow_4(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -2301,6 +2330,7 @@ check_cache_image_ctl_flow_4(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__GEN_MDCI_SBE_MESG, /* file_id_ptr */ &file_id, @@ -2343,6 +2373,7 @@ check_cache_image_ctl_flow_4(void) /* read_only */ TRUE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -2385,6 +2416,7 @@ check_cache_image_ctl_flow_4(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -2424,6 +2456,7 @@ check_cache_image_ctl_flow_4(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -2589,6 +2622,7 @@ check_cache_image_ctl_flow_5(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__GEN_MDCI_SBE_MESG, /* file_id_ptr */ &file_id, @@ -2648,6 +2682,7 @@ check_cache_image_ctl_flow_5(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__GEN_MDCI_SBE_MESG, /* file_id_ptr */ &file_id, @@ -2700,6 +2735,7 @@ check_cache_image_ctl_flow_5(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -2867,6 +2903,7 @@ check_cache_image_ctl_flow_6(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__GEN_MDCI_SBE_MESG, /* file_id_ptr */ &file_id, @@ -2915,6 +2952,7 @@ check_cache_image_ctl_flow_6(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__GEN_MDCI_SBE_MESG, /* file_id_ptr */ &file_id, @@ -2957,6 +2995,7 @@ check_cache_image_ctl_flow_6(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -3149,6 +3188,7 @@ cache_image_smoke_check_1(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -3215,6 +3255,7 @@ cache_image_smoke_check_1(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -3280,6 +3321,7 @@ cache_image_smoke_check_1(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -3346,6 +3388,7 @@ cache_image_smoke_check_1(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -3388,6 +3431,7 @@ cache_image_smoke_check_1(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -3567,6 +3611,7 @@ cache_image_smoke_check_2(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ TRUE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -3630,6 +3675,7 @@ cache_image_smoke_check_2(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -3675,6 +3721,7 @@ cache_image_smoke_check_2(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -3865,6 +3912,7 @@ cache_image_smoke_check_3(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ TRUE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -3927,6 +3975,7 @@ cache_image_smoke_check_3(void) /* read_only */ TRUE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -3992,6 +4041,7 @@ cache_image_smoke_check_3(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -4055,6 +4105,7 @@ cache_image_smoke_check_3(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -4248,6 +4299,7 @@ cache_image_smoke_check_4(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ TRUE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -4312,6 +4364,7 @@ cache_image_smoke_check_4(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -4379,6 +4432,7 @@ cache_image_smoke_check_4(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -4442,6 +4496,7 @@ cache_image_smoke_check_4(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -4493,7 +4548,6 @@ cache_image_smoke_check_4(void) if ( show_progress ) HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); - /* 13) Delete the file */ @@ -4653,6 +4707,7 @@ cache_image_smoke_check_5(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ TRUE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -4742,6 +4797,7 @@ cache_image_smoke_check_5(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -4842,6 +4898,7 @@ cache_image_smoke_check_5(void) /* read_only */ TRUE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -4893,6 +4950,7 @@ cache_image_smoke_check_5(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -4953,6 +5011,7 @@ cache_image_smoke_check_5(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -5161,6 +5220,7 @@ cache_image_smoke_check_6(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ TRUE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -5225,6 +5285,7 @@ cache_image_smoke_check_6(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -5303,6 +5364,7 @@ cache_image_smoke_check_6(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -5362,6 +5424,7 @@ cache_image_smoke_check_6(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -5559,6 +5622,7 @@ cache_image_api_error_check_1(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ TRUE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -5619,6 +5683,7 @@ cache_image_api_error_check_1(void) /* read_only */ TRUE, /* set_mdci_fapl */ TRUE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -5681,6 +5746,7 @@ cache_image_api_error_check_1(void) /* read_only */ TRUE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -5744,6 +5810,7 @@ cache_image_api_error_check_1(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -5931,6 +5998,7 @@ cache_image_api_error_check_2(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ TRUE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -5991,6 +6059,7 @@ cache_image_api_error_check_2(void) /* read_only */ TRUE, /* set_mdci_fapl */ TRUE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -6053,6 +6122,7 @@ cache_image_api_error_check_2(void) /* read_only */ TRUE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ 0, /* file_id_ptr */ &file_id, @@ -6116,6 +6186,7 @@ cache_image_api_error_check_2(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -6177,6 +6248,7 @@ cache_image_api_error_check_2(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -6337,6 +6409,7 @@ cache_image_api_error_check_3(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ TRUE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -6422,6 +6495,7 @@ cache_image_api_error_check_3(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ TRUE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -6743,6 +6817,7 @@ cache_image_api_error_check_4(void) /* read_only */ TRUE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -6997,6 +7072,7 @@ cache_image_api_error_check_4(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -7209,6 +7285,7 @@ get_free_sections_test(void) /* read_only */ FALSE, /* set_mdci_fapl */ TRUE, /* config_fsm */ TRUE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -7274,6 +7351,7 @@ get_free_sections_test(void) /* read_only */ TRUE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -7398,6 +7476,7 @@ get_free_sections_test(void) /* read_only */ FALSE, /* set_mdci_fapl */ FALSE, /* config_fsm */ FALSE, + /* set_eoc */ FALSE, /* hdf_file_name */ filename, /* cache_image_flags */ H5C_CI__ALL_FLAGS, /* file_id_ptr */ &file_id, @@ -7566,6 +7645,379 @@ get_free_sections_test(void) /*------------------------------------------------------------------------- + * Function: evict_on_close_test() + * + * Purpose: If a file containing a cache image which in turn + * contains dirty entries is opened R/O, the metadata + * cache must refuse to evict the dirty entries, as + * it will not be able to reload them from file. This + * is a bit tricky, as the dirty entries must marked as + * clean in the metadata cache so that the MDC will not + * attempt to flush then on file close. + * + * The objective of this test is to verify that the + * metadata will not evict dirty entries in the above + * context when the file is opened with the evict on close + * FAPL entry. + * + * Do this by creating a HDF5 file with a cache image + * containing dirty metadata. + * + * Then close the file, re-open it R/O, and scan its + * contents twice. If evict on close succeeds in evicting + * the dirty metadata, the second scan will fail, as valid + * versions of the dirty metadata will not be available. + * + * To make the test more useful, enable persistant free + * space managers. + * + * The test is set up as follows: + * + * 1) Create a HDF5 file without a cache image requested + * and persistant free space managers enabled. + * + * 2) Create some data sets and verify them. + * + * 3) Close the file. + * + * 4) Open the file R/W, and with cache image requested. + * + * 5) Verify the datasets created in 2) above. This will + * force their (clean) metadata into the metadata cache, + * and hence into the cache image. + * + * 6) Create some more datasets. + * + * 7) Close the file. + * + * 8) Open the file R/O and with evict on close enabled. + * + * 9) Verify all datasets twice. + * + * 10) Close the file. + * + * 11) Open the file R/W and with evict on close enabled. + * + * 12) Verify all datasets twice. + * + * 13) Close the file. + * + * 14) Discard the file. + * + * Return: void + * + * Programmer: John Mainzer + * 3/23/17 + * + *------------------------------------------------------------------------- + */ +static unsigned +evict_on_close_test(void) +{ + const char * fcn_name = "evict_on_close_test()"; + char filename[512]; + hbool_t show_progress = FALSE; + hbool_t verbose = FALSE; + hid_t file_id = -1; + H5F_t *file_ptr = NULL; + H5C_t *cache_ptr = NULL; + int cp = 0; + + TESTING("Cache image / evict on close interaction"); + + pass = TRUE; + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* setup the file name */ + if ( pass ) { + + if ( h5_fixname(FILENAMES[0], H5P_DEFAULT, filename, sizeof(filename)) + == NULL ) { + + pass = FALSE; + failure_mssg = "h5_fixname() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 1) Create a HDF5 file without a cache image requested + * and persistant free space managers enabled. + */ + if ( pass ) { + + open_hdf5_file(/* create_file */ TRUE, + /* mdci_sbem_expected */ FALSE, + /* read_only */ FALSE, + /* set_mdci_fapl */ FALSE, + /* config_fsm */ TRUE, + /* set_eoc */ FALSE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 2) Create some data sets and verify them. */ + + if ( pass ) { + + create_datasets(file_id, 1, 10); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { + + verify_datasets(file_id, 1, 10); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 3) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed (1).\n"; + + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 4) Open the file R/W, and with cache image requested. */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ FALSE, + /* mdci_sbem_expected */ FALSE, + /* read_only */ FALSE, + /* set_mdci_fapl */ TRUE, + /* config_fsm */ FALSE, + /* set_eoc */ FALSE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 5) Verify the datasets created in 2) above. This will + * force their (clean) metadata into the metadata cache, + * and hence into the cache image. + */ + + if ( pass ) { + + verify_datasets(file_id, 1, 10); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 6) Create some more datasets and verify them */ + + if ( pass ) { + + create_datasets(file_id, 11, 20); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { + + verify_datasets(file_id, 11, 20); + } + + if ( verbose ) { + + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + + HDfprintf(stdout, "index size / index dirty size = %lld / %lld\n", + (long long)(cache_ptr->index_size), + (long long)(cache_ptr->dirty_index_size)); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 7) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed (2).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 8) Open the file R/O and with evict on close enabled. */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ FALSE, + /* mdci_sbem_expected */ TRUE, + /* read_only */ TRUE, + /* set_mdci_fapl */ FALSE, + /* config_fsm */ FALSE, + /* set_eoc */ TRUE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr); + } + + if ( show_progress ) + HDfprintf(stdout, "%s*: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 9) Verify all datasets twice */ + + if ( pass ) { + + verify_datasets(file_id, 1, 20); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { + + verify_datasets(file_id, 1, 20); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 10) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed (3).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 11) Open the file R/w and with evict on close enabled. */ + + if ( pass ) { + + open_hdf5_file(/* create_file */ FALSE, + /* mdci_sbem_expected */ TRUE, + /* read_only */ FALSE, + /* set_mdci_fapl */ FALSE, + /* config_fsm */ FALSE, + /* set_eoc */ TRUE, + /* hdf_file_name */ filename, + /* cache_image_flags */ H5C_CI__ALL_FLAGS, + /* file_id_ptr */ &file_id, + /* file_ptr_ptr */ &file_ptr, + /* cache_ptr_ptr */ &cache_ptr); + } + + if ( show_progress ) + HDfprintf(stdout, "%s*: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 12) Verify all datasets twice */ + + if ( pass ) { + + verify_datasets(file_id, 1, 20); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { + + verify_datasets(file_id, 1, 20); + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 13) Close the file. */ + + if ( pass ) { + + if ( H5Fclose(file_id) < 0 ) { + + pass = FALSE; + failure_mssg = "H5Fclose() failed (3).\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + + /* 14) Discard the file. */ + + if ( pass ) { + + if ( HDremove(filename) < 0 ) { + + pass = FALSE; + failure_mssg = "HDremove() failed.\n"; + } + } + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + if ( pass ) { PASSED(); } else { H5_FAILED(); } + + if ( ! pass ) + HDfprintf(stdout, "%s: failure_mssg = \"%s\".\n", + FUNC, failure_mssg); + + return !pass; + +} /* evict_on_close_test() */ + + +/*------------------------------------------------------------------------- * Function: main * * Purpose: Run tests on the cache code contained in H5C.c @@ -7614,6 +8066,7 @@ main(void) nerrs += cache_image_api_error_check_4(); nerrs += get_free_sections_test(); + nerrs += evict_on_close_test(); return(nerrs > 0); diff --git a/test/page_buffer.c b/test/page_buffer.c index b94ad03..54a25d6 100644 --- a/test/page_buffer.c +++ b/test/page_buffer.c @@ -52,6 +52,10 @@ static unsigned test_raw_data_handling(hid_t orig_fapl, const char *env_h5_drvr) static unsigned test_lru_processing(hid_t orig_fapl, const char *env_h5_drvr); static unsigned test_min_threshold(hid_t orig_fapl, const char *env_h5_drvr); static unsigned test_stats_collection(hid_t orig_fapl, const char *env_h5_drvr); +#ifdef H5_HAVE_PARALLEL +static unsigned verify_page_buffering_disabled(hid_t orig_fapl, + const char *env_h5_drvr); +#endif /* H5_HAVE_PARALLEL */ const char *FILENAME[] = { "filepaged", @@ -2007,6 +2011,121 @@ error: /*------------------------------------------------------------------------- + * Function: verify_page_buffering_disabled() + * + * Purpose: This function should only be called in parallel + * builds. + * + * At present, page buffering should be disabled in parallel + * builds. Verify this. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: John Mainzer + * 03/21/17 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ + +#ifdef H5_HAVE_PARALLEL +static unsigned +verify_page_buffering_disabled(hid_t orig_fapl, const char *env_h5_drvr) +{ + char filename[FILENAME_LEN]; /* Filename to use */ + hid_t file_id = -1; /* File ID */ + hid_t fcpl = -1; + hid_t fapl = -1; + + TESTING("Page Buffering Disabled"); + h5_fixname(FILENAME[0], orig_fapl, filename, sizeof(filename)); + + + /* first, try to create a file with page buffering enabled */ + + if((fapl = H5Pcopy(orig_fapl)) < 0) + TEST_ERROR + + if(set_multi_split(env_h5_drvr, fapl, 4096) != 0) + TEST_ERROR; + + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR; + + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + FAIL_STACK_ERROR; + + if(H5Pset_file_space_page_size(fcpl, 4096) < 0) + FAIL_STACK_ERROR; + + if(H5Pset_page_buffer_size(fapl, 4096*8, 0, 0) < 0) + FAIL_STACK_ERROR; + + /* try to create the file -- should fail */ + H5E_BEGIN_TRY { + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl); + } H5E_END_TRY; + + if(file_id >= 0) + TEST_ERROR; + + /* now, create a file, close it, and then try to open it with page + * buffering enabled. + */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + FAIL_STACK_ERROR; + + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1) < 0) + FAIL_STACK_ERROR; + + if(H5Pset_file_space_page_size(fcpl, 4096) < 0) + FAIL_STACK_ERROR; + + /* create the file */ + if((file_id = H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + + /* close the file */ + if(H5Fclose(file_id) < 0) + FAIL_STACK_ERROR; + + /* try to open the file using the fapl prepared above which enables + * page buffering. Should fail. + */ + H5E_BEGIN_TRY { + file_id = H5Fopen(filename, H5F_ACC_RDWR, fapl); + } H5E_END_TRY; + + if(file_id >= 0) + TEST_ERROR; + + if(H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + + if(H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + + PASSED() + + return 0; + +error: + + H5E_BEGIN_TRY { + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(file_id); + } H5E_END_TRY; + + return 1; + +} /* verify_page_buffering_disabled() */ +#endif /* H5_HAVE_PARALLEL */ + + +/*------------------------------------------------------------------------- * Function: main() * * Purpose: Main function for the page buffer tests. @@ -2026,8 +2145,6 @@ main(void) unsigned nerrors = 0; /* Cumulative error count */ const char *env_h5_drvr = NULL; /* File Driver value from environment */ -#ifndef H5_HAVE_PARALLEL - h5_reset(); /* Get the VFD to use */ @@ -2052,12 +2169,21 @@ main(void) PUTS_ERROR("Can't get VFD-dependent fapl") } /* end if */ +#ifdef H5_HAVE_PARALLEL + + HDputs("Page Buffering is disabled for parallel."); + nerrors += verify_page_buffering_disabled(fapl, env_h5_drvr); + +#else /* H5_HAVE_PARALLEL */ + nerrors += test_args(fapl, env_h5_drvr); nerrors += test_raw_data_handling(fapl, env_h5_drvr); nerrors += test_lru_processing(fapl, env_h5_drvr); nerrors += test_min_threshold(fapl, env_h5_drvr); nerrors += test_stats_collection(fapl, env_h5_drvr); +#endif /* H5_HAVE_PARALLEL */ + h5_clean_files(FILENAME, fapl); if(nerrors) @@ -2065,11 +2191,6 @@ main(void) HDputs("All Page Buffering tests passed."); -#else - SKIPPED() - HDputs("Page Buffering is disabled for parallel."); -#endif - HDexit(EXIT_SUCCESS); error: diff --git a/testpar/t_cache_image.c b/testpar/t_cache_image.c index a28af8e..f65b248 100644 --- a/testpar/t_cache_image.c +++ b/testpar/t_cache_image.c @@ -46,7 +46,9 @@ const char *FILENAMES[] = { /* local utility function declarations */ static void create_data_sets(hid_t file_id, int min_dset, int max_dset); +#if 0 /* keep pending full parallel cache image */ static void delete_data_sets(hid_t file_id, int min_dset, int max_dset); +#endif static void open_hdf5_file(const hbool_t create_file, const hbool_t mdci_sbem_expected, @@ -796,7 +798,12 @@ create_data_sets(hid_t file_id, int min_dset, int max_dset) * *------------------------------------------------------------------------- */ - +#if 0 +/* this code will be needed to test full support of cache image + * in parallel -- keep it around against that day. + * + * -- JRM + */ static void delete_data_sets(hid_t file_id, int min_dset, int max_dset) { @@ -843,6 +850,7 @@ delete_data_sets(hid_t file_id, int min_dset, int max_dset) return; } /* delete_data_sets() */ +#endif /*------------------------------------------------------------------------- @@ -2756,12 +2764,22 @@ usage(void) "The \"setup\" parameter indicates that t_cache_image is being \n", "invokde for this purpose.\n", "\n", - "usage: t_cache_image [setup]\n", + "Similarly, only a serial process can add a cache image to an\n", + "existing file.\n", + "\n", + "Here again, one process forks a serial version of the test program\n", + "with the \"ici\" parameter.\n" + "\n", + "usage: t_cache_image [setup|ici m n]\n", "\n", "where:\n", "\n", " setup parameter forces creation of test file\n", "\n", + " ici parameter forces insertion of a cache image into the \n", + " m th test file, created by a parallel computation with .\n", + " n processes\n", + "\n", "Returns 0 on success, 1 on failure.\n", "\n", NULL, @@ -3141,11 +3159,6 @@ verify_cache_image_RO(int file_name_id, int md_write_strat, int mpi_rank) /* 1) Open the test file created at the beginning of this test. * * Verify that the file contains a cache image. - * - * Verify that only process 0 reads the cache image. - * - * Verify that all other processes receive the cache - * image block from process 0. */ if ( pass ) { @@ -3172,36 +3185,57 @@ verify_cache_image_RO(int file_name_id, int md_write_strat, int mpi_rank) if ( show_progress ) HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); - /* Verify that only process 0 reads the cache image. */ - - if ( show_progress ) - HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); - /* Verify that all other processes receive the cache image block - * from process 0. + /* 2) Verify that the file contains the expected data. + * + * Verify that only process 0 reads the cache image. + * + * Verify that all other processes receive the cache + * image block from process 0. */ - if ( show_progress ) - HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + if ( pass ) { + verify_data_sets(file_id, 0, MAX_NUM_DSETS - 1); + } - /* 2) Verify that the file contains the expected data. */ + /* Verify that only process 0 reads the cache image. */ +#if H5C_COLLECT_CACHE_STATS if ( pass ) { - verify_data_sets(file_id, 0, MAX_NUM_DSETS - 1); + if ( ( ( mpi_rank == 0 ) && ( cache_ptr->images_read != 1 ) ) || + ( ( mpi_rank > 0 ) && ( cache_ptr->images_read != 0 ) ) ) { + + pass = FALSE; + failure_mssg = "unexpected images_read."; + } } +#endif /* H5C_COLLECT_CACHE_STATS */ + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* Verify that all other processes receive the cache image block + * from process 0. + * + * Since we have alread verified that only process 0 has read the + * image, it is sufficient to verify that the image was loaded on + * all processes. + */ #if H5C_COLLECT_CACHE_STATS if ( pass ) { - if ( cache_ptr->images_loaded == 0 ) { + if ( cache_ptr->images_loaded != 1 ) { pass = FALSE; - failure_mssg = "metadata cache image block not loaded(1)."; + failure_mssg = "Image not loaded?."; } } #endif /* H5C_COLLECT_CACHE_STATS */ + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + /* 3) Close the file. */ @@ -3317,12 +3351,13 @@ verify_cache_image_RO(int file_name_id, int md_write_strat, int mpi_rank) * * Verify that the file contains a cache image. * + * 2) Verify that the file contains the expected data. + * * Verify that only process 0 reads the cache image. * * Verify that all other processes receive the cache * image block from process 0. * - * 2) Verify that the file contains the expected data. * * 3) Close the file. * @@ -3434,36 +3469,56 @@ verify_cache_image_RW(int file_name_id, int md_write_strat, int mpi_rank) if ( show_progress ) HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); - /* Verify that only process 0 reads the cache image. */ - if ( show_progress ) - HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); - - /* Verify that all other processes receive the cache image block - * from process 0. + /* 2) Verify that the file contains the expected data. + * + * Verify that only process 0 reads the cache image. + * + * Verify that all other processes receive the cache + * image block from process 0. */ + if ( pass ) { - if ( show_progress ) - HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); - + verify_data_sets(file_id, 0, MAX_NUM_DSETS - 1); + } - /* 2) Verify that the file contains the expected data. */ + /* Verify that only process 0 reads the cache image. */ +#if H5C_COLLECT_CACHE_STATS if ( pass ) { - verify_data_sets(file_id, 0, MAX_NUM_DSETS - 1); + if ( ( ( mpi_rank == 0 ) && ( cache_ptr->images_read != 1 ) ) || + ( ( mpi_rank > 0 ) && ( cache_ptr->images_read != 0 ) ) ) { + + pass = FALSE; + failure_mssg = "unexpected images_read."; + } } +#endif /* H5C_COLLECT_CACHE_STATS */ + + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + /* Verify that all other processes receive the cache image block + * from process 0. + * + * Since we have alread verified that only process 0 has read the + * image, it is sufficient to verify that the image was loaded on + * all processes. + */ #if H5C_COLLECT_CACHE_STATS if ( pass ) { - if ( cache_ptr->images_loaded == 0 ) { + if ( cache_ptr->images_loaded != 1 ) { pass = FALSE; - failure_mssg = "metadata cache image block not loaded(2)."; + failure_mssg = "Image not loaded?."; } } #endif /* H5C_COLLECT_CACHE_STATS */ + if ( show_progress ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + /* 3) Close the file. */ @@ -3754,7 +3809,13 @@ smoke_check_1(MPI_Comm mpi_comm, MPI_Info mpi_info, int mpi_rank, int mpi_size) HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); - /* 7) Verify the datasets in the file backwards */ + /* 7) Verify the datasets in the file backwards + * + * Verify that only process 0 reads the cache image. + * + * Verify that all other processes receive the cache + * image block from process 0. + */ i = num_dsets - 1; while ( ( pass ) && ( i >= 0 ) ) { @@ -3763,6 +3824,46 @@ smoke_check_1(MPI_Comm mpi_comm, MPI_Info mpi_info, int mpi_rank, int mpi_size) i--; } + if ( ( mpi_rank == 0 ) && ( show_progress ) ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* Verify that only process 0 reads the cache image. */ +#if H5C_COLLECT_CACHE_STATS + if ( pass ) { + + if ( ( ( mpi_rank == 0 ) && ( cache_ptr->images_read != 1 ) ) || + ( ( mpi_rank > 0 ) && ( cache_ptr->images_read != 0 ) ) ) { + + pass = FALSE; + failure_mssg = "unexpected images_read."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + if ( ( mpi_rank == 0 ) && ( show_progress ) ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* Verify that all other processes receive the cache image block + * from process 0. + * + * Since we have alread verified that only process 0 has read the + * image, it is sufficient to verify that the image was loaded on + * all processes. + */ +#if H5C_COLLECT_CACHE_STATS + if ( pass ) { + + if ( cache_ptr->images_loaded != 1 ) { + + pass = FALSE; + failure_mssg = "Image not loaded?."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + if ( ( mpi_rank == 0 ) && ( show_progress ) ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + /* 8) Close the file */ @@ -3776,6 +3877,9 @@ smoke_check_1(MPI_Comm mpi_comm, MPI_Info mpi_info, int mpi_rank, int mpi_size) } } + if ( ( mpi_rank == 0 ) && ( show_progress ) ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + /* 9) Open the file */ @@ -3804,7 +3908,13 @@ smoke_check_1(MPI_Comm mpi_comm, MPI_Info mpi_info, int mpi_rank, int mpi_size) HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); - /* 10) Verify the datasets in the file */ + /* 10) Verify the datasets in the file + * + * Verify that only process 0 reads the cache image. + * + * Verify that all other processes receive the cache + * image block from process 0. + */ i = 0; while ( ( pass ) && ( i < num_dsets ) ) { @@ -3816,6 +3926,43 @@ smoke_check_1(MPI_Comm mpi_comm, MPI_Info mpi_info, int mpi_rank, int mpi_size) if ( ( mpi_rank == 0 ) && ( show_progress ) ) HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + /* Verify that only process 0 reads the cache image. */ +#if H5C_COLLECT_CACHE_STATS + if ( pass ) { + + if ( ( ( mpi_rank == 0 ) && ( cache_ptr->images_read != 1 ) ) || + ( ( mpi_rank > 0 ) && ( cache_ptr->images_read != 0 ) ) ) { + + pass = FALSE; + failure_mssg = "unexpected images_read."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + if ( ( mpi_rank == 0 ) && ( show_progress ) ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + + /* Verify that all other processes receive the cache image block + * from process 0. + * + * Since we have alread verified that only process 0 has read the + * image, it is sufficient to verify that the image was loaded on + * all processes. + */ +#if H5C_COLLECT_CACHE_STATS + if ( pass ) { + + if ( cache_ptr->images_loaded != 1 ) { + + pass = FALSE; + failure_mssg = "Image not loaded?."; + } + } +#endif /* H5C_COLLECT_CACHE_STATS */ + + if ( ( mpi_rank == 0 ) && ( show_progress ) ) + HDfprintf(stdout, "%s: cp = %d, pass = %d.\n", fcn_name, cp++, pass); + /* 11) Delete the datasets in the file */ -- cgit v0.12 From ef297238bd51d4a7fa530172a547952207c4902d Mon Sep 17 00:00:00 2001 From: lrknox Date: Wed, 29 Mar 2017 13:36:52 -0500 Subject: Add LBNL license file and modify COPYING file accordingly. --- COPYING | 15 +++++++++++---- COPYING_LBNL_HDF5 | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 COPYING_LBNL_HDF5 diff --git a/COPYING b/COPYING index 2166ced..51cb4e6 100644 --- a/COPYING +++ b/COPYING @@ -1,13 +1,12 @@ - Copyright Notice and License Terms for HDF5 (Hierarchical Data Format 5) Software Library and Utilities ----------------------------------------------------------------------------- HDF5 (Hierarchical Data Format 5) Software Library and Utilities -Copyright 2006-2016 by The HDF Group. +Copyright (c) 2006, The HDF Group. NCSA HDF5 (Hierarchical Data Format 5) Software Library and Utilities -Copyright 1998-2006 by the Board of Trustees of the University of Illinois. +Copyright (c) 1998-2006, The Board of Trustees of the University of Illinois. All rights reserved. @@ -47,13 +46,21 @@ advised of the possibility of such damage. ----------------------------------------------------------------------------- ----------------------------------------------------------------------------- +Limited portions of HDF5 were developed by Lawrence Berkeley National +Laboratory (LBNL). LBNL's Copyright Notice and Licensing Terms can be +found here: COPYING_LBNL_HDF5 file in this directory or at +http://support.hdfgroup.org/ftp/HDF5/releases/COPYING_LBNL_HDF5. + +----------------------------------------------------------------------------- +----------------------------------------------------------------------------- + Contributors: National Center for Supercomputing Applications (NCSA) at the University of Illinois, Fortner Software, Unidata Program Center (netCDF), The Independent JPEG Group (JPEG), Jean-loup Gailly and Mark Adler (gzip), and Digital Equipment Corporation (DEC). ----------------------------------------------------------------------------- - + Portions of HDF5 were developed with support from the Lawrence Berkeley National Laboratory (LBNL) and the United States Department of Energy under Prime Contract No. DE-AC02-05CH11231. diff --git a/COPYING_LBNL_HDF5 b/COPYING_LBNL_HDF5 new file mode 100644 index 0000000..16fba5d --- /dev/null +++ b/COPYING_LBNL_HDF5 @@ -0,0 +1,49 @@ +Copyright Notice and License Terms for +HDF5 (Hierarchical Data Format 5) Software Library and Utilities +----------------------------------------------------------------------------- + +HDF5 (Hierarchical Data Format 5) +Copyright (c) 2016, The Regents of the University of California, through +Lawrence Berkeley National Laboratory (subject to receipt of any required +approvals from the U.S. Dept. of Energy). + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the documentation + and/or materials provided with the distribution. + +3. Neither the name of the University of California, Lawrence Berkeley +National Laboratory, U.S. Dept. of Energy nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. + +You are under no obligation whatsoever to provide any bug fixes, patches, +or upgrades to the features, functionality or performance of the source +code ("Enhancements") to anyone; however, if you choose to make your +Enhancements available either publicly, or directly to Lawrence Berkeley +National Laboratory, without imposing a separate written license agreement +for such Enhancements, then you hereby grant the following license: +a non-exclusive, royalty-free perpetual license to install, use, modify, +prepare derivative works, incorporate into other computer software, +distribute, and sublicense such enhancements or derivative works thereof, +in binary and source code form. + -- cgit v0.12 From c1f95b28caae290a7c0136537daca9004db055b8 Mon Sep 17 00:00:00 2001 From: lrknox Date: Wed, 29 Mar 2017 13:55:41 -0500 Subject: Revert "Clear hdf5 1.10 entries from RELEASE.txt in the develop branch. Entries" This reverts commit 1a114c191d6366558202522df96c2726c0512cc0. --- release_docs/RELEASE.txt | 359 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 332 insertions(+), 27 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index d2e73e0..22accd3 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -4,8 +4,8 @@ HDF5 version 1.11.0 currently under development INTRODUCTION -This document describes the differences between HDF5-1.11.x and -HDF5 1.10.1, and contains information on the platforms tested and known problems in HDF5-1.11.x. +This document describes the differences between HDF5-1.10.0-patch1 and +HDF5 1.10.1, and contains information on the platforms tested and known problems in HDF5-1.10.1. For more details check the HISTORY*.txt files in the HDF5 source. @@ -28,8 +28,8 @@ in 1.10.1?" document: https://support.hdfgroup.org/HDF5/doc/ADGuide/WhatsNew1101.html All new and modified APIs are listed in detail in the "HDF5 Software Changes -from Release to Release" document, in the section "Release 1.10.1 (current -release) versus Release 1.11.x +from Release to Release" document, in the section "Release 1.8.19 (current +release) versus Release 1.10.1 https://support.hdfgroup.org/HDF5/doc1.10/ADGuide/Changes.html @@ -42,7 +42,7 @@ CONTENTS - New Features - Support for new platforms and languages -- Bug Fixes since HDF5-1.10.1 +- Bug Fixes since HDF5-1.10.0-patch1 - Supported Platforms - Tested Configuration Features Summary - More Tested Platforms @@ -54,11 +54,33 @@ New Features Configuration: ------------- - - + - CMake minimum is now 3.2.2. + (ADB 2017/01/10) + + - Tools folder is separated into source and test folders. This + allows autotools to skip the make command and just execute + the make check command. + (HDFFV-9719 ADB 2016/10/27) Library: -------- - - + - Paged Aggregation + + This is one of the file space management strategies. This strategy + aggregates small metadata and raw data allocations into constant-sized + well-aligned pages, which are suitable for page caching. Paged + aggregation together with the page buffering feature will allow efficient + I/O accesses. + + - Page Buffering + + The page buffering layer in the HDF5 library absorbs small accesses to + the file system. Each page in memory corresponds to a page allocated in + the file. Access to the file system is then performed as a single page + or multiple of pages, if they are contiguous. This ensures that small + accesses to the file system are avoided while providing another caching + layer for improved I/O performance. This feature works in conjunction + with the paged aggregation feature. Parallel Library: ----------------- @@ -74,11 +96,13 @@ New Features Tools: ------ - - + - Add options to h5clear: + -s: clear the status_flags field in the file's superblock + -m: Remove the metadata cache image from the file + (Pull Request #361 QK 2017/03/22) High-Level APIs: --------------- - - C Packet Table API ------------------ @@ -95,16 +119,44 @@ Support for new platforms, languages and compilers. ======================================= - -Bug Fixes since HDF5-1.10.1 release +Bug Fixes since HDF5-1.10.0-patch1 release ================================== Library ------- - - + - Changed the plugins dlopen option from RTLD_NOW to RTLD_LAZY + (PR 201 ADB 2016/12/12) + + - Fix error when copying dataset with attribute which is a compound datatype + consisting of a variable length string. + (HDFFV-7991 VC 2016/08/19, 2016/08/21, 2016/08/24) + + - H5DOappend will not fail if a dataset has no append callback registered. + (HDFFV-9960 VC 2016/08/05, 2016/08/14) + + - Fix the problem where the committed datatype's file location is different + from the file location of an attribute with that committed datatype. + (HDFFV-9940 VC 2016/07/03, 2016/07/06) + + - (a) Throw an error instead of assertion when v1 btree level hits the 1 byte limit. + (b) Modifications to better handle error recovery when conversion by + h5format_convert fails. + (HDFFV-9434 VC 2016/05/29) Configuration ------------- - - + - Configuration will check for the strtoll and strtoull functions + before using alternatives + (PR 340 ADB 2017/03/17) + + - CMake uses a Windows pdb directory variable if available and + will generate both static and shared pdb files. + (HDFFV-9875 ADB 2017/02/06) + + - CMake now builds shared versions of tools. + (HDFFV-10123 ADB 2017/02/01) + + Performance ------------- - @@ -114,7 +166,11 @@ Bug Fixes since HDF5-1.10.1 release Tools ----- - - + - h5diff correctly ignores strpad in comparing strings. + (HDFFV-10128 ADB 2017/03/03) + + - h5repack now correctly parses the command line filter options. + (HDFFV-10046 ADB 2017/01/24) High-Level APIs: ------ @@ -145,6 +201,18 @@ Bug Fixes since HDF5-1.10.1 release Supported Platforms =================== + Linux 2.6.32-573.22.1.el6 GNU C (gcc), Fortran (gfortran), C++ (g++) + #1 SMP x86_64 GNU/Linux compilers: + (mayll/platypus) Version 4.4.7 20120313 + Version 4.8.4 + PGI C, Fortran, C++ for 64-bit target on + x86-64; + Version 16.10-0 + Intel(R) C (icc), C++ (icpc), Fortran (icc) + compilers: + Version 15.0.3.187 (Build 20150407) + MPICH 3.1.4 compiled with GCC 4.9.3 + Linux 2.6.32-573.18.1.el6.ppc64 gcc (GCC) 4.4.7 20120313 (Red Hat 4.4.7-4) #1 SMP ppc64 GNU/Linux g++ (GCC) 4.4.7 20120313 (Red Hat 4.4.7-4) (ostrich) GNU Fortran (GCC) 4.4.7 20120313 (Red Hat 4.4.7-4) @@ -223,7 +291,9 @@ Mac OS X Mountain Lion 10.8.5 64-bit n y/y n y y y Mac OS X Mavericks 10.9.5 64-bit n y/y n y y ? Mac OS X Yosemite 10.10.5 64-bit n y/y n y y ? Mac OS X El Capitan 10.11.6 64-bit n y/y n y y ? -CentOS 7.2 Linux 2.6.32 x86_64 PGI n y/y n y y y +CentOS 6.7 Linux 2.6.18 x86_64 GNU n y/y n y y y +CentOS 6.7 Linux 2.6.18 x86_64 Intel n y/y n y y y +CentOS 6.7 Linux 2.6.32 x86_64 PGI n y/y n y y y CentOS 7.2 Linux 2.6.32 x86_64 GNU y y/y y y y y CentOS 7.2 Linux 2.6.32 x86_64 Intel n y/y n y y y Linux 2.6.32-573.18.1.el6.ppc64 n y/n n y y y @@ -243,7 +313,9 @@ Mac OS X Mountain Lion 10.8.5 64-bit y n y y Mac OS X Mavericks 10.9.5 64-bit y n y y Mac OS X Yosemite 10.10.5 64-bit y n y y Mac OS X El Capitan 10.11.6 64-bit y n y y -CentOS 7.2 Linux 2.6.32 x86_64 PGI y y y n +CentOS 6.7 Linux 2.6.18 x86_64 GNU y y y y +CentOS 6.7 Linux 2.6.18 x86_64 Intel y y y n +CentOS 6.7 Linux 2.6.32 x86_64 PGI y y y n CentOS 7.2 Linux 2.6.32 x86_64 GNU y y y n CentOS 7.2 Linux 2.6.32 x86_64 Intel y y y n Linux 2.6.32-573.18.1.el6.ppc64 y y y n @@ -256,18 +328,6 @@ More Tested Platforms ===================== The following platforms are not supported but have been tested for this release. - Linux 2.6.32-573.22.1.el6 GNU C (gcc), Fortran (gfortran), C++ (g++) - #1 SMP x86_64 GNU/Linux compilers: - (mayll/platypus) Version 4.4.7 20120313 - Version 4.8.4 - PGI C, Fortran, C++ for 64-bit target on - x86-64; - Version 16.10-0 - Intel(R) C (icc), C++ (icpc), Fortran (icc) - compilers: - Version 15.0.3.187 (Build 20150407) - MPICH 3.1.4 compiled with GCC 4.9.3 - Linux 2.6.32-573.18.1.el6.ppc64 MPICH mpich 3.1.4 compiled with #1 SMP ppc64 GNU/Linux IBM XL C/C++ for Linux, V13.1 (ostrich) and IBM XL Fortran for Linux, V15.1 @@ -290,4 +350,249 @@ The following platforms are not supported but have been tested for this release. Known Problems ============== +* "make check" fails on CYGWIN when building shared lib files is enabled. The + default on Cygwin has been changed to disable shared. It can be enabled with + the --enable-shared configure option but is likely to fail "make check" + with GCC compilers. (LK -2015/04/16) + +* CLANG compiler with the options -fcatch-undefined-behavior and -ftrapv + catches some undefined behavior in the alignment algorithm of the macro DETECT_I + in H5detect.c (Issue 8147). Since the algorithm is trying to detect the alignment + of integers, ideally the flag -fcatch-undefined-behavior shouldn't to be used for + H5detect.c. In the future, we can separate flags for H5detect.c from the rest of + the library. (SLU - 2013/10/16) + +* The 5.9 C++ compiler on Sun failed to compile a C++ test ttypes.cpp. It + complains with this message: + "/home/hdf5/src/H5Vprivate.h", line 130: Error: __func__ is not defined. + + The reason is that __func__ is a predefined identifier in C99 standard. The + HDF5 C library uses it in H5private.h. The test ttypes.cpp includes + H5private.h (H5Tpkg.h<-H5Fprivate.h<-H5Vprivate.h<-H5private.h). Sun's 5.9 + C++ compiler doesn't support __func__, thus fails to compile the C++ test. + But 5.11 C++ compiler does. To check whether your Sun C++ compiler knows this + identifier, try to compile the following simple C++ program: + #include + + int main(void) + { + printf("%s\n", __func__); + return 0; + } + (SLU - 2012/11/5) + +* The C++ and FORTRAN bindings are not currently working on FreeBSD with the + native release 8.2 compilers (4.2.1), but are working with gcc 4.6 from the + ports (and probably gcc releases after that). + (QAK - 2012/10/19) + +* The data conversion test dt_arith.c has failures (segmentation fault) from + "long double" to other datatypes during hard conversion when the library + is built with the default GCC 4.2.1 on Mac Lion system. It only happens + with optimization (-O3, -O2, and -O1). Some newer versions of GCC do not + have this problem. Users should disable optimization or try newer version + of GCC. (Issue 8017. SLU - 2012/6/12) + +* The data conversion test dt_arith.c fails in "long double" to integer + conversion on Ubuntu 11.10 (3.0.0.13 kernal) with GCC 4.6.1 if the library + is built with optimization -O3 or -O2. The older GCC (4.5) or newer kernal + (3.2.2 on Fedora) doesn't have the problem. Users should lower down the + optimization level (-O1 or -O0) by defining CFLAGS in the command line of + "configure" like: + + CFLAGS=-O1 ./configure + + It will overwrite the library's default optimization level. (Issue 7829. + SLU - 2012/2/7) + +* --with-mpe configure option does not work with Mpich2. AKC - 2011/03/10) + +* While working on the 1.8.6 release of HDF5, a bug was discovered that can + occur when reading from a dataset in parallel shortly after it has been + written to collectively. The issue was exposed by a new test in the parallel + HDF5 test suite, but had existed before that. We believe the problem lies with + certain MPI implementations and/or filesystems. + + We have provided a pure MPI test program, as well as a standalone HDF5 + program, that can be used to determine if this is an issue on your system. + They should be run across multiple nodes with a varying number of processes. + These programs can be found at: + http://www.hdfgroup.org/ftp/HDF5/examples/known_problems/ + +* Parallel mode in AIX will fail some of the testcheck_version.sh tests where + it treats "exit(134) the same as if process 0 had received an abort signal. + This is fixed and will be available in the next release. AKC - 2009/11/3 + +* The PathScale MPI implementation, accessing a Panasas file system, would + cause H5Fcreate() with H5F_ACC_EXCL to fail even when the file is not + existing. This is due to the MPI_File_open() call failing if the amode has + the MPI_MODE_EXCL bit set. (See bug 1468 for details.) AKC - 2009/8/11 + +* Parallel tests failed with 16 processes with data inconsistency at testphdf5 + / dataset_readAll. Parallel tests also failed with 32 and 64 processes with + collective abort of all ranks at t_posix_compliant / allwrite_allread_blocks + with MPI IO. (CMC - 2009/04/28) + +* For SNL, spirit/liberty/thunderbird: The serial tests pass but parallel + tests failed with MPI-IO file locking message. AKC - 2007/6/25. +* On Intel 64 Linux cluster (RH 4, Linux 2.6.9) with Intel 10.0 compilers use + -mp -O1 compilation flags to build the libraries. Higher level of optimization + causes failures in several HDF5 library tests. +* For HPUX 11.23 many tools tests failed for 64-bit version when linked to the + shared libraries (tested for 1.8.0-beta2) +* For SNL, Red Storm: only paralle HDF5 is supported. The serial tests pass + and the parallel tests also pass with lots of non-fatal error messages. +* on SUN 5.10 C++ test fails in the "Testing Shared Datatypes with Attributes" test +* configuring with --enable-debug=all produces compiler errors on most + platforms. Users who want to run HDF5 in debug mode should use + --enable-debug rather than --enable-debug=all to enable debugging + information on most modules. +* On Mac OS 10.4, test/dt_arith.c has some errors in conversion from long + double to (unsigned) long long and from (unsigned)long long to long double. +* On Altix SGI with Intel 9.0 testmeta.c would not compile with -O3 + optimization flag. +* On VAX, Scaleoffset filter isn't supported. The filter cannot be applied to + HDF5 data generated on VAX. Scaleoffset filter only supports IEEE standard + for floating-point data. +* On Cray X1, a lone colon on the command line of h5dump --xml (as in + the testh5dumpxml.sh script) is misinterpereted by the operating system + and causes an error. +* On mpich 1.2.5 and 1.2.6, we found that if more than two processes + contribute no IO and the application asks to do IO with collective, we found + that when using 4 processors, a simple collective write will be hung + sometimes. This can be verified with t_mpi test under testpar. +* The dataset created or rewritten with the v1.6.3 library or after can't + be read with the v1.6.2 library or before when Fletcher32 EDC(filter) is + enabled. There was a bug in the calculating code of the Fletcher32 + checksum in the library before v1.6.3. The checksum value wasn't consistent + between big-endian and little-endian systems. This bug was fixed in + Release 1.6.3. However, after fixing the bug, the checksum value is no + longer the same as before on little-endian system. The library release + after 1.6.4 can still read the dataset created or rewritten with the library + of v1.6.2 or before. SLU - 2005/6/30 +* For the version 6(6.02 and 6.04) of Portland Group compiler on AMD Opteron + processor, there's a bug in the compiler for optimization(-O2). The library + failed in several tests but all related to multi driver. The problem has + been reported to the vendor. +* On IBM AIX systems, parallel HDF5 mode will fail some tests with error + messages like "INFO: 0031-XXX ...". This is from the command poe. + Set the environment variable MP_INFOLEVEL to 0 to minimize the messages + and run the tests again. + The tests may fail with messages like "The socket name is already + in use". HDF5 does not use sockets (except for stream-VFD). This is + due to problems of the poe command trying to set up the debug socket. + Check if there are many old /tmp/s.pedb.* staying around. These are + sockets used by the poe command and left behind due to failed commands. + Ask your system administrator to clean them out. Lastly, request IBM + to provide a mean to run poe without the debug socket. + +* The C++ library's tests fails when compiling with PGI C++ compiler. The + workaround until the problem is correctly handled is to use the + flag "--instantiate=local" prior to the configure and build steps, as: + setenv CXX "pgCC --instantiate=local" for pgCC 5.02 and higher + + +* The stream-vfd test uses ip port 10007 for testing. If another + application is already using that port address, the test will hang + indefinitely and has to be terminated by the kill command. To try the + test again, change the port address in test/stream_test.c to one not + being used in the host. + +* The --enable-static-exec configure flag will only statically link libraries + if the static version of that library is present. If only the shared version + of a library exists (i.e., most system libraries on Solaris, AIX, and Mac, + for example, only have shared versions), the flag should still result in a + successful compilation, but note that the installed executables will not be + fully static. Thus, the only guarantee on these systems is that the + executable is statically linked with just the HDF5 library. + +* With the gcc 2.95.2 compiler, HDF 5 uses the `-ansi' flag during + compilation. The ANSI version of the compiler complains about not being + able to handle the `long long' datatype with the warning: + + warning: ANSI C does not support `long long' + + This warning is innocuous and can be safely ignored. + +* Certain platforms give false negatives when testing h5ls: + - Cray J90 and Cray T90IEEE give errors during testing when displaying + some floating-point values. These are benign differences due to + the different precision in the values displayed and h5ls appears to + be dumping floating-point numbers correctly. + +* Not all platforms behave correctly with szip's shared libraries. Szip is + disabled in these cases, and a message is relayed at configure time. Static + libraries should be working on all systems that support szip, and should be + used when shared libraries are unavailable. There is also a configure error + on Altix machines that incorrectly reports when a version of szip without + an encoder is being used. + +* On some platforms that use Intel and Absoft compilers to build HDF5 fortran library, + compilation may fail for fortranlib_test.f90, fflush1.f90 and fflush2.f90 + complaining about exit subroutine. Comment out the line + IF (total_error .ne. 0) CALL exit (total_error) + +* Information about building with PGI and Intel compilers is available in + INSTALL file sections 5.7 and 5.8 + +* On at least one system, (SDSC DataStar), the scheduler (in this case + LoadLeveler) sends job status updates to standard error when you run + any executable that was compiled with the parallel compilers. + + This causes problems when running "make check" on parallel builds, as + many of the tool tests function by saving the output from test runs, + and comparing it to an exemplar. + + The best solution is to reconfigure the target system so it no longer + inserts the extra text. However, this may not be practical. + + In such cases, one solution is to "setenv HDF5_Make_Ignore yes" prior to + the configure and build. This will cause "make check" to continue after + detecting errors in the tool tests. However, in the case of SDSC DataStar, + it also leaves you with some 150 "failed" tests to examine by hand. + + A second solution is to write a script to run serial tests and filter + out the text added by the scheduler. A sample script used on SDSC + DataStar is given below, but you will probably have to customize it + for your installation. + + Observe that the basic idea is to insert the script as the first item + on the command line which executes the the test. The script then + executes the test and filters out the offending text before passing + it on. + + #!/bin/csh + + set STDOUT_FILE=~/bin/serial_filter.stdout + set STDERR_FILE=~/bin/serial_filter.stderr + + rm -f $STDOUT_FILE $STDERR_FILE + + ($* > $STDOUT_FILE) >& $STDERR_FILE + + set RETURN_VALUE=$status + + cat $STDOUT_FILE + + tail +3 $STDERR_FILE + + exit $RETURN_VALUE + + You get the HDF make files and test scipts to execute your filter script + by setting the environment variable "RUNSERIAL" to the full path of the + script prior to running configure for parallel builds. Remember to + "unsetenv RUNSERIAL" before running configure for a serial build. + + Note that the RUNSERIAL environment variable exists so that we can + can prefix serial runs as necessary on the target system. On DataStar, + no prefix is necessary. However on an MPICH system, the prefix might + have to be set to something like "/usr/local/mpi/bin/mpirun -np 1" to + get the serial tests to run at all. + + In such cases, you will have to include the regular prefix in your + filter script. + +* H5Ocopy() does not copy reg_ref attributes correctly when shared-message + is turn on. The value of the reference in the destination attriubte is + wrong. This H5Ocopy problem will affect h5copy tool -- cgit v0.12 From 8eef625ef49ac53c6e3c6d41ca8ec3a620db981b Mon Sep 17 00:00:00 2001 From: lrknox Date: Thu, 30 Mar 2017 10:33:51 -0500 Subject: Add new file COPYING_LBNL_HDF5. --- MANIFEST | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST b/MANIFEST index 3d252eb..a19c850 100644 --- a/MANIFEST +++ b/MANIFEST @@ -26,6 +26,7 @@ ./.h5chkright.ini _DO_NOT_DISTRIBUTE_ ./ACKNOWLEDGMENTS ./COPYING +./COPYING_LBNL_HDF5 ./MANIFEST ./Makefile.dist ./Makefile.am -- cgit v0.12 From 194f604ad04ec976a1d761090aa892f9f28514c3 Mon Sep 17 00:00:00 2001 From: lrknox Date: Fri, 31 Mar 2017 12:17:50 -0500 Subject: Small corrections to DOPYING file. --- COPYING | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/COPYING b/COPYING index 51cb4e6..90aa75a 100644 --- a/COPYING +++ b/COPYING @@ -49,7 +49,7 @@ advised of the possibility of such damage. Limited portions of HDF5 were developed by Lawrence Berkeley National Laboratory (LBNL). LBNL's Copyright Notice and Licensing Terms can be found here: COPYING_LBNL_HDF5 file in this directory or at -http://support.hdfgroup.org/ftp/HDF5/releases/COPYING_LBNL_HDF5. +https://support.hdfgroup.org/ftp/HDF5/releases/COPYING_LBNL_HDF5. ----------------------------------------------------------------------------- ----------------------------------------------------------------------------- @@ -99,7 +99,7 @@ and/or accompanying materials: HDF5 is available with the SZIP compression library but SZIP is not part of HDF5 and has separate copyright and license terms. See “Szip Compression -in HDF Products” (www.hdfgroup.org/doc_resource/SZIP/) for further details. +in HDF Products” (https://support.hdfgroup.org/doc_resource/SZIP/) for further details. ----------------------------------------------------------------------------- -- cgit v0.12