diff options
48 files changed, 503 insertions, 123 deletions
diff --git a/BRANCH.txt b/BRANCH.txt new file mode 100644 index 0000000..9efaed8 --- /dev/null +++ b/BRANCH.txt @@ -0,0 +1,8 @@ +evict_on_close + +This branch is for the development of the "evict on close" feature +which will cause the library to evict all cached information +on object close. + +Dana Robinson is in charge of the branch, which follows the trunk. +It will be deleted when the feature is merged to the trunk. @@ -906,6 +906,7 @@ ./test/enc_dec_plist.c ./test/enc_dec_plist_cross_platform.c ./test/enum.c +./test/evict_on_close.c ./test/extend.c ./test/external.c ./test/error_test.c diff --git a/fortran/src/H5Eff.F90 b/fortran/src/H5Eff.F90 index 4198321..7a0b15b 100644 --- a/fortran/src/H5Eff.F90 +++ b/fortran/src/H5Eff.F90 @@ -141,9 +141,8 @@ CONTAINS INTEGER FUNCTION h5eprint_c2() BIND(C,NAME='h5eprint_c2') END FUNCTION h5eprint_c2 END INTERFACE - + namelen = LEN(NAME) IF (PRESENT(name)) THEN - namelen = LEN(NAME) hdferr = h5eprint_c1(name, namelen) ELSE hdferr = h5eprint_c2() diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90 index 532ecc6..e052ea0 100644 --- a/fortran/src/H5Pff.F90 +++ b/fortran/src/H5Pff.F90 @@ -6260,11 +6260,11 @@ SUBROUTINE h5pset_attr_phase_change_f(ocpl_id, max_compact, min_dense, hdferr) INTEGER(HID_T), INTENT(IN) :: type_id ! Datatype identifier of ! of fillvalue datatype ! (in memory) - CHARACTER(LEN=1), INTENT(IN), TARGET :: fillvalue ! Fillvalue + CHARACTER, INTENT(IN), TARGET :: fillvalue ! Fillvalue INTEGER, INTENT(OUT) :: hdferr ! Error code TYPE(C_PTR) :: f_ptr ! C address - f_ptr = C_LOC(fillvalue(1:1)) + f_ptr = C_LOC(fillvalue) hdferr = h5pset_fill_value_c(prp_id, type_id, f_ptr) END SUBROUTINE h5pset_fill_value_char @@ -6275,7 +6275,7 @@ SUBROUTINE h5pset_attr_phase_change_f(ocpl_id, max_compact, min_dense, hdferr) INTEGER(HID_T), INTENT(IN) :: type_id ! Datatype identifier of ! of fillvalue datatype ! (in memory) - CHARACTER(LEN=*), INTENT(OUT) :: fillvalue ! Fillvalue + CHARACTER, INTENT(OUT) :: fillvalue ! Fillvalue INTEGER, INTENT(OUT) :: hdferr ! Error code INTEGER :: i @@ -6286,7 +6286,7 @@ SUBROUTINE h5pset_attr_phase_change_f(ocpl_id, max_compact, min_dense, hdferr) ! To resolve Issue #1 outlined in the preamble of this file we ! need to pack the character string into an array. - chr_len = LEN(fillvalue(1:1)) + chr_len = LEN(fillvalue) ALLOCATE(chr(1:chr_len), STAT=hdferr) IF (hdferr .NE. 0) THEN hdferr = -1 @@ -2383,7 +2383,7 @@ done: *------------------------------------------------------------------------------ */ herr_t -H5AC_evict_tagged_metadata(H5F_t * f, haddr_t metadata_tag, hid_t dxpl_id) +H5AC_evict_tagged_metadata(H5F_t * f, haddr_t metadata_tag, hbool_t match_global, hid_t dxpl_id) { /* Variable Declarations */ herr_t ret_value = SUCCEED; @@ -2396,7 +2396,7 @@ H5AC_evict_tagged_metadata(H5F_t * f, haddr_t metadata_tag, hid_t dxpl_id) HDassert(f->shared); /* Call cache level function to evict metadata entries with specified tag */ - if(H5C_evict_tagged_entries(f, dxpl_id, metadata_tag) < 0) + if(H5C_evict_tagged_entries(f, dxpl_id, metadata_tag, match_global) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Cannot evict metadata") done: diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index d1d41c9..84c9e93 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -364,7 +364,7 @@ H5_DLL herr_t H5AC_validate_config(H5AC_cache_config_t *config_ptr); /* Tag & Ring routines */ H5_DLL herr_t H5AC_tag(hid_t dxpl_id, haddr_t metadata_tag, haddr_t *prev_tag); H5_DLL herr_t H5AC_flush_tagged_metadata(H5F_t * f, haddr_t metadata_tag, hid_t dxpl_id); -H5_DLL herr_t H5AC_evict_tagged_metadata(H5F_t * f, haddr_t metadata_tag, hid_t dxpl_id); +H5_DLL herr_t H5AC_evict_tagged_metadata(H5F_t * f, haddr_t metadata_tag, hbool_t match_global, hid_t dxpl_id); H5_DLL herr_t H5AC_retag_copied_metadata(const H5F_t *f, haddr_t metadata_tag); H5_DLL herr_t H5AC_ignore_tags(const H5F_t *f); H5_DLL herr_t H5AC_cork(H5F_t *f, haddr_t obj_addr, unsigned action, hbool_t *corked); diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index 975ea3e..4137edf 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -1970,7 +1970,7 @@ H5_DLL herr_t H5C_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, haddr_t addr, unsigned flags); H5_DLL herr_t H5C_flush_cache(H5F_t *f, hid_t dxpl_id, unsigned flags); H5_DLL herr_t H5C_flush_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag); -H5_DLL herr_t H5C_evict_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag); +H5_DLL herr_t H5C_evict_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag, hbool_t match_global); H5_DLL herr_t H5C_expunge_tag_type_metadata(H5F_t *f, hid_t dxpl_id, haddr_t tag, int type_id, unsigned flags); #if H5C_DO_TAGGING_SANITY_CHECKS herr_t H5C_verify_tag(int id, haddr_t tag, H5C_tag_globality_t globality); diff --git a/src/H5Ctag.c b/src/H5Ctag.c index d560e25..fc01ecb 100644 --- a/src/H5Ctag.c +++ b/src/H5Ctag.c @@ -352,7 +352,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5C_evict_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag) +H5C_evict_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag, hbool_t match_global) { H5C_t *cache; /* Pointer to cache structure */ H5C_tag_iter_evict_ctx_t ctx; /* Context for iterator callback */ @@ -379,8 +379,8 @@ H5C_evict_tagged_entries(H5F_t * f, hid_t dxpl_id, haddr_t tag) ctx.evicted_entries_last_pass = FALSE; /* Iterate through entries in the cache */ - if(H5C__iter_tagged_entries(cache, tag, TRUE, H5C__evict_tagged_entries_cb, &ctx) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_BADITER, FAIL, "Iteration of tagged entries failed") + if(H5C__iter_tagged_entries(cache, tag, match_global, H5C__evict_tagged_entries_cb, &ctx) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_BADITER, match_global, "Iteration of tagged entries failed") /* Keep doing this until we have stopped evicted entries */ } while(TRUE == ctx.evicted_entries_last_pass); @@ -308,37 +308,37 @@ done: /*------------------------------------------------------------------------- - * Function: H5Dclose + * Function: H5Dclose * - * Purpose: Closes access to a dataset (DATASET_ID) and releases - * resources used by it. It is illegal to subsequently use that - * same dataset ID in calls to other dataset functions. + * Purpose: Closes access to a dataset (DATASET_ID) and releases + * resources used by it. It is illegal to subsequently use that + * same dataset ID in calls to other dataset functions. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * - * Programmer: Robb Matzke - * Thursday, December 4, 1997 + * Programmer: Robb Matzke + * Thursday, December 4, 1997 * *------------------------------------------------------------------------- */ herr_t H5Dclose(hid_t dset_id) { - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE1("e", "i", dset_id); /* Check args */ if(NULL == H5I_object_verify(dset_id, H5I_DATASET)) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset") + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset") /* * Decrement the counter on the dataset. It will be freed if the count * reaches zero. */ if(H5I_dec_app_ref_always_close(dset_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTDEC, FAIL, "can't decrement count on dataset ID") + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEC, FAIL, "can't decrement count on dataset ID") done: FUNC_LEAVE_API(ret_value) diff --git a/src/H5Dint.c b/src/H5Dint.c index 8a00be2..5ae3bd3 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -1780,9 +1780,9 @@ done: herr_t H5D_close(H5D_t *dataset) { - hbool_t free_failed = FALSE; - hbool_t corked; /* Whether the dataset is corked or not */ - herr_t ret_value = SUCCEED; /* Return value */ + hbool_t free_failed = FALSE; /* Set if freeing sub-components failed */ + hbool_t corked; /* Whether the dataset is corked or not */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -1797,6 +1797,7 @@ H5D_close(H5D_t *dataset) dataset->shared->fo_count--; if(dataset->shared->fo_count == 0) { + /* Flush the dataset's information. Continue to close even if it fails. */ if(H5D__flush_real(dataset, H5AC_ind_read_dxpl_id) < 0) HDONE_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to flush cached dataset info") @@ -1891,12 +1892,12 @@ H5D_close(H5D_t *dataset) (H5O_msg_reset(H5O_FILL_ID, &dataset->shared->dcpl_cache.fill) < 0) || (H5O_msg_reset(H5O_EFL_ID, &dataset->shared->dcpl_cache.efl) < 0); - /* Uncork cache entries with object address tag */ - if(H5AC_cork(dataset->oloc.file, dataset->oloc.addr, H5AC__GET_CORKED, &corked) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve an object's cork status") - if(corked) - if(H5AC_cork(dataset->oloc.file, dataset->oloc.addr, H5AC__UNCORK, NULL) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTUNCORK, FAIL, "unable to uncork an object") + /* Uncork cache entries with object address tag */ + if(H5AC_cork(dataset->oloc.file, dataset->oloc.addr, H5AC__GET_CORKED, &corked) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve an object's cork status") + if(corked) + if(H5AC_cork(dataset->oloc.file, dataset->oloc.addr, H5AC__UNCORK, NULL) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTUNCORK, FAIL, "unable to uncork an object") /* * Release datatype, dataspace and creation property list -- there isn't @@ -1917,6 +1918,21 @@ H5D_close(H5D_t *dataset) if(H5O_close(&(dataset->oloc)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release object header") + /* Evict dataset metadata if evicting on close */ + if(H5F_SHARED(dataset->oloc.file) && H5F_EVICT_ON_CLOSE(dataset->oloc.file)) { +// printf("EVICTING DATASET (TAG: 0x%3llx)\n", dataset->oloc.addr); +// printf("DUMPING CACHE - BEFORE FLUSH\n"); +// H5AC_dump_cache(dataset->oloc.file); + if(H5AC_flush_tagged_metadata(dataset->oloc.file, dataset->oloc.addr, H5AC_ind_read_dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush tagged metadata") +// printf("DUMPING CACHE - BETWEEN FLUSH AND EVICT\n"); +// H5AC_dump_cache(dataset->oloc.file); + if(H5AC_evict_tagged_metadata(dataset->oloc.file, dataset->oloc.addr, FALSE, H5AC_ind_read_dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to evict tagged metadata") +// printf("DUMPING CACHE - AFTER EVICT\n"); +// H5AC_dump_cache(dataset->oloc.file); + } /* end if */ + /* * Free memory. Before freeing the memory set the file pointer to NULL. * We always check for a null file pointer in other H5D functions to be @@ -1924,8 +1940,8 @@ H5D_close(H5D_t *dataset) * above). */ dataset->oloc.file = NULL; - dataset->shared = H5FL_FREE(H5D_shared_t, dataset->shared); + } /* end if */ else { /* Decrement the ref. count for this object in the top file */ @@ -1943,16 +1959,16 @@ H5D_close(H5D_t *dataset) HGOTO_ERROR(H5E_DATASET, H5E_CANTRELEASE, FAIL, "problem attempting to free location") } /* end else */ - /* Release the dataset's path info */ - if(H5G_name_free(&(dataset->path)) < 0) - free_failed = TRUE; + /* Release the dataset's path info */ + if(H5G_name_free(&(dataset->path)) < 0) + free_failed = TRUE; /* Free the dataset's memory structure */ dataset = H5FL_FREE(H5D_t, dataset); /* Check if anything failed in the middle... */ if(free_failed) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "couldn't free a component of the dataset, but the dataset was freed anyway.") + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "couldn't free a component of the dataset, but the dataset was freed anyway.") done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Fint.c b/src/H5Fint.c index 80c593b..5b7fdda 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -954,6 +954,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_class_t *drvr; /*file driver class info */ H5P_genplist_t *a_plist; /*file access property list */ H5F_close_degree_t fc_degree; /*file close degree */ + hbool_t evict_on_close; /* evict on close value from plist */ H5F_t *ret_value = NULL; /*actual return value */ FUNC_ENTER_NOAPI(NULL) @@ -1118,6 +1119,21 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "file close degree doesn't match") } /* end if */ + /* Record the evict-on-close MDC behavior. If it's the first time opening + * the file, set it to access property list value; if it's the second time + * or later, verify that the access property list value matches the value + * in shared file structure. + */ + if(H5P_get(a_plist, H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME, &evict_on_close) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get evict on close value") + + if(shared->nrefs == 1) { + shared->evict_on_close = evict_on_close; + } else if(shared->nrefs > 1) { + if(shared->evict_on_close != evict_on_close) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "file evict-on-close value doesn't match") + } /* end if */ + /* Formulate the absolute path for later search of target file for external links */ if(H5_build_extpath(name, &file->extpath) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "unable to build extpath") diff --git a/src/H5Fio.c b/src/H5Fio.c index b9bd354..2ccd3f3 100644 --- a/src/H5Fio.c +++ b/src/H5Fio.c @@ -271,7 +271,7 @@ H5F_evict_tagged_metadata(H5F_t * f, haddr_t tag, hid_t dxpl_id) f->shared->sblock = NULL; /* Evict the object's metadata */ - if(H5AC_evict_tagged_metadata(f, tag, dxpl_id)<0) + if(H5AC_evict_tagged_metadata(f, tag, TRUE, dxpl_id) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTEXPUNGE, FAIL, "unable to evict tagged metadata") /* Re-read the superblock. */ diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index c5aed3b..8ec2a93 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -261,6 +261,7 @@ struct H5F_file_t { /* not change thereafter. */ hid_t fcpl_id; /* File creation property list ID */ H5F_close_degree_t fc_degree; /* File close behavior degree */ + hbool_t evict_on_close; /* If the file's objects should be evicted from the metadata cache on close */ size_t rdcc_nslots; /* Size of raw data chunk cache (slots) */ size_t rdcc_nbytes; /* Size of raw data chunk cache (bytes) */ double rdcc_w0; /* Preempt read chunks first? [0.0..1.0]*/ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index b5c24e9..40e5ff2 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -303,6 +303,7 @@ #define H5F_SET_SOHM_NINDEXES(F, N) ((F)->shared->sohm_nindexes = (N)) #define H5F_FCPL(F) ((F)->shared->fcpl_id) #define H5F_GET_FC_DEGREE(F) ((F)->shared->fc_degree) +#define H5F_EVICT_ON_CLOSE(F) ((F)->shared->evict_on_close) #define H5F_RDCC_NSLOTS(F) ((F)->shared->rdcc_nslots) #define H5F_RDCC_NBYTES(F) ((F)->shared->rdcc_nbytes) #define H5F_RDCC_W0(F) ((F)->shared->rdcc_w0) @@ -348,6 +349,7 @@ #define H5F_SET_SOHM_NINDEXES(F, N) (H5F_set_sohm_nindexes((F), (N))) #define H5F_FCPL(F) (H5F_get_fcpl(F)) #define H5F_GET_FC_DEGREE(F) (H5F_get_fc_degree(F)) +#define H5F_EVICT_ON_CLOSE(F) (H5F_get_evict_on_close(F)) #define H5F_RDCC_NSLOTS(F) (H5F_rdcc_nslots(F)) #define H5F_RDCC_NBYTES(F) (H5F_rdcc_nbytes(F)) #define H5F_RDCC_W0(F) (H5F_rdcc_w0(F)) @@ -463,6 +465,7 @@ #define H5F_ACS_EFC_SIZE_NAME "efc_size" /* Size of external file cache */ #define H5F_ACS_FILE_IMAGE_INFO_NAME "file_image_info" /* struct containing initial file image and callback info */ #define H5F_ACS_CORE_WRITE_TRACKING_FLAG_NAME "core_write_tracking_flag" /* Whether or not core VFD backing store write tracking is enabled */ +#define H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME "evict_on_close_flag" /* Whether or not the metadata cache will evict objects on close */ #define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_NAME "core_write_tracking_page_size" /* The page size in kiB when core VFD write tracking is enabled */ #define H5F_ACS_COLL_MD_WRITE_FLAG_NAME "collective_metadata_write" /* property indicating whether metadata writes are done collectively or not */ @@ -660,6 +663,7 @@ H5_DLL unsigned H5F_get_sohm_nindexes(const H5F_t *f); H5_DLL herr_t H5F_set_sohm_nindexes(H5F_t *f, unsigned nindexes); H5_DLL hid_t H5F_get_fcpl(const H5F_t *f); H5_DLL H5F_close_degree_t H5F_get_fc_degree(const H5F_t *f); +H5_DLL hbool_t H5F_get_evict_on_close(const H5F_t *f); H5_DLL size_t H5F_rdcc_nbytes(const H5F_t *f); H5_DLL size_t H5F_rdcc_nslots(const H5F_t *f); H5_DLL double H5F_rdcc_w0(const H5F_t *f); diff --git a/src/H5Fquery.c b/src/H5Fquery.c index dd6e8e3..f59a9b7 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -835,6 +835,33 @@ H5F_get_fc_degree(const H5F_t *f) /*------------------------------------------------------------------------- + * Function: H5F_get_evict_on_close + * + * Purpose: Retrieve the 'file close degree' for the file. + * + * Return: Success: Flag indicating whether the evict-on-close + * property was set for the file. + * Failure: (can't happen) + * + * Programmer: Dana Robinson + * Spring 2016 + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_get_evict_on_close(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->evict_on_close) +} /* end H5F_get_evict_on_close() */ + + +/*------------------------------------------------------------------------- * Function: H5F_store_msg_crt_idx * * Purpose: Retrieve the 'store message creation index' flag for the file. @@ -715,14 +715,14 @@ done: herr_t H5Gclose(hid_t group_id) { - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE1("e", "i", group_id); /* Check args */ if(NULL == H5I_object_verify(group_id,H5I_GROUP)) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a group") + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a group") /* * Decrement the counter on the group atom. It will be freed if the count diff --git a/src/H5Gint.c b/src/H5Gint.c index f5bccc3..ff92ecc 100644 --- a/src/H5Gint.c +++ b/src/H5Gint.c @@ -486,7 +486,7 @@ H5G_close(H5G_t *grp) if(0 == grp->shared->fo_count) { HDassert(grp != H5G_rootof(H5G_fileof(grp))); - /* Uncork cache entries with object address tag */ + /* Uncork cache entries with object address tag */ if(H5AC_cork(grp->oloc.file, grp->oloc.addr, H5AC__GET_CORKED, &corked) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTGET, FAIL, "unable to retrieve an object's cork status") if(corked) @@ -500,6 +500,23 @@ H5G_close(H5G_t *grp) HGOTO_ERROR(H5E_SYM, H5E_CANTRELEASE, FAIL, "can't remove group from list of open objects") if(H5O_close(&(grp->oloc)) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to close") + + /* Evict group metadata if evicting on close */ + if(H5F_SHARED(grp->oloc.file) && H5F_EVICT_ON_CLOSE(grp->oloc.file)) { +// printf("EVICTING GROUP (TAG: 0x%3llx)\n", grp->oloc.addr); +// printf("DUMPING CACHE - BEFORE FLUSH\n"); +// H5AC_dump_cache(grp->oloc.file); + if(H5AC_flush_tagged_metadata(grp->oloc.file, grp->oloc.addr, H5AC_ind_read_dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush tagged metadata") +// printf("DUMPING CACHE - BETWEEN FLUSH AND EVICT\n"); +// H5AC_dump_cache(grp->oloc.file); + if(H5AC_evict_tagged_metadata(grp->oloc.file, grp->oloc.addr, FALSE, H5AC_ind_read_dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to evict tagged metadata") +// printf("DUMPING CACHE - AFTER EVICT\n"); +// H5AC_dump_cache(grp->oloc.file); + } /* end if */ + + /* Free memory */ grp->shared = H5FL_FREE(H5G_shared_t, grp->shared); } else { /* Decrement the ref. count for this object in the top file */ @@ -1051,7 +1051,7 @@ done: * the same effect as calling H5Gclose, H5Dclose, or H5Tclose. * * Return: Success: Non-negative - * Failure: Negative + * Failure: Negative * * Programmer: James Laird * July 14 2006 @@ -1069,14 +1069,18 @@ H5Oclose(hid_t object_id) /* Get the type of the object and close it in the correct way */ switch(H5I_get_type(object_id)) { case H5I_GROUP: - case H5I_DATATYPE: case H5I_DATASET: if(H5I_object(object_id) == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a valid object") if(H5I_dec_app_ref(object_id) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTRELEASE, FAIL, "unable to close object") break; - + case H5I_DATATYPE: + if(H5I_object(object_id) == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a valid object") + if(H5T_close_id(object_id) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTRELEASE, FAIL, "unable to close object") + break; case H5I_UNINIT: case H5I_BADID: case H5I_FILE: diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 96de39c..d96bd54 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -181,6 +181,11 @@ /* Definition for object flush callback */ #define H5F_ACS_OBJECT_FLUSH_CB_SIZE sizeof(H5F_object_flush_t) #define H5F_ACS_OBJECT_FLUSH_CB_DEF {NULL, NULL} +/* Definition for evict on close property */ +#define H5F_ACS_EVICT_ON_CLOSE_FLAG_SIZE sizeof(hbool_t) +#define H5F_ACS_EVICT_ON_CLOSE_FLAG_DEF FALSE +#define H5F_ACS_EVICT_ON_CLOSE_FLAG_ENC H5P__encode_hbool_t +#define H5F_ACS_EVICT_ON_CLOSE_FLAG_DEC H5P__decode_hbool_t #ifdef H5_HAVE_PARALLEL /* Definition of collective metadata read mode flag */ #define H5F_ACS_COLL_MD_READ_FLAG_SIZE sizeof(H5P_coll_md_read_flag_t) @@ -296,6 +301,7 @@ static const H5FD_file_image_info_t H5F_def_file_image_info_g = H5F_ACS_FILE_IMA static const hbool_t H5F_def_core_write_tracking_flag_g = H5F_ACS_CORE_WRITE_TRACKING_FLAG_DEF; /* Default setting for core VFD write tracking */ static const size_t H5F_def_core_write_tracking_page_size_g = H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEF; /* Default core VFD write tracking page size */ static const H5F_object_flush_t H5F_def_object_flush_cb_g = H5F_ACS_OBJECT_FLUSH_CB_DEF; /* Default setting for object flush callback */ +static const hbool_t H5F_def_evict_on_close_flag_g = H5F_ACS_EVICT_ON_CLOSE_FLAG_DEF; /* Default setting for evict on close property */ #ifdef H5_HAVE_PARALLEL static const H5P_coll_md_read_flag_t H5F_def_coll_md_read_flag_g = H5F_ACS_COLL_MD_READ_FLAG_DEF; /* Default setting for the collective metedata read flag */ static const hbool_t H5F_def_coll_md_write_flag_g = H5F_ACS_COLL_MD_WRITE_FLAG_DEF; /* Default setting for the collective metedata write flag */ @@ -462,6 +468,12 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the evict on close flag */ + if(H5P_register_real(pclass, H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME, H5F_ACS_EVICT_ON_CLOSE_FLAG_SIZE, &H5F_def_evict_on_close_flag_g, + NULL, NULL, NULL, H5F_ACS_EVICT_ON_CLOSE_FLAG_ENC, H5F_ACS_EVICT_ON_CLOSE_FLAG_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + #ifdef H5_HAVE_PARALLEL /* Register the metadata collective read flag */ if(H5P_register_real(pclass, H5_COLL_MD_READ_FLAG_NAME, H5F_ACS_COLL_MD_READ_FLAG_SIZE, &H5F_def_coll_md_read_flag_g, @@ -3605,6 +3617,81 @@ done: FUNC_LEAVE_API(ret_value) } /* H5Pget_obj_flush_cb() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_evict_on_close + * + * Purpose: + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Spring 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_evict_on_close(hid_t fapl_id, hbool_t evict_on_close) +{ + H5P_genplist_t *plist; /* property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ib", fapl_id, evict_on_close); + + /* Compare the property list's class against the other class */ + if(TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not a file access plist") + + /* Get the plist structure */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Set values */ + if(H5P_set(plist, H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME, &evict_on_close) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set evict on close property") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_evict_on_close() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_evict_on_close + * + * Purpose: + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Spring 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_evict_on_close(hid_t fapl_id, hbool_t *evict_on_close) +{ + H5P_genplist_t *plist; /* property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*b", fapl_id, evict_on_close); + + /* Compare the property list's class against the other class */ + if(TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not an access plist") + + /* Get the plist structure */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + if(H5P_get(plist, H5F_ACS_EVICT_ON_CLOSE_FLAG_NAME, evict_on_close) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get evict on close property") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_evict_on_close() */ + #ifdef H5_HAVE_PARALLEL /*------------------------------------------------------------------------- diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 0318c8f..b807a37 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -353,6 +353,8 @@ H5_DLL herr_t H5Pset_core_write_tracking(hid_t fapl_id, hbool_t is_enabled, size H5_DLL herr_t H5Pget_core_write_tracking(hid_t fapl_id, hbool_t *is_enabled, size_t *page_size); H5_DLL herr_t H5Pset_object_flush_cb(hid_t plist_id, H5F_flush_cb_t func, void *udata); H5_DLL herr_t H5Pget_object_flush_cb(hid_t plist_id, H5F_flush_cb_t *func, void **udata); +H5_DLL herr_t H5Pset_evict_on_close(hid_t fapl_id, hbool_t evict_on_close); +H5_DLL herr_t H5Pget_evict_on_close(hid_t fapl_id, hbool_t *evict_on_close); #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5Pset_all_coll_metadata_ops(hid_t plist_id, hbool_t is_collective); H5_DLL herr_t H5Pget_all_coll_metadata_ops(hid_t plist_id, hbool_t *is_collective); @@ -1696,41 +1696,95 @@ done: /*------------------------------------------------------------------------- - * Function: H5Tclose + * Function: H5Tclose * - * Purpose: Frees a datatype and all associated memory. + * Purpose: Frees a datatype and all associated memory. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * - * Programmer: Robb Matzke - * Tuesday, December 9, 1997 - * - * Modifications: + * Programmer: Robb Matzke + * Tuesday, December 9, 1997 * *------------------------------------------------------------------------- */ herr_t H5Tclose(hid_t type_id) { - H5T_t *dt; /* Pointer to datatype to close */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE1("e", "i", type_id); + /* Call internal function */ + if(H5T_close_id(type_id) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTRELEASE, FAIL, "unable to close object") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Tclose() */ + + +/*------------------------------------------------------------------------- + * Function: H5T_close_id + * + * Purpose: Internal function to free a datatype and all associated + * memory. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Summer 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +H5T_close_id(hid_t type_id) +{ + H5T_t *dt; /* Pointer to datatype to close */ + H5F_t *file = NULL; /* File */ + hbool_t evict = FALSE; /* Evict metadata on close? */ + haddr_t tag = HADDR_UNDEF; /* Metadata tag for evictions */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + /* Check args */ if(NULL == (dt = (H5T_t *)H5I_object_verify(type_id, H5I_DATATYPE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype") + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype") if(H5T_STATE_IMMUTABLE == dt->shared->state) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "immutable datatype") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "immutable datatype") + + /* Check if this is the last object reference and we'll be evicting + * on close. We need to cache this info since it will be gone after the + * decrement call frees the struct. + */ + file = dt->oloc.file; + if(file && 1 == dt->shared->fo_count && H5F_EVICT_ON_CLOSE(file)) { + evict = TRUE; + tag = dt->oloc.addr; + } /* end if */ /* When the reference count reaches zero the resources are freed */ if(H5I_dec_app_ref(type_id) < 0) - HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "problem freeing id") + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "problem freeing id") + + /* Clean up metadata in the metadata cache if evicting on close */ + if(evict && H5F_SHARED(file)) { +// printf("DUMPING CACHE - BEFORE FLUSH\n"); +// H5AC_dump_cache(file); + if(H5AC_flush_tagged_metadata(file, tag, H5AC_ind_read_dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush tagged metadata") +// printf("DUMPING CACHE - BETWEEN FLUSH AND EVICT\n"); +// H5AC_dump_cache(file); + if(H5AC_evict_tagged_metadata(file, tag, FALSE, H5AC_ind_read_dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to evict tagged metadata") +// printf("DUMPING CACHE - AFTER EVICT\n"); +// H5AC_dump_cache(file); + } /* end if */ done: - FUNC_LEAVE_API(ret_value) -} /* end H5Tclose() */ + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5T_close_id() */ /*------------------------------------------------------------------------- diff --git a/src/H5Tprivate.h b/src/H5Tprivate.h index 7efcb41..5fd0cf3 100644 --- a/src/H5Tprivate.h +++ b/src/H5Tprivate.h @@ -108,6 +108,7 @@ H5_DLL herr_t H5T_init(void); H5_DLL H5T_t *H5T_copy(H5T_t *old_dt, H5T_copy_t method); H5_DLL herr_t H5T_lock(H5T_t *dt, hbool_t immutable); H5_DLL herr_t H5T_close(H5T_t *dt); +H5_DLL herr_t H5T_close_id(hid_t type_id); H5_DLL H5T_t *H5T_get_super(const H5T_t *dt); H5_DLL H5T_class_t H5T_get_class(const H5T_t *dt, htri_t internal); H5_DLL htri_t H5T_detect_class(const H5T_t *dt, H5T_class_t cls, hbool_t from_api); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1ab4165..6e6a0b9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -204,6 +204,7 @@ set (H5_TESTS ohdr stab gheap + evict_on_close farray earray btree2 diff --git a/test/CMakeTests.cmake b/test/CMakeTests.cmake index ca890ba..1e7940a 100644 --- a/test/CMakeTests.cmake +++ b/test/CMakeTests.cmake @@ -513,6 +513,7 @@ set (H5TEST_TESTS ohdr stab gheap + evict_on_close farray earray btree2 @@ -865,6 +866,7 @@ if (HDF5_TEST_VFD) ohdr stab gheap + evict_on_close pool # accum farray diff --git a/test/Makefile.am b/test/Makefile.am index 9e0f9c5..7d062f4 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -42,7 +42,7 @@ check_SCRIPTS = $(TEST_SCRIPT) # As an exception, long-running tests should occur earlier in the list. # This gives them more time to run when tests are executing in parallel. TEST_PROG= testhdf5 cache cache_api cache_tagging lheap ohdr stab gheap \ - farray earray btree2 fheap \ + evict_on_close farray earray btree2 fheap \ pool accum hyperslab istore bittests dt_arith \ dtypes dsets cmpd_dset filter_fail extend external efc objcopy links unlink \ big mtime fillval mount flush1 flush2 app_ref enum \ diff --git a/test/evict_on_close.c b/test/evict_on_close.c new file mode 100644 index 0000000..bcbd030 --- /dev/null +++ b/test/evict_on_close.c @@ -0,0 +1,144 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Dana Robinson + * Spring 2016 + * + * Purpose: Tests the basic operation of the evict-on-close cache + * behavior. Tests that ensure the tagging is handled correctly + * are located in cache.c. + */ + +#include "h5test.h" + +static herr_t check_evict_on_close_api(void); + + +/*------------------------------------------------------------------------- + * Function: check_evict_on_close_api() + * + * Purpose: Verify that the H5Pset/get_evict_on_close() calls behave + * correctly. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Spring 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +check_evict_on_close_api(void) +{ + hid_t fapl_id = -1; + hid_t dapl_id = -1; + hbool_t evict_on_close; + herr_t status; + + TESTING("evict on close API"); + + /* Create a fapl */ + if((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + FAIL_STACK_ERROR; + + /* Check the default */ + evict_on_close = TRUE; + if(H5Pget_evict_on_close(fapl_id, &evict_on_close) < 0) + FAIL_STACK_ERROR; + if(evict_on_close != FALSE) + FAIL_PUTS_ERROR("Incorrect default evict on close value."); + + /* Set the evict on close property */ + evict_on_close = TRUE; + if(H5Pset_evict_on_close(fapl_id, evict_on_close) < 0) + FAIL_STACK_ERROR; + + /* Make sure we can get it back out */ + evict_on_close = FALSE; + if(H5Pget_evict_on_close(fapl_id, &evict_on_close) < 0) + FAIL_STACK_ERROR; + if(evict_on_close != TRUE) + FAIL_PUTS_ERROR("Incorrect evict on close value."); + + /* close fapl */ + if(H5Pclose(fapl_id) < 0) + FAIL_STACK_ERROR; + + /**********************************************/ + /* Trying passing in a non-fapl property list */ + /**********************************************/ + + if((dapl_id = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR; + + /* ensure using an incorrect access plist fails */ + H5E_BEGIN_TRY { + status = H5Pset_evict_on_close(dapl_id, evict_on_close); + } H5E_END_TRY; + if(status >= 0) + FAIL_PUTS_ERROR("H5Pset_evict_on_close() accepted invalid access plist."); + + /* ensure an invalid plist fails */ + H5E_BEGIN_TRY { + status = H5Pget_evict_on_close((hid_t)-1, &evict_on_close); + } H5E_END_TRY; + if(status >= 0) + FAIL_PUTS_ERROR("H5Pget_evict_on_close() accepted invalid hid_t."); + + /* close dapl */ + if(H5Pclose(dapl_id) < 0) + FAIL_STACK_ERROR; + + PASSED(); + return SUCCEED; + +error: + H5_FAILED(); + return FAIL; + +} /* check_evict_on_close_api() */ + + +/*------------------------------------------------------------------------- + * Function: main + * + * Return: EXIT_FAILURE/EXIT_SUCCESS + * + * Programmer: Dana Robinson + * Spring 2016 + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + unsigned nerrors = 0; + + printf("Testing evict-on-close cache behavior.\n"); + + nerrors += check_evict_on_close_api() < 0 ? 1 : 0; + + if(nerrors) { + printf("***** %u evict-on-close test%s FAILED! *****\n", + nerrors, nerrors > 1 ? "S" : ""); + return EXIT_FAILURE; + } + + printf("All evict-on-close tests passed.\n"); + + return EXIT_SUCCESS; +} + diff --git a/tools/h5format_convert/testfiles/h5fc_ext1_f.ddl b/tools/h5format_convert/testfiles/h5fc_ext1_f.ddl index dae9284..fb5192d 100644 --- a/tools/h5format_convert/testfiles/h5fc_ext1_f.ddl +++ b/tools/h5format_convert/testfiles/h5fc_ext1_f.ddl @@ -1,4 +1,4 @@ -HDF5 "./testfiles/dmptmp.h5" { +HDF5 "./testfiles/tmp.h5" { SUPER_BLOCK { SUPERBLOCK_VERSION 2 FREELIST_VERSION 0 diff --git a/tools/h5format_convert/testfiles/h5fc_ext1_i.ddl b/tools/h5format_convert/testfiles/h5fc_ext1_i.ddl index 8ec4656..2fff4ac 100644 --- a/tools/h5format_convert/testfiles/h5fc_ext1_i.ddl +++ b/tools/h5format_convert/testfiles/h5fc_ext1_i.ddl @@ -1,4 +1,4 @@ -HDF5 "./testfiles/dmptmp.h5" { +HDF5 "./testfiles/tmp.h5" { SUPER_BLOCK { SUPERBLOCK_VERSION 2 FREELIST_VERSION 0 diff --git a/tools/h5format_convert/testfiles/h5fc_ext1_s.ddl b/tools/h5format_convert/testfiles/h5fc_ext1_s.ddl index dae9284..fb5192d 100644 --- a/tools/h5format_convert/testfiles/h5fc_ext1_s.ddl +++ b/tools/h5format_convert/testfiles/h5fc_ext1_s.ddl @@ -1,4 +1,4 @@ -HDF5 "./testfiles/dmptmp.h5" { +HDF5 "./testfiles/tmp.h5" { SUPER_BLOCK { SUPERBLOCK_VERSION 2 FREELIST_VERSION 0 diff --git a/tools/h5format_convert/testfiles/h5fc_ext2_if.ddl b/tools/h5format_convert/testfiles/h5fc_ext2_if.ddl index 8ec4656..2fff4ac 100644 --- a/tools/h5format_convert/testfiles/h5fc_ext2_if.ddl +++ b/tools/h5format_convert/testfiles/h5fc_ext2_if.ddl @@ -1,4 +1,4 @@ -HDF5 "./testfiles/dmptmp.h5" { +HDF5 "./testfiles/tmp.h5" { SUPER_BLOCK { SUPERBLOCK_VERSION 2 FREELIST_VERSION 0 diff --git a/tools/h5format_convert/testfiles/h5fc_ext2_is.ddl b/tools/h5format_convert/testfiles/h5fc_ext2_is.ddl index 8ec4656..2fff4ac 100644 --- a/tools/h5format_convert/testfiles/h5fc_ext2_is.ddl +++ b/tools/h5format_convert/testfiles/h5fc_ext2_is.ddl @@ -1,4 +1,4 @@ -HDF5 "./testfiles/dmptmp.h5" { +HDF5 "./testfiles/tmp.h5" { SUPER_BLOCK { SUPERBLOCK_VERSION 2 FREELIST_VERSION 0 diff --git a/tools/h5format_convert/testfiles/h5fc_ext2_sf.ddl b/tools/h5format_convert/testfiles/h5fc_ext2_sf.ddl index dae9284..fb5192d 100644 --- a/tools/h5format_convert/testfiles/h5fc_ext2_sf.ddl +++ b/tools/h5format_convert/testfiles/h5fc_ext2_sf.ddl @@ -1,4 +1,4 @@ -HDF5 "./testfiles/dmptmp.h5" { +HDF5 "./testfiles/tmp.h5" { SUPER_BLOCK { SUPERBLOCK_VERSION 2 FREELIST_VERSION 0 diff --git a/tools/h5format_convert/testfiles/h5fc_ext3_isf.ddl b/tools/h5format_convert/testfiles/h5fc_ext3_isf.ddl index 8ec4656..2fff4ac 100644 --- a/tools/h5format_convert/testfiles/h5fc_ext3_isf.ddl +++ b/tools/h5format_convert/testfiles/h5fc_ext3_isf.ddl @@ -1,4 +1,4 @@ -HDF5 "./testfiles/dmptmp.h5" { +HDF5 "./testfiles/tmp.h5" { SUPER_BLOCK { SUPERBLOCK_VERSION 2 FREELIST_VERSION 0 diff --git a/tools/h5format_convert/testfiles/h5fc_v_all.ddl b/tools/h5format_convert/testfiles/h5fc_v_all.ddl index a1af831..5e7365d 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_all.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_all.ddl @@ -1,5 +1,5 @@ Process command line options -Open the file outtmp.h5 +Open the file tmp.h5 Processing all datasets in the file... Going to process dataset:/DSET_CONTIGUOUS... Open the dataset diff --git a/tools/h5format_convert/testfiles/h5fc_v_bt1.ddl b/tools/h5format_convert/testfiles/h5fc_v_bt1.ddl index 31de12a..c501eb0 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_bt1.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_bt1.ddl @@ -1,5 +1,5 @@ Process command line options -Open the file outtmp.h5 +Open the file tmp.h5 Going to process dataset: /GROUP/DSET_BT2... Open the dataset Retrieve the dataset's layout diff --git a/tools/h5format_convert/testfiles/h5fc_v_err.ddl b/tools/h5format_convert/testfiles/h5fc_v_err.ddl index b671db0..4a728e8 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_err.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_err.ddl @@ -1,5 +1,5 @@ Process command line options -Open the file outtmp.h5 +Open the file tmp.h5 Processing all datasets in the file... Going to process dataset:/DSET_ERR... Open the dataset diff --git a/tools/h5format_convert/testfiles/h5fc_v_n_1d.ddl b/tools/h5format_convert/testfiles/h5fc_v_n_1d.ddl index fcdadd8..ff5da4a 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_n_1d.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_n_1d.ddl @@ -1,6 +1,6 @@ Process command line options It is noop... -Open the file outtmp.h5 +Open the file tmp.h5 Going to process dataset: /DSET_EA... Open the dataset Retrieve the dataset's layout diff --git a/tools/h5format_convert/testfiles/h5fc_v_n_all.ddl b/tools/h5format_convert/testfiles/h5fc_v_n_all.ddl index 074ce6f..d2ffbbf 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_n_all.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_n_all.ddl @@ -1,6 +1,6 @@ Process command line options It is noop... -Open the file outtmp.h5 +Open the file tmp.h5 Processing all datasets in the file... Going to process dataset:/DSET_CONTIGUOUS... Open the dataset diff --git a/tools/h5format_convert/testfiles/h5fc_v_ndata_bt1.ddl b/tools/h5format_convert/testfiles/h5fc_v_ndata_bt1.ddl index c75699a..ba794a7 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_ndata_bt1.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_ndata_bt1.ddl @@ -1,6 +1,6 @@ Process command line options It is noop... -Open the file outtmp.h5 +Open the file tmp.h5 Going to process dataset: /DSET_NDATA_BT2... Open the dataset Retrieve the dataset's layout diff --git a/tools/h5format_convert/testfiles/h5fc_v_non_chunked.ddl b/tools/h5format_convert/testfiles/h5fc_v_non_chunked.ddl index 5945389..aba0740 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_non_chunked.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_non_chunked.ddl @@ -1,5 +1,5 @@ Process command line options -Open the file outtmp.h5 +Open the file tmp.h5 Going to process dataset: /DSET_CONTIGUOUS... Open the dataset Retrieve the dataset's layout diff --git a/tools/h5format_convert/testfiles/old_h5fc_ext1_f.ddl b/tools/h5format_convert/testfiles/old_h5fc_ext1_f.ddl index dae9284..fb5192d 100644 --- a/tools/h5format_convert/testfiles/old_h5fc_ext1_f.ddl +++ b/tools/h5format_convert/testfiles/old_h5fc_ext1_f.ddl @@ -1,4 +1,4 @@ -HDF5 "./testfiles/dmptmp.h5" { +HDF5 "./testfiles/tmp.h5" { SUPER_BLOCK { SUPERBLOCK_VERSION 2 FREELIST_VERSION 0 diff --git a/tools/h5format_convert/testfiles/old_h5fc_ext1_i.ddl b/tools/h5format_convert/testfiles/old_h5fc_ext1_i.ddl index d1768c8..c906082 100644 --- a/tools/h5format_convert/testfiles/old_h5fc_ext1_i.ddl +++ b/tools/h5format_convert/testfiles/old_h5fc_ext1_i.ddl @@ -1,4 +1,4 @@ -HDF5 "./testfiles/dmptmp.h5" { +HDF5 "./testfiles/tmp.h5" { SUPER_BLOCK { SUPERBLOCK_VERSION 1 FREELIST_VERSION 0 diff --git a/tools/h5format_convert/testfiles/old_h5fc_ext1_s.ddl b/tools/h5format_convert/testfiles/old_h5fc_ext1_s.ddl index dae9284..fb5192d 100644 --- a/tools/h5format_convert/testfiles/old_h5fc_ext1_s.ddl +++ b/tools/h5format_convert/testfiles/old_h5fc_ext1_s.ddl @@ -1,4 +1,4 @@ -HDF5 "./testfiles/dmptmp.h5" { +HDF5 "./testfiles/tmp.h5" { SUPER_BLOCK { SUPERBLOCK_VERSION 2 FREELIST_VERSION 0 diff --git a/tools/h5format_convert/testfiles/old_h5fc_ext2_if.ddl b/tools/h5format_convert/testfiles/old_h5fc_ext2_if.ddl index 8ec4656..2fff4ac 100644 --- a/tools/h5format_convert/testfiles/old_h5fc_ext2_if.ddl +++ b/tools/h5format_convert/testfiles/old_h5fc_ext2_if.ddl @@ -1,4 +1,4 @@ -HDF5 "./testfiles/dmptmp.h5" { +HDF5 "./testfiles/tmp.h5" { SUPER_BLOCK { SUPERBLOCK_VERSION 2 FREELIST_VERSION 0 diff --git a/tools/h5format_convert/testfiles/old_h5fc_ext2_is.ddl b/tools/h5format_convert/testfiles/old_h5fc_ext2_is.ddl index 8ec4656..2fff4ac 100644 --- a/tools/h5format_convert/testfiles/old_h5fc_ext2_is.ddl +++ b/tools/h5format_convert/testfiles/old_h5fc_ext2_is.ddl @@ -1,4 +1,4 @@ -HDF5 "./testfiles/dmptmp.h5" { +HDF5 "./testfiles/tmp.h5" { SUPER_BLOCK { SUPERBLOCK_VERSION 2 FREELIST_VERSION 0 diff --git a/tools/h5format_convert/testfiles/old_h5fc_ext2_sf.ddl b/tools/h5format_convert/testfiles/old_h5fc_ext2_sf.ddl index dae9284..fb5192d 100644 --- a/tools/h5format_convert/testfiles/old_h5fc_ext2_sf.ddl +++ b/tools/h5format_convert/testfiles/old_h5fc_ext2_sf.ddl @@ -1,4 +1,4 @@ -HDF5 "./testfiles/dmptmp.h5" { +HDF5 "./testfiles/tmp.h5" { SUPER_BLOCK { SUPERBLOCK_VERSION 2 FREELIST_VERSION 0 diff --git a/tools/h5format_convert/testfiles/old_h5fc_ext3_isf.ddl b/tools/h5format_convert/testfiles/old_h5fc_ext3_isf.ddl index 8ec4656..2fff4ac 100644 --- a/tools/h5format_convert/testfiles/old_h5fc_ext3_isf.ddl +++ b/tools/h5format_convert/testfiles/old_h5fc_ext3_isf.ddl @@ -1,4 +1,4 @@ -HDF5 "./testfiles/dmptmp.h5" { +HDF5 "./testfiles/tmp.h5" { SUPER_BLOCK { SUPERBLOCK_VERSION 2 FREELIST_VERSION 0 diff --git a/tools/h5format_convert/testh5fc.sh.in b/tools/h5format_convert/testh5fc.sh.in index 0d74697..000425b 100644 --- a/tools/h5format_convert/testh5fc.sh.in +++ b/tools/h5format_convert/testh5fc.sh.in @@ -69,10 +69,7 @@ TESTDIR=./testfiles test -d $TESTDIR || mkdir $TESTDIR # Copy the testfile to a temporary file for testing as h5format_convert is changing the file in place -TMPOUTFILE=outtmp.h5 TMPFILE=tmp.h5 -TMPCHKFILE=chktmp.h5 -TMPDMPFILE=dmptmp.h5 ###################################################################### # test files @@ -173,7 +170,7 @@ CLEAN_TESTFILES_AND_TESTDIR() # skip rm if srcdir is same as destdir # this occurs when build/test performed in source dir and # make cp fail - SDIR=$SRC_H5FORMCONV_TESTFILES + SDIR=`$DIRNAME $tstfile` INODE_SDIR=`$LS -i -d $SDIR | $AWK -F' ' '{print $1}'` INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'` if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then @@ -216,11 +213,11 @@ TOOLTEST_OUT() { actual_err_sav=${actual_err}-sav # Prepare the test file - $RM $TESTDIR/$TMPOUTFILE + $RM $TESTDIR/$TMPFILE TFILE=$2 if [ ! -z "$2" ] && [ -e $TESTDIR/$2 ] ; then - $CP $TESTDIR/$2 $TESTDIR/$TMPOUTFILE - TFILE=$TMPOUTFILE + $CP $TESTDIR/$2 $TESTDIR/$TMPFILE + TFILE=$TMPFILE fi # Run test. @@ -248,15 +245,14 @@ TOOLTEST_OUT() { # $1 is the test file name # --fname exists # --fname is copied to a temporary file for testing -# $2 is the temporary file name -# $3 to at most $5--options to the tool such as: +# $2 to at most $4--options to the tool such as: # -d dname # -n TOOLTEST() { - TESTING $FORMCONV $3 $4 $5 $1 - $RM $TESTDIR/$2 - $CP $TESTDIR/$1 $TESTDIR/$2 - $RUNSERIAL $FORMCONV_BIN $3 $4 $5 $TESTDIR/$2 + TESTING $FORMCONV $2 $3 $4 $1 + $RM $TESTDIR/$TMPFILE + $CP $TESTDIR/$1 $TESTDIR/$TMPFILE + $RUNSERIAL $FORMCONV_BIN $2 $3 $4 $TESTDIR/$TMPFILE exitcode=$? if [ $exitcode -ne 0 ]; then echo "*FAILED*" @@ -276,7 +272,7 @@ CHECKING() { # $1 dataset name IDX_CHECK() { CHECKING $1 - $RUNSERIAL $CHK_IDX_BIN $TESTDIR/$TMPCHKFILE $1 + $RUNSERIAL $CHK_IDX_BIN $TESTDIR/$TMPFILE $1 ret=$? if [ $ret -eq 0 ]; then echo " PASSED" @@ -308,7 +304,7 @@ H5DUMP_CHECK() { expect="$TESTDIR/$2" actual="$TESTDIR/`basename $2 .ddl`.out" actual_err="$TESTDIR/`basename $2 .ddl`.err" - $RUNSERIAL $H5DUMP_BIN -BH $TESTDIR/$TMPDMPFILE > $actual 2>$actual_err + $RUNSERIAL $H5DUMP_BIN -BH $TESTDIR/$TMPFILE > $actual 2>$actual_err cat $actual_err >> $actual # Compare output @@ -391,28 +387,28 @@ TOOLTEST_OUT h5fc_v_err.ddl h5fc_err_level.h5 -v # h5format_convert -d /GROUP/DSET_FA h5fc_ext_none.h5 # h5format_convert -d /DSET_NONE h5fc_ext_none.h5 # h5format_convert -d /GROUP/DSET_NDATA_NONE h5fc_ext_none.h5 -TOOLTEST h5fc_ext_none.h5 $TMPCHKFILE -d /DSET_EA +TOOLTEST h5fc_ext_none.h5 -d /DSET_EA IDX_CHECK /DSET_EA # -TOOLTEST h5fc_ext_none.h5 $TMPCHKFILE -d /GROUP/DSET_NDATA_EA +TOOLTEST h5fc_ext_none.h5 -d /GROUP/DSET_NDATA_EA IDX_CHECK /GROUP/DSET_NDATA_EA # -TOOLTEST h5fc_ext_none.h5 $TMPCHKFILE -d /GROUP/DSET_BT2 +TOOLTEST h5fc_ext_none.h5 -d /GROUP/DSET_BT2 IDX_CHECK /GROUP/DSET_BT2 # -TOOLTEST h5fc_ext_none.h5 $TMPCHKFILE -d /DSET_NDATA_BT2 +TOOLTEST h5fc_ext_none.h5 -d /DSET_NDATA_BT2 IDX_CHECK /DSET_NDATA_BT2 # -TOOLTEST h5fc_ext_none.h5 $TMPCHKFILE -d /DSET_FA +TOOLTEST h5fc_ext_none.h5 -d /DSET_FA IDX_CHECK /DSET_FA # -TOOLTEST h5fc_ext_none.h5 $TMPCHKFILE -d /GROUP/DSET_NDATA_FA +TOOLTEST h5fc_ext_none.h5 -d /GROUP/DSET_NDATA_FA IDX_CHECK /GROUP/DSET_NDATA_FA # -TOOLTEST h5fc_ext_none.h5 $TMPCHKFILE -d /DSET_NONE +TOOLTEST h5fc_ext_none.h5 -d /DSET_NONE IDX_CHECK /DSET_NONE # -TOOLTEST h5fc_ext_none.h5 $TMPCHKFILE -d /GROUP/DSET_NDATA_NONE +TOOLTEST h5fc_ext_none.h5 -d /GROUP/DSET_NDATA_NONE IDX_CHECK /GROUP/DSET_NDATA_NONE # # @@ -420,16 +416,16 @@ IDX_CHECK /GROUP/DSET_NDATA_NONE # No output from tests: just check exit code # h5format_convert -d /DSET_NDATA_BT2 old_h5fc_ext_none.h5 (v1-btree dataset) # h5format_convert -d /DSET_CONTIGUOUS h5fc_non_v3.h5 (non-chunked dataset) -TOOLTEST old_h5fc_ext_none.h5 $TMPFILE -d /DSET_NDATA_BT2 -TOOLTEST h5fc_non_v3.h5 $TMPFILE -d /DSET_CONTIGUOUS +TOOLTEST old_h5fc_ext_none.h5 -d /DSET_NDATA_BT2 +TOOLTEST h5fc_non_v3.h5 -d /DSET_CONTIGUOUS # # # # No output from tests: just check exit code # h5format_convert -d /GROUP/DSET_BT2 -n h5fc_non_v3.h5 (noop, one dataset) # h5format_convert -n h5fc_non_v3.h5 (noop, all datasets) -TOOLTEST h5fc_non_v3.h5 $TMPFILE -d /GROUP/DSET_BT2 -n -TOOLTEST h5fc_non_v3.h5 $TMPFILE -n +TOOLTEST h5fc_non_v3.h5 -d /GROUP/DSET_BT2 -n +TOOLTEST h5fc_non_v3.h5 -n # # # @@ -437,7 +433,7 @@ TOOLTEST h5fc_non_v3.h5 $TMPFILE -n # h5format_convert h5fc_non_v3.h5 # 1) convert all datasets # 2) verify indexing types -TOOLTEST h5fc_non_v3.h5 $TMPCHKFILE +TOOLTEST h5fc_non_v3.h5 IDX_CHECK /DSET_NDATA_EA IDX_CHECK /DSET_NDATA_BT2 IDX_CHECK /GROUP/DSET_BT2 @@ -449,47 +445,47 @@ IDX_CHECK /GROUP/DSET_EA # h5format_convert h5fc_edge_v3.h5 # 1) convert the chunked dataset (filter, no-filter-edge-chunk) # 2) verify the indexing type -TOOLTEST h5fc_edge_v3.h5 $TMPCHKFILE +TOOLTEST h5fc_edge_v3.h5 IDX_CHECK /DSET_EDGE # # # The following test files have messages in the superblock extension. # Verify h5dump output for correctness after conversion -TOOLTEST h5fc_ext1_i.h5 $TMPDMPFILE +TOOLTEST h5fc_ext1_i.h5 H5DUMP_CHECK h5fc_ext1_i.h5 h5fc_ext1_i.ddl -TOOLTEST h5fc_ext1_s.h5 $TMPDMPFILE +TOOLTEST h5fc_ext1_s.h5 H5DUMP_CHECK h5fc_ext1_s.h5 h5fc_ext1_s.ddl -TOOLTEST h5fc_ext1_f.h5 $TMPDMPFILE +TOOLTEST h5fc_ext1_f.h5 H5DUMP_CHECK h5fc_ext1_f.h5 h5fc_ext1_f.ddl # -TOOLTEST h5fc_ext2_if.h5 $TMPDMPFILE +TOOLTEST h5fc_ext2_if.h5 H5DUMP_CHECK h5fc_ext2_if.h5 h5fc_ext2_if.ddl -TOOLTEST h5fc_ext2_is.h5 $TMPDMPFILE +TOOLTEST h5fc_ext2_is.h5 H5DUMP_CHECK h5fc_ext2_is.h5 h5fc_ext2_is.ddl -TOOLTEST h5fc_ext2_sf.h5 $TMPDMPFILE +TOOLTEST h5fc_ext2_sf.h5 H5DUMP_CHECK h5fc_ext2_sf.h5 h5fc_ext2_sf.ddl # -TOOLTEST h5fc_ext3_isf.h5 $TMPDMPFILE +TOOLTEST h5fc_ext3_isf.h5 H5DUMP_CHECK h5fc_ext3_isf.h5 h5fc_ext3_isf.ddl # # # -TOOLTEST old_h5fc_ext1_i.h5 $TMPDMPFILE +TOOLTEST old_h5fc_ext1_i.h5 H5DUMP_CHECK old_h5fc_ext1_i.h5 old_h5fc_ext1_i.ddl -TOOLTEST old_h5fc_ext1_s.h5 $TMPDMPFILE +TOOLTEST old_h5fc_ext1_s.h5 H5DUMP_CHECK old_h5fc_ext1_s.h5 old_h5fc_ext1_s.ddl -TOOLTEST old_h5fc_ext1_f.h5 $TMPDMPFILE +TOOLTEST old_h5fc_ext1_f.h5 H5DUMP_CHECK old_h5fc_ext1_f.h5 old_h5fc_ext1_f.ddl # -TOOLTEST old_h5fc_ext2_if.h5 $TMPDMPFILE +TOOLTEST old_h5fc_ext2_if.h5 H5DUMP_CHECK old_h5fc_ext2_if.h5 old_h5fc_ext2_if.ddl -TOOLTEST old_h5fc_ext2_is.h5 $TMPDMPFILE +TOOLTEST old_h5fc_ext2_is.h5 H5DUMP_CHECK old_h5fc_ext2_is.h5 old_h5fc_ext2_is.ddl -TOOLTEST old_h5fc_ext2_sf.h5 $TMPDMPFILE +TOOLTEST old_h5fc_ext2_sf.h5 H5DUMP_CHECK old_h5fc_ext2_sf.h5 old_h5fc_ext2_sf.ddl # -TOOLTEST old_h5fc_ext3_isf.h5 $TMPDMPFILE +TOOLTEST old_h5fc_ext3_isf.h5 H5DUMP_CHECK old_h5fc_ext3_isf.h5 old_h5fc_ext3_isf.ddl # # Clean up temporary files/directories |