summaryrefslogtreecommitdiffstats
path: root/src/H5Fsuper.c
diff options
context:
space:
mode:
Diffstat (limited to 'src/H5Fsuper.c')
-rw-r--r--src/H5Fsuper.c252
1 files changed, 196 insertions, 56 deletions
diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c
index 5896e32..97a6d6f 100644
--- a/src/H5Fsuper.c
+++ b/src/H5Fsuper.c
@@ -23,6 +23,7 @@
/***********/
#include "H5private.h" /* Generic Functions */
#include "H5ACprivate.h" /* Metadata cache */
+#include "H5CXprivate.h" /* API Contexts */
#include "H5Eprivate.h" /* Error handling */
#include "H5Fpkg.h" /* File access */
#include "H5FDprivate.h" /* File drivers */
@@ -59,7 +60,6 @@ static herr_t H5F__update_super_ext_driver_msg(H5F_t *f);
/* Package Variables */
/*********************/
-
/*****************************/
/* Library Private Variables */
/*****************************/
@@ -72,6 +72,12 @@ H5FL_DEFINE(H5F_super_t);
/* Local Variables */
/*******************/
+/* Format version bounds for superblock */
+static const unsigned HDF5_superblock_ver_bounds[] = {
+ HDF5_SUPERBLOCK_VERSION_DEF, /* H5F_LIBVER_EARLIEST */
+ HDF5_SUPERBLOCK_VERSION_2, /* H5F_LIBVER_V18 */
+ HDF5_SUPERBLOCK_VERSION_LATEST /* H5F_LIBVER_LATEST */
+};
/*-------------------------------------------------------------------------
@@ -316,7 +322,7 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
-H5F__super_read(H5F_t *f, hbool_t initial_read)
+H5F__super_read(H5F_t *f, H5P_genplist_t *fa_plist, hbool_t initial_read)
{
H5AC_ring_t orig_ring = H5AC_RING_INV;
H5F_super_t * sblock = NULL; /* Superblock structure */
@@ -328,11 +334,10 @@ H5F__super_read(H5F_t *f, hbool_t initial_read)
haddr_t eof; /* End of file address */
unsigned rw_flags; /* Read/write permissions for file */
hbool_t skip_eof_check = FALSE; /* Whether to skip checking the EOF value */
- herr_t ret_value = SUCCEED; /* Return value */
#ifdef H5_HAVE_PARALLEL
- int mpi_rank = 0, mpi_size = 1;
- int mpi_result;
+ int mpi_size = 1;
#endif /* H5_HAVE_PARALLEL */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_PACKAGE_TAG(H5AC__SUPERBLOCK_TAG)
@@ -346,14 +351,9 @@ H5F__super_read(H5F_t *f, hbool_t initial_read)
/* Find the superblock */
#ifdef H5_HAVE_PARALLEL
- if(H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) {
-
- if((mpi_rank = H5F_mpi_get_rank(f)) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "Can't get MPI rank")
-
+ if(H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI))
if((mpi_size = H5F_mpi_get_size(f)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't retrieve MPI communicator size")
- }
/* If we are an MPI application with at least two processes, the
* following superblock signature location optimization is applicable.
@@ -365,17 +365,36 @@ H5F__super_read(H5F_t *f, hbool_t initial_read)
* our MPI rank and size.
*/
if(mpi_size > 1) {
- MPI_Comm this_comm = MPI_COMM_NULL;
+ MPI_Comm this_comm;
+ int mpi_rank;
+ int mpi_result;
- if(0 == mpi_rank)
- if(H5FD_locate_signature(file, &super_addr) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "unable to locate file signature")
+ /* Sanity check */
HDassert(H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI));
+ /* Set up MPI info */
+ if((mpi_rank = H5F_mpi_get_rank(f)) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "Can't get MPI rank")
if(MPI_COMM_NULL == (this_comm = H5F_mpi_get_comm(f)))
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get MPI communicator")
-
- if(MPI_SUCCESS != (mpi_result = MPI_Bcast(&super_addr,sizeof(super_addr), MPI_BYTE, 0, this_comm)))
+
+ /* Search for the file's signature only with rank 0 process */
+ if(0 == mpi_rank) {
+ herr_t status;
+
+ /* Try detecting file's siganture */
+ /* (Don't leave before Bcast, to avoid hang on error) */
+ H5E_BEGIN_TRY {
+ status = H5FD_locate_signature(file, &super_addr);
+ } H5E_END_TRY;
+
+ /* Set superblock address to undefined on error */
+ if(status < 0)
+ super_addr = HADDR_UNDEF;
+ } /* end if */
+
+ /* Broadcast superblock address to other processes */
+ if(MPI_SUCCESS != (mpi_result = MPI_Bcast(&super_addr, sizeof(super_addr), MPI_BYTE, 0, this_comm)))
HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_result)
} /* end if */
else {
@@ -388,7 +407,7 @@ H5F__super_read(H5F_t *f, hbool_t initial_read)
#ifdef H5_HAVE_PARALLEL
} /* end else */
#endif /* H5_HAVE_PARALLEL */
- if(HADDR_UNDEF == super_addr)
+ if(!H5F_addr_defined(super_addr))
HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "file signature not found")
/* Check for userblock present */
@@ -435,13 +454,46 @@ H5F__super_read(H5F_t *f, hbool_t initial_read)
if(NULL == (sblock = (H5F_super_t *)H5AC_protect(f, H5AC_SUPERBLOCK, (haddr_t)0, &udata, rw_flags)))
HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "unable to load superblock")
- if(H5F_INTENT(f) & H5F_ACC_SWMR_WRITE)
- if(sblock->super_vers < HDF5_SUPERBLOCK_VERSION_3)
- HGOTO_ERROR(H5E_FILE, H5E_CANTPROTECT, FAIL, "invalid superblock version for SWMR_WRITE")
+ /*
+ * When opening a file with SWMR-write access, the library will first
+ * check to ensure that superblock version 3 is used. Otherwise fail
+ * file open.
+ *
+ * Then the library will upgrade the file's low_bound depending on
+ * superblock version as follows:
+ * --version 0 or 1: no change to low_bound
+ * --version 2: upgrade low_bound to at least V18
+ * --version 3: upgrade low_bound to at least V110
+ *
+ * Upgrading low_bound will give the best format versions available for
+ * that superblock version. Due to the possible upgrade, the fapl returned
+ * from H5Fget_access_plist() might indicate a low_bound higher than what
+ * the user originally set.
+ *
+ * After upgrading low_bound, the library will check to ensure that the
+ * superblock version does not exceed the version allowed by high_bound.
+ * Otherise fail file open.
+ *
+ * For details, please see RFC:Setting Bounds for Object Creation in HDF5 1.10.0.
+ */
+
+ /* Check to ensure that superblock version 3 is used for SWMR-write access */
+ if(H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) {
+ if(sblock->super_vers < HDF5_SUPERBLOCK_VERSION_3)
+ HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "superblock version for SWMR is less than 3")
+ }
+
+ /* Upgrade low_bound to at least V18 when encountering version 2 superblock */
+ if(sblock->super_vers == HDF5_SUPERBLOCK_VERSION_2)
+ f->shared->low_bound = MAX(H5F_LIBVER_V18, f->shared->low_bound);
- /* Enable all latest version support when file has v3 superblock */
+ /* Upgrade low_bound to at least V110 when encountering version 3 superblock */
if(sblock->super_vers >= HDF5_SUPERBLOCK_VERSION_3)
- f->shared->latest_flags |= H5F_LATEST_ALL_FLAGS;
+ f->shared->low_bound = MAX(H5F_LIBVER_V110, f->shared->low_bound);
+
+ /* Version bounds check */
+ if(sblock->super_vers > HDF5_superblock_ver_bounds[f->shared->high_bound])
+ HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "superblock version exceeds high bound")
/* Pin the superblock in the cache */
if(H5AC_pin_protected_entry(sblock) < 0)
@@ -540,6 +592,17 @@ H5F__super_read(H5F_t *f, hbool_t initial_read)
* as the file can appear truncated if only part of it has been
* been flushed to disk by the SWMR writer process.
*/
+ /* The EOF check is also skipped when the private property
+ * H5F_ACS_SKIP_EOF_CHECK_NAME exists in the fapl.
+ * This property is enabled by the tool h5clear with these
+ * two options: (1) --filesize (2) --increment
+ */
+
+ /* Check if this private property exists in fapl */
+ if(H5P_exist_plist(fa_plist, H5F_ACS_SKIP_EOF_CHECK_NAME) > 0)
+ if(H5P_get(fa_plist, H5F_ACS_SKIP_EOF_CHECK_NAME, &skip_eof_check) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get skip EOF check value")
+
if(H5F_INTENT(f) & H5F_ACC_SWMR_READ) {
/*
* When the file is opened for SWMR read access, skip the check if:
@@ -698,7 +761,6 @@ H5F__super_read(H5F_t *f, hbool_t initial_read)
if((status = H5O_msg_exists(&ext_loc, H5O_FSINFO_ID)) < 0)
HGOTO_ERROR(H5E_FILE, H5E_EXISTS, FAIL, "unable to read object header")
if(status) {
- H5O_fsinfo_t fsinfo; /* File space info message from superblock extension */
uint8_t flags; /* Message flags */
/* Get message flags */
@@ -707,6 +769,15 @@ H5F__super_read(H5F_t *f, hbool_t initial_read)
/* If message is NOT marked "unknown"--set up file space info */
if(!(flags & H5O_MSG_FLAG_WAS_UNKNOWN)) {
+ H5O_fsinfo_t fsinfo; /* File space info message from superblock extension */
+ hbool_t null_fsm_addr = FALSE; /* Whether to drop free-space to the floor */
+
+ /* The h5clear tool uses this property to tell the library
+ * to drop free-space to the floor
+ */
+ if(H5P_exist_plist(fa_plist, H5F_ACS_NULL_FSM_ADDR_NAME) > 0)
+ if(H5P_get(fa_plist, H5F_ACS_NULL_FSM_ADDR_NAME, &null_fsm_addr) < 0)
+ HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get clearance for persisting fsm addr")
/* Retrieve the 'file space info' structure */
if(NULL == H5O_msg_read(&ext_loc, H5O_FSINFO_ID, &fsinfo))
@@ -751,21 +822,50 @@ H5F__super_read(H5F_t *f, hbool_t initial_read)
if(f->shared->eoa_pre_fsm_fsalloc != fsinfo.eoa_pre_fsm_fsalloc)
f->shared->eoa_pre_fsm_fsalloc = fsinfo.eoa_pre_fsm_fsalloc;
- /* f->shared->eoa_pre_fsm_fsalloc must always be HADDR_UNDEF
- * in the absence of persistant free space managers.
+ /* f->shared->eoa_pre_fsm_fsalloc must always be HADDR_UNDEF
+ * in the absence of persistant free space managers.
+ */
+ /* If the following two conditions are true:
+ * (1) skipping EOF check (skip_eof_check)
+ * (2) dropping free-space to the floor (null_fsm_addr)
+ * skip the asserts as "eoa_pre_fsm_fsalloc" may be undefined
+ * for a crashed file with persistant free space managers.
+ * #1 and #2 are enabled when the tool h5clear --increment
+ * option is used.
*/
- HDassert((!f->shared->fs_persist) || (f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF));
- HDassert(!f->shared->first_alloc_dealloc);
+ if(!skip_eof_check && !null_fsm_addr) {
+ HDassert((!f->shared->fs_persist) || (f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF));
+ HDassert(!f->shared->first_alloc_dealloc);
+ } /* end if */
- if((f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF) &&
- (H5F_INTENT(f) & H5F_ACC_RDWR))
+ /* As "eoa_pre_fsm_fsalloc" may be undefined for a crashed file
+ * with persistant free space managers, therefore, set
+ * "first_alloc_dealloc" when the condition
+ * "dropping free-space to the floor is true.
+ * This will ensure that no action is done to settle things on file
+ * close via H5MF_settle_meta_data_fsm() and H5MF_settle_raw_data_fsm().
+ */
+ if((f->shared->eoa_pre_fsm_fsalloc != HADDR_UNDEF || null_fsm_addr) &&
+ (H5F_INTENT(f) & H5F_ACC_RDWR))
f->shared->first_alloc_dealloc = TRUE;
f->shared->fs_addr[0] = HADDR_UNDEF;
for(u = 1; u < NELMTS(f->shared->fs_addr); u++)
f->shared->fs_addr[u] = fsinfo.fs_addr[u - 1];
- if(fsinfo.mapped && (rw_flags & H5AC__READ_ONLY_FLAG) == 0) {
+ /* If the following two conditions are true:
+ * (1) file is persisting free-space
+ * (2) dropping free-space to the floor (null_fsm_addr)
+ * nullify the addresses of the FSMs
+ */
+ if(f->shared->fs_persist && null_fsm_addr)
+ for(u = 0; u < NELMTS(fsinfo.fs_addr); u++)
+ f->shared->fs_addr[u] = fsinfo.fs_addr[u] = HADDR_UNDEF;
+
+ /* For fsinfo.mapped: remove the FSINFO message from the superblock extension
+ and write a new message to the extension */
+ /* For null_fsm_addr: just update FSINFO message in the superblock extension */
+ if(((fsinfo.mapped || null_fsm_addr) && (rw_flags & H5AC__READ_ONLY_FLAG) == 0)) {
/* Do the same kluge until we know for sure. VC */
#if 1 /* bug fix test code -- tidy this up if all goes well */ /* JRM */
@@ -779,16 +879,22 @@ H5F__super_read(H5F_t *f, hbool_t initial_read)
f->shared->sblock = sblock;
#endif /* JRM */
- if(H5F__super_ext_remove_msg(f, H5O_FSINFO_ID) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTDELETE, FAIL, "error in removing message from superblock extension")
-
- if(H5F__super_ext_write_msg(f, H5O_FSINFO_ID, &fsinfo, TRUE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing fsinfo message to superblock extension")
+ if(null_fsm_addr) {
+ if(H5F__super_ext_write_msg(f, H5O_FSINFO_ID, &fsinfo, FALSE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing fsinfo message to superblock extension")
+ } /* end if */
+ else {
+ if(H5F__super_ext_remove_msg(f, H5O_FSINFO_ID) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTDELETE, FAIL, "error in removing message from superblock extension")
+
+ if(H5F__super_ext_write_msg(f, H5O_FSINFO_ID, &fsinfo, TRUE, H5O_MSG_FLAG_MARK_IF_UNKNOWN) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing fsinfo message to superblock extension")
+ } /* end else */
#if 1 /* bug fix test code -- tidy this up if all goes well */ /* JRM */
f->shared->sblock = NULL;
#endif /* JRM */
- }
+ } /* end if */
} /* end if not marked "unknown" */
} /* end if */
@@ -998,14 +1104,42 @@ H5F__super_init(H5F_t *f)
f->shared->fs_page_size == H5F_FILE_SPACE_PAGE_SIZE_DEF))
non_default_fs_settings = TRUE;
- /* Bump superblock version if latest superblock version support is enabled */
- if(H5F_USE_LATEST_FLAGS(f, H5F_LATEST_SUPERBLOCK))
- super_vers = HDF5_SUPERBLOCK_VERSION_LATEST;
- /* Bump superblock version to use version 3 superblock for SWMR writing */
- else if((H5F_INTENT(f) & H5F_ACC_SWMR_WRITE))
+ /*
+ * When creating a file with write access, the library will:
+ * -- set superblock version to 0, 1 or 2 based on feature enabled
+ * -- no change to low_bound
+ * When creating a file with SWMR-write access, the library will:
+ * (See explanation (#) below.)
+ * -- set superblock version to 3
+ * -- upgrade low_bound to at least V110
+ *
+ * Then the library will finalize superblock version to that allowed by
+ * low_bound if that is higher.
+ * Lastly, the library will check to ensure the superblock version does not
+ * exceed the version allowed by high_bound. Otherwise fail file open.
+ *
+ * For details, please see RFC:Setting Bounds for Object Creation in HDF5 1.10.0.
+ *
+ * (#)
+ * Version 3 superblock is introduced in 1.10 for SWMR due to the problem of
+ * the status_flags field in the superblock. The problem is discussed in
+ * jira issue SWMR-79 and also in the RFC: File Format Changes in HDF5 1.10.0.
+ * The file's low_bound is upgraded for SWMR so that the library will
+ * use the best format versions available in 1.10.
+ * Due to the possible upgrade, the fapl returned from H5Fget_access_plist()
+ * might indicate a low_bound higher than what the user originally set.
+ */
+
+ /*
+ * Creating a file with SWMR-write access will
+ * upgrade superblock version and low_bound
+ */
+ if(H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) {
super_vers = HDF5_SUPERBLOCK_VERSION_3;
+ f->shared->low_bound = MAX(H5F_LIBVER_V110, f->shared->low_bound);
+
/* Bump superblock version to create superblock extension for SOHM info */
- else if(f->shared->sohm_nindexes > 0)
+ } else if(f->shared->sohm_nindexes > 0)
super_vers = HDF5_SUPERBLOCK_VERSION_2;
/*
* Bump superblock version to create superblock extension for:
@@ -1023,6 +1157,13 @@ H5F__super_init(H5F_t *f)
else if(sblock->btree_k[H5B_CHUNK_ID] != HDF5_BTREE_CHUNK_IK_DEF)
super_vers = HDF5_SUPERBLOCK_VERSION_1;
+ /* Finalize superblock version to that allowed by the file's low bound if higher */
+ super_vers = MAX(super_vers, HDF5_superblock_ver_bounds[f->shared->low_bound]);
+
+ /* Version bounds check */
+ if(super_vers > HDF5_superblock_ver_bounds[f->shared->high_bound])
+ HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "superblock version out of bounds")
+
/* If a newer superblock version is required, set it here */
if(super_vers != HDF5_SUPERBLOCK_VERSION_DEF) {
H5P_genplist_t *c_plist; /* Property list */
@@ -1047,8 +1188,8 @@ H5F__super_init(H5F_t *f)
/* Sanity check the userblock size vs. the file's allocation alignment */
if(userblock_size > 0) {
- /* Set up the alignment to use for page or aggr fs */
- hsize_t alignment = H5F_PAGED_AGGR(f) ? f->shared->fs_page_size : f->shared->alignment;
+ /* Set up the alignment to use for page or aggr fs */
+ hsize_t alignment = H5F_PAGED_AGGR(f) ? f->shared->fs_page_size : f->shared->alignment;
if(userblock_size < alignment)
HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "userblock size must be > file object alignment")
@@ -1362,10 +1503,9 @@ H5F_eoa_dirty(H5F_t *f)
HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark drvinfo as dirty")
} /* end if */
/* If the driver info is stored as a message, update that instead */
- else if(f->shared->drvinfo_sb_msg_exists) {
+ else if(f->shared->drvinfo_sb_msg_exists)
if(H5F__update_super_ext_driver_msg(f) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark drvinfo message as dirty")
- } /* end if */
done:
FUNC_LEAVE_NOAPI(ret_value)
@@ -1536,8 +1676,8 @@ H5F__super_ext_write_msg(H5F_t *f, unsigned id, void *mesg,
/* Open/create the superblock extension object header */
if(H5F_addr_defined(f->shared->sblock->ext_addr)) {
- if(H5F_super_ext_open(f, f->shared->sblock->ext_addr, &ext_loc) < 0)
- HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, FAIL, "unable to open file's superblock extension")
+ if(H5F_super_ext_open(f, f->shared->sblock->ext_addr, &ext_loc) < 0)
+ HGOTO_ERROR(H5E_FILE, H5E_CANTOPENOBJ, FAIL, "unable to open file's superblock extension")
} /* end if */
else {
HDassert(may_create);
@@ -1550,24 +1690,24 @@ H5F__super_ext_write_msg(H5F_t *f, unsigned id, void *mesg,
/* Check if message with ID does not exist in the object header */
if((status = H5O_msg_exists(&ext_loc, id)) < 0)
- HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "unable to check object header for message or message exists")
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to check object header for message or message exists")
/* Check for creating vs. writing */
if(may_create) {
- if(status)
- HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "Message should not exist")
+ if(status)
+ HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "Message should not exist")
/* Create the message with ID in the superblock extension */
if(H5O_msg_create(&ext_loc, id, (mesg_flags | H5O_MSG_FLAG_DONTSHARE), H5O_UPDATE_TIME, mesg) < 0)
- HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "unable to create the message in object header")
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to create the message in object header")
} /* end if */
else {
- if(!status)
- HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "Message should exist")
+ if(!status)
+ HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "Message should exist")
/* Update the message with ID in the superblock extension */
if(H5O_msg_write(&ext_loc, id, (mesg_flags | H5O_MSG_FLAG_DONTSHARE), H5O_UPDATE_TIME, mesg) < 0)
- HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "unable to write the message in object header")
+ HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to write the message in object header")
} /* end else */
done: