diff options
39 files changed, 1458 insertions, 309 deletions
diff --git a/c++/src/H5FaccProp.cpp b/c++/src/H5FaccProp.cpp index 5a478e7..a028e00 100644 --- a/c++/src/H5FaccProp.cpp +++ b/c++/src/H5FaccProp.cpp @@ -685,6 +685,53 @@ unsigned FileAccPropList::getGcReferences() const } //-------------------------------------------------------------------------- +// Function: FileAccPropList::setFileLocking +///\brief Sets file locking flags. +///\param use_file_locking - IN: Flag that determines if file locks should +// be used or not. +///\param ignore_when_disabled - IN: Flag that determines if file locks +// should be be used when disabled on the file system or not. +///\exception H5::PropListIException +///\par Description +/// For information, please refer to the H5Pset_file_locking API in +/// the HDF5 C Reference Manual. +// Programmer Dana Robinson - 2020 +//-------------------------------------------------------------------------- +void FileAccPropList::setFileLocking(hbool_t use_file_locking, hbool_t ignore_when_disabled) const +{ + herr_t ret_value = H5Pset_file_locking(id, use_file_locking, ignore_when_disabled); + + if (ret_value < 0) + { + throw PropListIException("FileAccPropList::setFileLocking", "H5Pset_file_locking failed"); + } +} + +//-------------------------------------------------------------------------- +// Function: FileAccPropList::getFileLocking +///\brief Gets file locking flags. +///\param use_file_locking - OUT: Flag that determines if file locks +// should be used or not. +///\param ignore_when_disabled - OUT: Flag that determines if file locks +// should be be used when disabled on the file system or not. +///\exception H5::PropListIException +///\par Description +/// For information, please refer to the H5Pget_file_locking API in +/// the HDF5 C Reference Manual. +// Programmer Dana Robinson - 2020 +//-------------------------------------------------------------------------- +void FileAccPropList::getFileLocking(hbool_t& use_file_locking, hbool_t& ignore_when_disabled) const +{ + herr_t ret_value = H5Pget_file_locking(id, &use_file_locking, &ignore_when_disabled); + + if (ret_value < 0) + { + throw PropListIException("FileAccPropList::getFileLocking", "H5Pget_file_locking failed"); + } +} + + +//-------------------------------------------------------------------------- // Function: FileAccPropList::setLibverBounds ///\brief Sets bounds on versions of library format to be used when creating /// or writing objects. diff --git a/c++/src/H5FaccProp.h b/c++/src/H5FaccProp.h index 58f049e..2475cf5 100644 --- a/c++/src/H5FaccProp.h +++ b/c++/src/H5FaccProp.h @@ -126,6 +126,12 @@ class H5_DLLCPP FileAccPropList : public PropList { // Returns garbage collecting references setting. unsigned getGcReferences() const; + // Sets file locking parameters + void setFileLocking(hbool_t use_file_locking, hbool_t ignore_when_disabled) const; + + // Gets file locking parameters + void getFileLocking(hbool_t& use_file_locking, hbool_t& ignore_when_disabled) const; + // Sets bounds on versions of library format to be used when creating // or writing objects. void setLibverBounds(H5F_libver_t libver_low, H5F_libver_t libver_high) const; diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake index 2930969..ab121e9 100644 --- a/config/cmake/ConfigureChecks.cmake +++ b/config/cmake/ConfigureChecks.cmake @@ -61,6 +61,31 @@ if (HDF5_ENABLE_CODESTACK) endif () MARK_AS_ADVANCED (HDF5_ENABLE_CODESTACK) +# ---------------------------------------------------------------------- +# Check if they would like to use file locking by default +#----------------------------------------------------------------------------- +option (HDF5_USE_FILE_LOCKING "Use file locking by default (mainly for SWMR)" ON) +if (HDF5_USE_FILE_LOCKING) + set (${HDF_PREFIX}_USE_FILE_LOCKING 1) +endif () + +# ---------------------------------------------------------------------- +# Check if they would like to ignore file locks when disabled on a file system +#----------------------------------------------------------------------------- +option (HDF5_IGNORE_DISABLED_FILE_LOCKS "Ignore file locks when disabled on file system" ON) +if (HDF5_IGNORE_DISABLED_FILE_LOCKS) + set (${HDF_PREFIX}_IGNORE_DISABLED_FILE_LOCKS 1) +endif () + +# Set the libhdf5.settings file variable +if (HDF5_IGNORE_DISABLED_FILE_LOCKS AND HDF5_USE_FILE_LOCKING) + set (HDF5_FILE_LOCKING_SETTING "best-effort") +elseif (HDF5_IGNORE_DISABLED_FILE_LOCKS) + set (HDF5_FILE_LOCKING_SETTING "yes") +else () + set (HDF5_FILE_LOCKING_SETTING "no") +endif () + #----------------------------------------------------------------------------- # Are we going to use HSIZE_T #----------------------------------------------------------------------------- diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index 64b4852..fdf4c7d 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -489,6 +489,9 @@ /* Define if the compiler understands __inline__ */ #cmakedefine H5_HAVE___INLINE__ @H5_HAVE___INLINE__@ +/* Define if the library will ignore file locks when disabled */ +#cmakedefine H5_IGNORE_DISABLED_FILE_LOCKS @H5_IGNORE_DISABLED_FILE_LOCKS@ + /* Define if the high-level library headers should be included in hdf5.h */ #cmakedefine H5_INCLUDE_HL @H5_INCLUDE_HL@ @@ -729,6 +732,9 @@ /* Define using v1.14 public API symbols by default */ #cmakedefine H5_USE_114_API_DEFAULT @H5_USE_114_API_DEFAULT@ +/* Define if the library will use file locking */ +#cmakedefine H5_FILE_LOCKING @H5_USE_FILE_LOCKING@ + /* Define if a memory checking tool will be used on the library, to cause library to be very picky about memory operations and also disable the internal free list manager code. */ diff --git a/config/cmake/libhdf5.settings.cmake.in b/config/cmake/libhdf5.settings.cmake.in index b745765..ebcbd61 100644 --- a/config/cmake/libhdf5.settings.cmake.in +++ b/config/cmake/libhdf5.settings.cmake.in @@ -85,5 +85,6 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ Using memory checker: @HDF5_ENABLE_USING_MEMCHECKER@ Memory allocation sanity checks: @HDF5_MEMORY_ALLOC_SANITY_CHECK@ Function Stack Tracing: @HDF5_ENABLE_CODESTACK@ + Use file locking: @HDF5_FILE_LOCKING_SETTING@ Strict File Format Checks: @HDF5_STRICT_FORMAT_CHECKS@ Optimization Instrumentation: @HDF5_Enable_Instrument@ diff --git a/configure.ac b/configure.ac index f79e9e6..6e0b4c7 100644 --- a/configure.ac +++ b/configure.ac @@ -2316,6 +2316,56 @@ case "X-$OPTIMIZATION" in esac ## ---------------------------------------------------------------------- +## Check if file locking should be used +## +AC_MSG_CHECKING([enable file locking]) +AC_ARG_ENABLE([file-locking], + [AS_HELP_STRING([--enable-file-locking=(yes|no|best-effort)], + [Sets the default for whether or not to use file + locking when opening files. Can be overridden + with the HDF5_USE_FILE_LOCKING environment variable + and the H5Pset_file_locking() API call. + best-effort attempts to use file locking but does + not fail when file locks have been disabled on + the file system (useful with Lustre). + [default=best-effort] + ])], + [DESIRED_FILE_LOCKING=$enableval]) + +## Set defaults +if test "X-$DESIRED_FILE_LOCKING" = X- ; then + DESIRED_FILE_LOCKING=best-effort +fi + +## Allow this variable to be substituted in +## other files (src/libhdf5.settings.in, etc.) +AC_SUBST([DESIRED_FILE_LOCKING]) +AC_SUBST([USE_FILE_LOCKING]) +AC_SUBST([IGNORE_DISABLED_FILE_LOCKS]) + +case "X-$DESIRED_FILE_LOCKING" in + X-best-effort) + AC_MSG_RESULT([best-effort]) + AC_DEFINE([USE_FILE_LOCKING], [1], + [Define if the library will use file locking]) + AC_DEFINE([IGNORE_DISABLED_FILE_LOCKS], [1], + [Define if the library will ignore file locks when disabled]) + ;; + X-yes) + AC_MSG_RESULT([yes]) + AC_DEFINE([USE_FILE_LOCKING], [1], + [Define if the library will use file locking]) + ;; + X-no) + AC_MSG_RESULT([no]) + ;; + *) + AC_MSG_ERROR([Unrecognized value: $USE_FILE_LOCKING]) + ;; +esac + + +## ---------------------------------------------------------------------- ## Enable/disable internal package-level debugging output ## AC_MSG_CHECKING([for internal debug output]) diff --git a/fortran/src/H5Pf.c b/fortran/src/H5Pf.c index 7cb3db8..9816d7b 100644 --- a/fortran/src/H5Pf.c +++ b/fortran/src/H5Pf.c @@ -5241,6 +5241,83 @@ h5pget_file_image_c(hid_t_f *fapl_id, void **buf_ptr, size_t_f *buf_len_ptr) return ret_value; } +/****if* H5Pf/h5pset_file_locking_c + * NAME + * h5pset_file_locking_c + * PURPOSE + * Call H5Pset_file_locking to set file locking properties. + * INPUTS + * prp_id - file access property list identifier + * use_file_locking - TRUE/FALSE flag + * ignore_disabled_file_locking - TRUE/FALSE flag + * RETURNS + * 0 on success, -1 on failure + * AUTHOR + * Dana Robinson + * Summer 2020 + * SOURCE +*/ +int_f +h5pset_file_locking_c(hid_t_f *prp_id, int_f *use_file_locking, int_f *ignore_disabled_file_locking) +/******/ +{ + int ret_value = 0; + hid_t c_prp_id = H5I_INVALID_HID; + herr_t status; + hbool_t c_use_flag = 1; + hbool_t c_ignore_flag = 1; + + if (*use_file_locking == 0) c_use_flag = 0; + if (*ignore_disabled_file_locking == 0) c_ignore_flag = 1; + + c_prp_id = (hid_t)*prp_id; + + status = H5Pset_file_locking(c_prp_id, c_use_flag, c_ignore_flag); + + if ( status < 0 ) ret_value = -1; + + return ret_value; +} + + +/****if* H5Pf/h5pget_file_locking_c + * NAME + * h5pget_file_locking_c + * PURPOSE + * Call H5Pget_file_locking to get file locking properties. + * INPUTS + * prp_id - file access property list identifier + * use_file_locking - TRUE/FALSE flag + * ignore_disabled_file_locking - TRUE/FALSE flag + * RETURNS + * 0 on success, -1 on failure + * AUTHOR + * Dana Robinson + * Summer 2020 + * SOURCE +*/ +int_f +h5pget_file_locking_c(hid_t_f *prp_id, int_f *use_file_locking, int_f *ignore_disabled_file_locking) +/******/ +{ + int ret_value = 0; + hid_t c_prp_id = H5I_INVALID_HID; + hbool_t c_use_flag = 1; + hbool_t c_ignore_flag = 1; + herr_t c_ret; + + c_prp_id = (hid_t)*prp_id; + + c_ret = H5Pget_file_locking(c_prp_id, &c_use_flag, &c_ignore_flag); + + if ( c_ret < 0 ) ret_value = -1; + + *use_file_locking = (int_f)c_use_flag; + *ignore_disabled_file_locking = (int_f)c_ignore_flag; + + return ret_value; +} + #ifdef H5_HAVE_PARALLEL /****if* H5Pf/h5pset_fapl_mpio_c * NAME diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90 index 7e06cf3..40c4e95 100644 --- a/fortran/src/H5Pff.F90 +++ b/fortran/src/H5Pff.F90 @@ -8263,5 +8263,107 @@ END SUBROUTINE h5pget_virtual_dsetname_f END SUBROUTINE h5pget_vol_id_f +!****s* H5P (F03)/h5pget_file_locking_f_F03 +! +! NAME +! h5pget_file_locking_f +! +! PURPOSE +! Gets the file locking properties. File locking is mainly used to help +! enforce SWMR semantics. +! +! INPUTS +! fapl_id - Target file access property list identifier. +! +! OUTPUTS +! use_file_locking - Whether or not to use file locks. +! ignore_disabled_locks - Whether or not to ignore file locks when locking +! is disabled on a file system. +! hdferr - error code: +! 0 on success and -1 on failure +! +! AUTHOR +! Dana Robinson +! Summer 2020 +! +! Fortran2003 Interface: + SUBROUTINE h5pget_file_locking_f(fapl_id, use_file_locking, ignore_disabled_locks, hdferr) + IMPLICIT NONE + INTEGER(HID_T) , INTENT(IN) :: fapl_id + LOGICAL , INTENT(OUT) :: use_file_locking + LOGICAL , INTENT(OUT) :: ignore_disabled_locks + INTEGER , INTENT(OUT) :: hdferr +!***** + LOGICAL(C_BOOL) :: c_use_flag + LOGICAL(C_BOOL) :: c_ignore_flag + + INTERFACE + INTEGER FUNCTION h5pget_file_locking_c(fapl_id, use_file_locking, ignore_disabled_locks) BIND(C, NAME='H5Pget_file_locking') + IMPORT :: HID_T, C_BOOL + IMPLICIT NONE + INTEGER(HID_T), INTENT(IN), VALUE :: fapl_id + LOGICAL(C_BOOL), INTENT(OUT) :: use_file_locking + LOGICAL(C_BOOL), INTENT(OUT) :: ignore_disabled_locks + END FUNCTION h5pget_file_locking_c + END INTERFACE + + hdferr = INT(h5pget_file_locking_c(fapl_id, c_use_flag, c_ignore_flag)) + + ! Transfer value of C C_BOOL type to Fortran LOGICAL + use_file_locking = c_use_flag + ignore_disabled_locks = c_ignore_flag + + END SUBROUTINE h5pget_file_locking_f + +!****s* H5P (F03)/h5pset_file_locking_f_F03 +! +! NAME +! h5pset_file_locking_f +! +! PURPOSE +! Sets the file locking properties. File locking is mainly used to help +! enforce SWMR semantics. +! +! INPUTS +! fapl_id - Target file access property list identifier. +! use_file_locking - Whether or not to use file locks. +! ignore_disabled_locks - Whether or not to ignore file locks when locking +! is disabled on a file system. +! hdferr - error code: +! 0 on success and -1 on failure +! +! AUTHOR +! Dana Robinson +! Summer 2020 +! +! Fortran2003 Interface: + SUBROUTINE h5pset_file_locking_f(fapl_id, use_file_locking, ignore_disabled_locks, hdferr) + IMPLICIT NONE + INTEGER(HID_T) , INTENT(IN) :: fapl_id + LOGICAL , INTENT(IN) :: use_file_locking + LOGICAL , INTENT(IN) :: ignore_disabled_locks + INTEGER , INTENT(OUT) :: hdferr +!***** + LOGICAL(C_BOOL) :: c_use_flag + LOGICAL(C_BOOL) :: c_ignore_flag + + INTERFACE + INTEGER FUNCTION h5pset_file_locking_c(fapl_id, use_file_locking, ignore_disabled_locks) BIND(C, NAME='H5Pset_file_locking') + IMPORT :: HID_T, C_BOOL + IMPLICIT NONE + INTEGER(HID_T), INTENT(IN), VALUE :: fapl_id + LOGICAL(C_BOOL), INTENT(IN) :: use_file_locking + LOGICAL(C_BOOL), INTENT(IN) :: ignore_disabled_locks + END FUNCTION h5pset_file_locking_c + END INTERFACE + + ! Transfer value of Fortran LOGICAL to C C_BOOL type + c_use_flag = use_file_locking + c_ignore_flag = ignore_disabled_locks + + hdferr = INT(h5pset_file_locking_c(fapl_id, c_use_flag, c_ignore_flag)) + + END SUBROUTINE h5pset_file_locking_f + END MODULE H5P diff --git a/fortran/src/H5f90proto.h b/fortran/src/H5f90proto.h index 695efcd..56a685e 100644 --- a/fortran/src/H5f90proto.h +++ b/fortran/src/H5f90proto.h @@ -481,6 +481,8 @@ H5_FCDLL int_f h5pset_nlinks_c(hid_t_f *lapl_id, size_t_f *nlinks); H5_FCDLL int_f h5pget_nlinks_c(hid_t_f *lapl_id, size_t_f *nlinks); H5_FCDLL int_f h5pset_chunk_cache_c(hid_t_f *dapl_id, size_t_f *rdcc_nslots, size_t_f *rdcc_nbytes, real_f *rdcc_w0); H5_FCDLL int_f h5pget_chunk_cache_c(hid_t_f *dapl_id, size_t_f *rdcc_nslots, size_t_f *rdcc_nbytes, real_f *rdcc_w0); +H5_FCDLL int_f h5pset_file_locking_c(hid_t_f *prp_id, int_f *use_file_locking, int_f *ignore_disabled_file_locking); +H5_FCDLL int_f h5pget_file_locking_c(hid_t_f *prp_id, int_f *use_file_locking, int_f *ignore_disabled_file_locking); #ifdef H5_HAVE_PARALLEL H5_FCDLL int_f h5pget_mpio_actual_io_mode_c(hid_t_f *dxpl_id, int_f *actual_io_mode); H5_FCDLL int_f h5pget_fapl_mpio_c(hid_t_f *prp_id, int_f* comm, int_f* info); diff --git a/fortran/src/hdf5_fortrandll.def.in b/fortran/src/hdf5_fortrandll.def.in index 9c69e5a..4207239 100644 --- a/fortran/src/hdf5_fortrandll.def.in +++ b/fortran/src/hdf5_fortrandll.def.in @@ -339,6 +339,8 @@ H5P_mp_H5PGET_DSET_NO_ATTRS_HINT_F H5P_mp_H5PSET_DSET_NO_ATTRS_HINT_F H5P_mp_H5PSET_VOL_F H5P_mp_H5PGET_VOL_ID_F +H5P_mp_H5PSET_FILE_LOCKING_F +H5P_mp_H5PGET_FILE_LOCKING_F ; Parallel @H5_NOPAREXP@H5P_mp_H5PSET_FAPL_MPIO_F @H5_NOPAREXP@H5P_mp_H5PGET_FAPL_MPIO_F diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index 1527dc2..ae1eb3d 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -6687,6 +6687,51 @@ public class H5 implements java.io.Serializable { public synchronized static native void H5Pset_evict_on_close(long fapl_id, boolean evict_on_close) throws HDF5LibraryException; + /** + * H5Pget_use_file_locking retrieves whether we are using file locking. + * + * @param fapl_id + * IN: File access property list identifier + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * + **/ + public synchronized static native boolean H5Pget_use_file_locking(long fapl_id) + throws HDF5LibraryException; + + /** + * H5Pget_use_file_locking retrieves whether we ignore file locks when they are disabled. + * + * @param fapl_id + * IN: File access property list identifier + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * + **/ + public synchronized static native boolean H5Pget_ignore_disabled_file_locking(long fapl_id) + throws HDF5LibraryException; + + /** + * H5Pset_file_locking sets parameters related to file locking. + * + * @param fapl_id + * IN: File access property list identifier + * + * @param use_file_locking + * IN: Whether the library will use file locking when opening files (mainly for SWMR semantics). + * + * @param ignore_when_disabled + * IN: Whether file locking will be ignored when disabled on a file system (useful for Lustre). + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * + **/ + public synchronized static native void H5Pset_file_locking(long fapl_id, boolean use_file_locking, boolean ignore_when_disabled) + throws HDF5LibraryException; + // ///// unimplemented ///// // herr_t H5Pset_vol(hid_t plist_id, hid_t new_vol_id, const void *new_vol_info); // herr_t H5Pget_vol_id(hid_t plist_id, hid_t *vol_id); diff --git a/java/src/jni/h5pFAPLImp.c b/java/src/jni/h5pFAPLImp.c index 9ae8775..dbfbb5a 100644 --- a/java/src/jni/h5pFAPLImp.c +++ b/java/src/jni/h5pFAPLImp.c @@ -1377,6 +1377,78 @@ done: /* * Class: hdf_hdf5lib_H5 + * Method: H5Pset_file_locking + * Signature: (JZZ)V + */ +JNIEXPORT void JNICALL +Java_hdf_hdf5lib_H5_H5Pset_1file_1locking + (JNIEnv *env, jclass clss, jlong fapl_id, jboolean use_file_locking, jboolean ignore_when_disabled) +{ + hbool_t use_file_locking_val = TRUE; + hbool_t ignore_when_disabled_val = TRUE; + + UNUSED(clss); + + use_file_locking_val = (use_file_locking == JNI_TRUE) ? TRUE : FALSE; + ignore_when_disabled_val = (ignore_when_disabled == JNI_TRUE) ? TRUE : FALSE; + + if (H5Pset_file_locking((hid_t)fapl_id, use_file_locking_val, ignore_when_disabled_val) < 0) + H5_LIBRARY_ERROR(ENVONLY); + +done: + return; +} /* end Java_hdf_hdf5lib_H5_H5Pset_1file_1locking */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_use_file_locking + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1use_1file_1locking + (JNIEnv *env, jclass clss, jlong fapl_id) +{ + hbool_t use_file_locking_val = TRUE; + hbool_t unused = TRUE; + jboolean bval = JNI_FALSE; + + UNUSED(clss); + + if (H5Pget_file_locking((hid_t)fapl_id, &use_file_locking_val, &unused) < 0) + H5_LIBRARY_ERROR(ENVONLY); + + bval = (use_file_locking_val == TRUE) ? JNI_TRUE : JNI_FALSE; + +done: + return bval; +} /* end Java_hdf_hdf5lib_H5_H5Pget_1use_1file_1locking */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_ignore_disabled_file_locking + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1ignore_1disabled_1file_1locking + (JNIEnv *env, jclass clss, jlong fapl_id) +{ + hbool_t ignore_when_disabled_val = TRUE; + hbool_t unused = TRUE; + jboolean bval = JNI_FALSE; + + UNUSED(clss); + + if (H5Pget_file_locking((hid_t)fapl_id, &unused, &ignore_when_disabled_val) < 0) + H5_LIBRARY_ERROR(ENVONLY); + + bval = (ignore_when_disabled_val == TRUE) ? JNI_TRUE : JNI_FALSE; + +done: + return bval; +} /* end Java_hdf_hdf5lib_H5_H5Pget_1ignore_1disabled_1file_1locking */ + +/* + * Class: hdf_hdf5lib_H5 * Method: H5Pset_metadata_read_attempts * Signature: (JJ)V */ diff --git a/java/src/jni/h5pFAPLImp.h b/java/src/jni/h5pFAPLImp.h index 9b353e6..b9b4556 100644 --- a/java/src/jni/h5pFAPLImp.h +++ b/java/src/jni/h5pFAPLImp.h @@ -392,6 +392,33 @@ Java_hdf_hdf5lib_H5_H5Pget_1evict_1on_1close /* * Class: hdf_hdf5lib_H5 + * Method: H5Pset_file_locking + * Signature: (JZZ)V + */ +JNIEXPORT void JNICALL +Java_hdf_hdf5lib_H5_H5Pset_1file_1locking +(JNIEnv *env, jclass clss, jlong fapl_id, jboolean use_file_locking, jboolean ignore_when_disabled); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_use_file_locking + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1use_1file_1locking +(JNIEnv *env, jclass clss, jlong fapl_id); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_ignore_disabled_file_locking + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1ignore_1disabled_1file_1locking +(JNIEnv *env, jclass clss, jlong fapl_id); + +/* + * Class: hdf_hdf5lib_H5 * Method: H5Pset_metadata_read_attempts * Signature: (JJ)V */ diff --git a/java/test/TestH5Pfapl.java b/java/test/TestH5Pfapl.java index 10a79dd..4233580 100644 --- a/java/test/TestH5Pfapl.java +++ b/java/test/TestH5Pfapl.java @@ -1398,4 +1398,36 @@ public class TestH5Pfapl { fail("H5P_evict_on_close: " + err); } } + + @Test + public void testH5P_file_locking() { + boolean use_file_locking = false; + boolean ignore_disabled_file_locking = false; + try { + // false values (usually not the default) + H5.H5Pset_file_locking(fapl_id, false, false); + use_file_locking = H5.H5Pget_use_file_locking(fapl_id); + ignore_disabled_file_locking = H5.H5Pget_ignore_disabled_file_locking(fapl_id); + assertFalse("H5P_file_locking", use_file_locking); + assertFalse("H5P_file_locking", ignore_disabled_file_locking); + + // true values (typically the default) + H5.H5Pset_file_locking(fapl_id, true, true); + use_file_locking = H5.H5Pget_use_file_locking(fapl_id); + ignore_disabled_file_locking = H5.H5Pget_ignore_disabled_file_locking(fapl_id); + assertTrue("H5P_file_locking", use_file_locking); + assertTrue("H5P_file_locking", ignore_disabled_file_locking); + } + catch (HDF5PropertyListInterfaceException err) { + // parallel is not supported + if (err.getMinorErrorNumber() != HDF5Constants.H5E_UNSUPPORTED) { + err.printStackTrace(); + fail("H5P_test_file_locking: " + err); + } + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5P_test_file_locking: " + err); + } + } } diff --git a/java/test/testfiles/JUnit-TestH5Pfapl.txt b/java/test/testfiles/JUnit-TestH5Pfapl.txt index c4f37d0..c34ac1c 100644 --- a/java/test/testfiles/JUnit-TestH5Pfapl.txt +++ b/java/test/testfiles/JUnit-TestH5Pfapl.txt @@ -28,6 +28,7 @@ JUnit version 4.11 .testH5Pset_elink_fapl .testH5P_hyper_vector_size .testH5P_gc_references +.testH5P_file_locking .testH5P_family_offset .testH5P_fapl_core .testH5P_fapl_muti @@ -37,5 +38,5 @@ JUnit version 4.11 Time: XXXX -OK (35 tests) +OK (36 tests) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index b9641d1..6013603 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -330,6 +330,55 @@ New Features (ADB - 2018/07/16) + - Add file locking configure and CMake options + + HDF5 1.10.0 introduced a file locking scheme, primarily to help + enforce SWMR setup. Formerly, the only user-level control of the scheme + was via the HDF5_USE_FILE_LOCKING environment variable. + + This change introduces configure-time options that control whether + or not file locking will be used and whether or not the library + ignores errors when locking has been disabled on the file system + (useful on some HPC Lustre installations). + + In both the Autotools and CMake, the settings have the effect of changing + the default property list settings (see the H5Pset/get_file_locking() + entry, below). + + The yes/no/best-effort file locking configure setting has also been + added to the libhdf5.settings file. + + Autotools: + + An --enable-file-locking=(yes|no|best-effort) option has been added. + + yes: Use file locking. + no: Do not use file locking. + best-effort: Use file locking and ignore "disabled" errors. + + CMake: + + Two self-explanatory options have been added: + + HDF5_USE_FILE_LOCKING + HDF5_IGNORE_DISABLED_FILE_LOCKS + + Setting both of these to ON is the equivalent to the Autotools' + best-effort setting. + + NOTE: + The precedence order of the various file locking control mechanisms is: + + 1) HDF5_USE_FILE_LOCKING environment variable (highest) + + 2) H5Pset_file_locking() + + 3) configure/CMake options (which set the property list defaults) + + 4) library defaults (currently best-effort) + + (DER - 2020/07/30, HDFFV-11092) + Library: -------- @@ -507,6 +556,32 @@ New Features (DER - 2020/03/18, HDFFV-11057) + - Add BEST-EFFORT value to HDF5_USE_FILE_LOCKING environment variable + + This change adds a BEST-EFFORT to the TRUE/FALSE, 1/0 settings that + were previously accepted. This option turns on file locking but + ignores locking errors when the library detects that file locking + has been disabled on a file system (useful on some HPC Lustre + installations). + + The capitalization of BEST-EFFORT is mandatory. + + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. + + (DER - 2020/07/30, HDFFV-11092) + + + - Add H5Pset/get_file_locking() API calls + + This change adds new API calls which can be used to set or get the + file locking parameters. The single API call sets both the "use file + locking" flag and the "ignore disabled file locking" flag. + + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. + + (DER - 2020/07/30, HDFFV-11092) Parallel Library: ----------------- @@ -537,11 +612,21 @@ New Features (MSB, 2019/01/08, HDFFV-10443) + - Add wrappers for H5Pset/get_file_locking() API calls + + h5pget_file_locking_f() + h5pset_file_locking_f() + + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. + + (DER - 2020/07/30, HDFFV-11092) + C++ Library: ------------ - Added new wrappers for H5Pset/get_create_intermediate_group() - LinkCreatPropList::setCreateIntermediateGroup() LinkCreatPropList::getCreateIntermediateGroup() + LinkCreatPropList::setCreateIntermediateGroup() (BMR - 2019/04/22, HDFFV-10622) @@ -550,6 +635,16 @@ New Features (BMR - 2019/02/14, HDFFV-10532) + - Add wrappers for H5Pset/get_file_locking() API calls + + FileAccPropList::setFileLocking() + FileAccPropList::getFileLocking() + + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. + + (DER - 2020/07/30, HDFFV-11092) + Java Library: ---------------- @@ -587,6 +682,20 @@ New Features (DER - 2018/12/08, HDFFV-10252) + - Add wrappers for H5Pset/get_file_locking() API calls + + H5Pset_file_locking() + H5Pget_use_file_locking() + H5Pget_ignore_disabled_file_locking() + + Unlike the C++ and Fortran wrappers, there are separate getters for the + two file locking settings, each of which returns a boolean value. + + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. + + (DER - 2020/07/30, HDFFV-11092) + Tools: ------ @@ -73,8 +73,6 @@ typedef struct { /* Local Prototypes */ /********************/ -static herr_t H5F__close_cb(H5VL_object_t *file_vol_obj); - /* Callback for getting object counts in a file */ static int H5F__get_all_count_cb(void H5_ATTR_UNUSED *obj_ptr, hid_t H5_ATTR_UNUSED obj_id, void *key); @@ -86,9 +84,6 @@ static int H5F__get_all_ids_cb(void H5_ATTR_UNUSED *obj_ptr, hid_t obj_id, void /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ @@ -105,150 +100,6 @@ H5FL_EXTERN(H5VL_t); /* Declare a free list to manage the H5VL_object_t struct */ H5FL_EXTERN(H5VL_object_t); -/* File ID class */ -static const H5I_class_t H5I_FILE_CLS[1] = {{ - H5I_FILE, /* ID class value */ - 0, /* Class flags */ - 0, /* # of reserved IDs for class */ - (H5I_free_t)H5F__close_cb /* Callback routine for closing objects of this class */ -}}; - - - -/*------------------------------------------------------------------------- - * Function: H5F_init - * - * Purpose: Initialize the interface from some other layer. - * - * Return: Success: non-negative - * - * Failure: negative - *------------------------------------------------------------------------- - */ -herr_t -H5F_init(void) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - /* FUNC_ENTER() does all the work */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5F_init() */ - - -/*-------------------------------------------------------------------------- -NAME - H5F__init_package -- Initialize interface-specific information -USAGE - herr_t H5F__init_package() -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. - ---------------------------------------------------------------------------*/ -herr_t -H5F__init_package(void) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* - * Initialize the atom group for the file IDs. - */ - if(H5I_register_type(H5I_FILE_CLS) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to initialize interface") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5F__init_package() */ - - -/*------------------------------------------------------------------------- - * Function: H5F_term_package - * - * Purpose: Terminate this interface: free all memory and reset global - * variables to their initial values. Release all ID groups - * associated with this interface. - * - * Return: Success: Positive if anything was done that might - * have affected other interfaces; - * zero otherwise. - * - * Failure: Never fails - * - *------------------------------------------------------------------------- - */ -int -H5F_term_package(void) -{ - int n = 0; - - FUNC_ENTER_NOAPI_NOINIT_NOERR - - if(H5_PKG_INIT_VAR) { - if(H5I_nmembers(H5I_FILE) > 0) { - (void)H5I_clear_type(H5I_FILE, FALSE, FALSE); - n++; /*H5I*/ - } /* end if */ - else { - /* Make certain we've cleaned up all the shared file objects */ - H5F_sfile_assert_num(0); - - /* Destroy the file object id group */ - n += (H5I_dec_type_ref(H5I_FILE) > 0); - - /* Mark closed */ - if(0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end else */ - } /* end if */ - - FUNC_LEAVE_NOAPI(n) -} /* end H5F_term_package() */ - - -/*------------------------------------------------------------------------- - * Function: H5F__close_cb - * - * Purpose: Closes a file or causes the close operation to be pended. - * This function is called from the API and gets called - * by H5Fclose->H5I_dec_ref->H5F__close_cb when H5I_dec_ref() - * decrements the file ID reference count to zero. The file ID - * is removed from the H5I_FILE group by H5I_dec_ref() just - * before H5F__close_cb() is called. If there are open object - * headers then the close is pended by moving the file to the - * H5I_FILE_CLOSING ID group (the f->closing contains the ID - * assigned to file). - * - * Return: SUCCEED/FAIL - * - *------------------------------------------------------------------------- - */ -static herr_t -H5F__close_cb(H5VL_object_t *file_vol_obj) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity check */ - HDassert(file_vol_obj); - - /* Close the file */ - if(H5VL_file_close(file_vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close file") - - /* Free the VOL object */ - if(H5VL_free_object(file_vol_obj) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTDEC, FAIL, "unable to free VOL object") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5F__close_cb() */ /*------------------------------------------------------------------------- @@ -1642,7 +1642,7 @@ H5FDlock(H5FD_t *file, hbool_t rw) /* Call private function */ if(H5FD_lock(file, rw) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "file lock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "file lock request failed") done: FUNC_LEAVE_API(ret_value) @@ -1672,7 +1672,7 @@ H5FD_lock(H5FD_t *file, hbool_t rw) /* Dispatch to driver */ if(file->cls->lock && (file->cls->lock)(file, rw) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "driver lock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "driver lock request failed") done: FUNC_LEAVE_NOAPI(ret_value) @@ -1704,7 +1704,7 @@ H5FDunlock(H5FD_t *file) /* Call private function */ if(H5FD_unlock(file) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "file unlock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "file unlock request failed") done: FUNC_LEAVE_API(ret_value) @@ -1733,7 +1733,7 @@ H5FD_unlock(H5FD_t *file) /* Dispatch to driver */ if(file->cls->unlock && (file->cls->unlock)(file) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "driver unlock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "driver unlock request failed") done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDcore.c b/src/H5FDcore.c index 0551dd0..a20f0c1 100644 --- a/src/H5FDcore.c +++ b/src/H5FDcore.c @@ -36,6 +36,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_CORE_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* The skip list node type. Represents a region in the file. */ typedef struct H5FD_core_region_t { haddr_t start; /* Start address of the region */ @@ -56,6 +59,7 @@ typedef struct H5FD_core_t { hbool_t backing_store; /* write to file name on flush */ hbool_t write_tracking; /* Whether to track writes */ size_t bstore_page_size; /* backing store page size */ + hbool_t ignore_disabled_file_locks; int fd; /* backing store file descriptor */ /* Information for determining uniqueness of a file with a backing store */ #ifndef H5_HAVE_WIN32_API @@ -412,10 +416,20 @@ done: static herr_t H5FD__init_package(void) { + char *lock_env_var = NULL; /* Environment variable pointer */ herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && !HDstrcmp(lock_env_var, "BEST-EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if(lock_env_var && !HDstrcmp(lock_env_var, "TRUE")) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if(H5FD_core_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize core VFD") @@ -798,6 +812,16 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr /* Save file image callbacks */ file->fi_callbacks = file_image_info.callbacks; + /* Check the file locking flags in the fapl */ + if(ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if(H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + if(fd >= 0) { /* Retrieve information for determining uniqueness of file */ #ifdef H5_HAVE_WIN32_API @@ -1615,8 +1639,12 @@ H5FD_core_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if(HDflock(file->fd, lock_flags | LOCK_NB) < 0) { - if(ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") } /* end if */ @@ -1652,11 +1680,15 @@ H5FD_core_unlock(H5FD_t *_file) if(file->fd >= 0) { if(HDflock(file->fd, LOCK_UN) < 0) { - if(ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") - } /* end if */ + } } /* end if */ diff --git a/src/H5FDdirect.c b/src/H5FDdirect.c index 34c4346..dcd8b13 100644 --- a/src/H5FDdirect.c +++ b/src/H5FDdirect.c @@ -38,6 +38,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_DIRECT_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* File operations */ #define OP_UNKNOWN 0 #define OP_READ 1 @@ -71,6 +74,7 @@ typedef struct H5FD_direct_t { haddr_t pos; /*current file I/O position */ int op; /*last operation */ H5FD_direct_fapl_t fa; /*file access properties */ + hbool_t ignore_disabled_file_locks; #ifndef H5_HAVE_WIN32_API /* * On most systems the combination of device and i-node number uniquely @@ -193,10 +197,20 @@ DESCRIPTION static herr_t H5FD__init_package(void) { + char *lock_env_var = NULL; /* Environment variable pointer */ herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && !HDstrcmp(lock_env_var, "BEST-EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if(lock_env_var && !HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1")) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if(H5FD_direct_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize direct VFD") @@ -518,6 +532,16 @@ H5FD_direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxadd file->fa.fbsize = fa->fbsize; file->fa.cbsize = fa->cbsize; + /* Check the file locking flags in the fapl */ + if(ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if(H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + /* Try to decide if data alignment is required. The reason to check it here * is to handle correctly the case that the file is in a different file system * than the one where the program is running. @@ -1334,17 +1358,28 @@ done: static herr_t H5FD_direct_lock(H5FD_t *_file, hbool_t rw) { - H5FD_direct_t *file = (H5FD_direct_t*)_file; /* VFD file struct */ - const int lock = rw ? LOCK_EX : LOCK_SH; - herr_t ret_value = SUCCEED; /* Return value */ + H5FD_direct_t *file = (H5FD_direct_t*)_file; /* VFD file struct */ + int lock_flags; /* file locking flags */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT HDassert(file); - /* Place the lock with non-blocking */ - if(HDflock(file->fd, lock | LOCK_NB) < 0) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to flock file") + /* Set exclusive or shared lock based on rw status */ + lock_flags = rw ? LOCK_EX : LOCK_SH; + + /* Place a non-blocking lock on the file */ + if(HDflock(file->fd, lock_flags | LOCK_NB) < 0) { + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } + else + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file") + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -1372,8 +1407,16 @@ H5FD_direct_unlock(H5FD_t *_file) HDassert(file); - if(HDflock(file->fd, LOCK_UN) < 0) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to flock (unlock) file") + if(HDflock(file->fd, LOCK_UN) < 0) { + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } + else + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDfamily.c b/src/H5FDfamily.c index d110ef7..154878e 100644 --- a/src/H5FDfamily.c +++ b/src/H5FDfamily.c @@ -1372,9 +1372,9 @@ H5FD_family_lock(H5FD_t *_file, hbool_t rw) for(v = 0; v < u; v++) { if(H5FD_unlock(file->memb[v]) < 0) /* Push error, but keep going */ - HDONE_ERROR(H5E_IO, H5E_CANTUNLOCK, FAIL, "unable to unlock member files") + HDONE_ERROR(H5E_IO, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock member files") } /* end for */ - HGOTO_ERROR(H5E_IO, H5E_CANTLOCK, FAIL, "unable to lock member files") + HGOTO_ERROR(H5E_IO, H5E_CANTLOCKFILE, FAIL, "unable to lock member files") } /* end if */ done: @@ -1406,7 +1406,7 @@ H5FD_family_unlock(H5FD_t *_file) for(u = 0; u < file->nmembs; u++) if(file->memb[u]) if(H5FD_unlock(file->memb[u]) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTUNLOCK, FAIL, "unable to unlock member files") + HGOTO_ERROR(H5E_IO, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock member files") done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDlog.c b/src/H5FDlog.c index 78b7742..af6d8e0 100644 --- a/src/H5FDlog.c +++ b/src/H5FDlog.c @@ -40,6 +40,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_LOG_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* Driver-specific file access properties */ typedef struct H5FD_log_fapl_t { char *logfile; /* Allocated log file name */ @@ -80,6 +83,7 @@ typedef struct H5FD_log_t { haddr_t eof; /* end of file; current file size */ haddr_t pos; /* current file I/O position */ H5FD_file_op_t op; /* last operation */ + hbool_t ignore_disabled_file_locks; char filename[H5FD_MAX_FILENAME_LEN]; /* Copy of file name from open operation */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely @@ -234,10 +238,20 @@ H5FL_DEFINE_STATIC(H5FD_log_t); static herr_t H5FD__init_package(void) { + char *lock_env_var = NULL; /* Environment variable pointer */ herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && !HDstrcmp(lock_env_var, "BEST-EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if(lock_env_var && !HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1")) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if(H5FD_log_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize log VFD") @@ -625,6 +639,16 @@ H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) } /* end if */ } /* end if */ + /* Check the file locking flags in the fapl */ + if(ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if(H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + /* Check for non-default FAPL */ if(H5P_FILE_ACCESS_DEFAULT != fapl_id) { /* This step is for h5repart tool only. If user wants to change file driver from @@ -1671,11 +1695,15 @@ H5FD_log_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if(HDflock(file->fd, lock_flags | LOCK_NB) < 0) { - if(ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file") + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -1704,11 +1732,15 @@ H5FD_log_unlock(H5FD_t *_file) HDassert(file); if(HDflock(file->fd, LOCK_UN) < 0) { - if(ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDmulti.c b/src/H5FDmulti.c index 72f4da5..f80cac5 100644 --- a/src/H5FDmulti.c +++ b/src/H5FDmulti.c @@ -1856,7 +1856,7 @@ H5FD_multi_lock(H5FD_t *_file, hbool_t rw) } /* end if */ if(nerrors) - H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "error locking member files", -1) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTLOCKFILE, "error locking member files", -1) return 0; } /* H5FD_multi_lock() */ @@ -1893,7 +1893,7 @@ H5FD_multi_unlock(H5FD_t *_file) } END_MEMBERS; if(nerrors) - H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "error unlocking member files", -1) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTUNLOCKFILE, "error unlocking member files", -1) return 0; } /* H5FD_multi_unlock() */ diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c index 3551905..f0ce82d 100644 --- a/src/H5FDsec2.c +++ b/src/H5FDsec2.c @@ -39,6 +39,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_SEC2_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* The description of a file belonging to this driver. The 'eoa' and 'eof' * determine the amount of hdf5 address space in use and the high-water mark * of the file (the current size of the underlying filesystem file). The @@ -57,6 +60,7 @@ typedef struct H5FD_sec2_t { haddr_t eof; /* end of file; current file size */ haddr_t pos; /* current file I/O position */ H5FD_file_op_t op; /* last operation */ + hbool_t ignore_disabled_file_locks; char filename[H5FD_MAX_FILENAME_LEN]; /* Copy of file name from open operation */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely @@ -190,10 +194,20 @@ H5FL_DEFINE_STATIC(H5FD_sec2_t); static herr_t H5FD__init_package(void) { + char *lock_env_var = NULL; /* Environment variable pointer */ herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && !HDstrcmp(lock_env_var, "BEST-EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if(lock_env_var && !HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1")) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if(H5FD_sec2_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize sec2 VFD") @@ -219,7 +233,7 @@ done: hid_t H5FD_sec2_init(void) { - hid_t ret_value = H5I_INVALID_HID; /* Return value */ + hid_t ret_value = H5I_INVALID_HID; /* Return value */ FUNC_ENTER_NOAPI(H5I_INVALID_HID) @@ -316,6 +330,7 @@ H5FD_sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) struct _BY_HANDLE_FILE_INFORMATION fileinfo; #endif h5_stat_t sb; + H5P_genplist_t *plist; /* Property list pointer */ H5FD_t *ret_value = NULL; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -373,17 +388,26 @@ H5FD_sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) file->inode = sb.st_ino; #endif /* H5_HAVE_WIN32_API */ + /* Get the FAPL */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, NULL, "not a file access property list") + + /* Check the file locking flags in the fapl */ + if(ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if(H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + /* Retain a copy of the name used to open the file, for possible error reporting */ HDstrncpy(file->filename, name, sizeof(file->filename)); file->filename[sizeof(file->filename) - 1] = '\0'; /* Check for non-default FAPL */ if(H5P_FILE_ACCESS_DEFAULT != fapl_id) { - H5P_genplist_t *plist; /* Property list pointer */ - - /* Get the FAPL */ - if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) - HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, NULL, "not a file access property list") /* This step is for h5repart tool only. If user wants to change file driver from * family to one that uses single files (sec2, etc.) while using h5repart, this @@ -961,11 +985,15 @@ H5FD_sec2_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if(HDflock(file->fd, lock_flags | LOCK_NB) < 0) { - if(ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file") + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -994,11 +1022,15 @@ H5FD_sec2_unlock(H5FD_t *_file) HDassert(file); if(HDflock(file->fd, LOCK_UN) < 0) { - if(ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDsplitter.c b/src/H5FDsplitter.c index 4ed3c4a..eb30b1f 100644 --- a/src/H5FDsplitter.c +++ b/src/H5FDsplitter.c @@ -1221,16 +1221,12 @@ H5FD_splitter_lock(H5FD_t *_file, hbool_t rw) HDassert(file->rw_file); /* Place the lock on each file */ - if (H5FD_lock(file->rw_file, rw) < 0) { - HGOTO_ERROR(H5E_VFL, H5E_CANTLOCK, FAIL, "unable to lock R/W file") - } - if (file->wo_file != NULL) { - if (H5FD_lock(file->wo_file, rw) < 0) { - H5FD_SPLITTER_WO_ERROR(file, "H5FD_splitter_lock", - H5E_VFL, H5E_CANTLOCK, FAIL, - "unable to lock W/O file") - } - } + if (H5FD_lock(file->rw_file, rw) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock R/W file") + + if (file->wo_file != NULL) + if (H5FD_lock(file->wo_file, rw) < 0) + H5FD_SPLITTER_WO_ERROR(file, "H5FD_splitter_lock", H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock W/O file") done: FUNC_LEAVE_NOAPI(ret_value) @@ -1260,15 +1256,12 @@ H5FD_splitter_unlock(H5FD_t *_file) HDassert(file->rw_file); /* Remove the lock on each file */ - if (H5FD_unlock(file->rw_file) < 0) { - HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCK, FAIL, "unable to unlock R/W file") - } - if (file->wo_file != NULL) { - if (H5FD_unlock(file->wo_file) < 0) { - HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCK, FAIL, - "unable to unlock W/O file") - } - } + if (H5FD_unlock(file->rw_file) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock R/W file") + + if (file->wo_file != NULL) + if (H5FD_unlock(file->wo_file) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock W/O file") done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDstdio.c b/src/H5FDstdio.c index d29a1b4..4e9ef39 100644 --- a/src/H5FDstdio.c +++ b/src/H5FDstdio.c @@ -52,6 +52,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_STDIO_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = -1; + /* The maximum number of bytes which can be written in a single I/O operation */ static size_t H5_STDIO_MAX_IO_BYTES_g = (size_t)-1; @@ -82,6 +85,7 @@ typedef struct H5FD_stdio_t { haddr_t eof; /* end of file; current file size */ haddr_t pos; /* current file I/O position */ unsigned write_access; /* Flag to indicate the file was opened with write access */ + hbool_t ignore_disabled_file_locks; H5FD_stdio_file_op op; /* last operation */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely @@ -231,11 +235,23 @@ static const H5FD_class_t H5FD_stdio_g = { hid_t H5FD_stdio_init(void) { + char *lock_env_var = NULL; /* Environment variable pointer */ + /* Clear the error stack */ H5Eclear2(H5E_DEFAULT); - if (H5I_VFL!=H5Iget_type(H5FD_STDIO_g)) + /* Check the use disabled file locks environment variable */ + lock_env_var = getenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && !strcmp(lock_env_var, "BEST-EFFORT")) + ignore_disabled_file_locks_s = 1; /* Override: Ignore disabled locks */ + else if(lock_env_var && !strcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1")) + ignore_disabled_file_locks_s = 0; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = -1; /* Environment variable not set, or not set correctly */ + + if (H5I_VFL != H5Iget_type(H5FD_STDIO_g)) H5FD_STDIO_g = H5FDregister(&H5FD_stdio_g); + return H5FD_STDIO_g; } /* end H5FD_stdio_init() */ @@ -318,7 +334,7 @@ H5Pset_fapl_stdio(hid_t fapl_id) *------------------------------------------------------------------------- */ static H5FD_t * -H5FD_stdio_open( const char *name, unsigned flags, hid_t /*UNUSED*/ fapl_id, +H5FD_stdio_open( const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) { FILE *f = NULL; @@ -396,6 +412,21 @@ H5FD_stdio_open( const char *name, unsigned flags, hid_t /*UNUSED*/ fapl_id, file->eof = (haddr_t)x; } + /* Check the file locking flags in the fapl */ + if(ignore_disabled_file_locks_s != -1) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + hbool_t unused; + + /* Use the value in the property list */ + if(H5Pget_file_locking(fapl_id, &unused, &file->ignore_disabled_file_locks) < 0) { + free(file); + fclose(f); + H5Epush_ret(func, H5E_ERR_CLS, H5E_FILE, H5E_CANTGET, "unable to get use disabled file locks property", NULL); + } + } + /* Get the file descriptor (needed for truncate and some Windows information) */ #ifdef H5_HAVE_WIN32_API file->fd = _fileno(file->fp); @@ -1104,10 +1135,13 @@ H5FD_stdio_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if(flock(file->fd, lock_flags | LOCK_NB) < 0) { - if(ENOSYS == errno) - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)", -1) + if(file->ignore_disabled_file_locks && ENOSYS == errno) + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; else - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file lock failed", -1) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTLOCKFILE, "file lock failed", -1) } /* end if */ /* Flush the stream */ @@ -1152,10 +1186,13 @@ H5FD_stdio_unlock(H5FD_t *_file) /* Place a non-blocking lock on the file */ if(flock(file->fd, LOCK_UN) < 0) { - if(ENOSYS == errno) - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)", -1) + if(file->ignore_disabled_file_locks && ENOSYS == errno) + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; else - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file unlock failed", -1) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTUNLOCKFILE, "file unlock failed", -1) } /* end if */ #endif /* H5_HAVE_FLOCK */ diff --git a/src/H5Fint.c b/src/H5Fint.c index 0bda894..8b2d18f 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -76,12 +76,14 @@ typedef struct H5F_olist_t { /* Local Prototypes */ /********************/ +static herr_t H5F__close_cb(H5VL_object_t *file_vol_obj); static herr_t H5F__set_vol_conn(H5F_t *file); static herr_t H5F__get_objects(const H5F_t *f, unsigned types, size_t max_index, hid_t *obj_id_list, hbool_t app_ref, size_t *obj_id_count_ptr); static int H5F__get_objects_cb(void *obj_ptr, hid_t obj_id, void *key); static herr_t H5F__build_name(const char *prefix, const char *file_name, char **full_name/*out*/); static char *H5F__getenv_prefix_name(char **env_prefix/*in,out*/); static H5F_t *H5F__new(H5F_shared_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t *lf); +static herr_t H5F__check_if_using_file_locks(H5P_genplist_t *fapl, hbool_t *use_file_locking); static herr_t H5F__build_actual_name(const H5F_t *f, const H5P_genplist_t *fapl, const char *name, char ** /*out*/ actual_name); static herr_t H5F__flush_phase1(H5F_t *f); static herr_t H5F__flush_phase2(H5F_t *f, hbool_t closing); @@ -91,6 +93,15 @@ static herr_t H5F__flush_phase2(H5F_t *f, hbool_t closing); /* Package Variables */ /*********************/ +/* Package initialization variable */ +hbool_t H5_PKG_INIT_VAR = FALSE; + +/* Based on the value of the HDF5_USE_FILE_LOCKING environment variable. + * TRUE/FALSE have obvious meanings. FAIL means the environment variable was + * not set, so the code should ignore it and use the fapl value instead. + */ +htri_t use_locks_env_g = FAIL; + /*****************************/ /* Library Private Variables */ @@ -107,6 +118,186 @@ H5FL_DEFINE(H5F_t); /* Declare a free list to manage the H5F_shared_t struct */ H5FL_DEFINE(H5F_shared_t); +/* File ID class */ +static const H5I_class_t H5I_FILE_CLS[1] = {{ + H5I_FILE, /* ID class value */ + 0, /* Class flags */ + 0, /* # of reserved IDs for class */ + (H5I_free_t)H5F__close_cb /* Callback routine for closing objects of this class */ +}}; + + + +/*------------------------------------------------------------------------- + * Function: H5F_init + * + * Purpose: Initialize the interface from some other layer. + * + * Return: Success: non-negative + * + * Failure: negative + *------------------------------------------------------------------------- + */ +herr_t +H5F_init(void) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + /* FUNC_ENTER() does all the work */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_init() */ + + +/*-------------------------------------------------------------------------- +NAME + H5F__init_package -- Initialize interface-specific information +USAGE + herr_t H5F__init_package() +RETURNS + Non-negative on success/Negative on failure +DESCRIPTION + Initializes any interface-specific data or routines. + +--------------------------------------------------------------------------*/ +herr_t +H5F__init_package(void) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* + * Initialize the atom group for the file IDs. + */ + if(H5I_register_type(H5I_FILE_CLS) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to initialize interface") + + /* Check the file locking environment variable */ + if(H5F__parse_file_lock_env_var(&use_locks_env_g) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to parse file locking environment variable") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5F__init_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_term_package + * + * Purpose: Terminate this interface: free all memory and reset global + * variables to their initial values. Release all ID groups + * associated with this interface. + * + * Return: Success: Positive if anything was done that might + * have affected other interfaces; + * zero otherwise. + * + * Failure: Never fails + * + *------------------------------------------------------------------------- + */ +int +H5F_term_package(void) +{ + int n = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + if(H5_PKG_INIT_VAR) { + if(H5I_nmembers(H5I_FILE) > 0) { + (void)H5I_clear_type(H5I_FILE, FALSE, FALSE); + n++; /*H5I*/ + } /* end if */ + else { + /* Make certain we've cleaned up all the shared file objects */ + H5F_sfile_assert_num(0); + + /* Destroy the file object id group */ + n += (H5I_dec_type_ref(H5I_FILE) > 0); + + /* Mark closed */ + if(0 == n) + H5_PKG_INIT_VAR = FALSE; + } /* end else */ + } /* end if */ + + FUNC_LEAVE_NOAPI(n) +} /* end H5F_term_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5F__close_cb + * + * Purpose: Closes a file or causes the close operation to be pended. + * This function is called from the API and gets called + * by H5Fclose->H5I_dec_ref->H5F__close_cb when H5I_dec_ref() + * decrements the file ID reference count to zero. The file ID + * is removed from the H5I_FILE group by H5I_dec_ref() just + * before H5F__close_cb() is called. If there are open object + * headers then the close is pended by moving the file to the + * H5I_FILE_CLOSING ID group (the f->closing contains the ID + * assigned to file). + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F__close_cb(H5VL_object_t *file_vol_obj) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(file_vol_obj); + + /* Close the file */ + if(H5VL_file_close(file_vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close file") + + /* Free the VOL object */ + if(H5VL_free_object(file_vol_obj) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDEC, FAIL, "unable to free VOL object") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__close_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5F__parse_file_lock_env_var + * + * Purpose: Parses the HDF5_USE_FILE_LOCKING environment variable. + * + * NOTE: This is done in a separate function so we can call it from + * the test code. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5F__parse_file_lock_env_var(htri_t *use_locks) +{ + char *lock_env_var = NULL; /* Environment variable pointer */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Check the file locking environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && (!HDstrcmp(lock_env_var, "FALSE") || !HDstrcmp(lock_env_var, "0"))) + *use_locks = FALSE; /* Override: Never use locks */ + else if(lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "BEST-EFFORT") || !HDstrcmp(lock_env_var, "1"))) + *use_locks = TRUE; /* Override: Always use locks */ + else + *use_locks = FAIL; /* Environment variable not set, or not set correctly */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5F__parse_file_lock_env_var() */ /*------------------------------------------------------------------------- @@ -1426,6 +1617,52 @@ H5F__dest(H5F_t *f, hbool_t flush) /*------------------------------------------------------------------------- + * Function: H5F__check_if_using_file_locks + * + * Purpose: Determines if this file will use file locks. + * + * There are three ways that file locking can be controlled: + * + * 1) The configure/cmake option that sets the H5_USE_FILE_LOCKING + * symbol (which is used as the default fapl value). + * + * 2) The H5Pset_file_locking() API call, which will override + * the configuration default. + * + * 3) The HDF5_USE_FILE_LOCKING environment variable, which overrides + * everything above. + * + * The main reason to disable file locking is to prevent errors on file + * systems where locking is not supported or has been disabled (as is + * often the case in parallel file systems). + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5F__check_if_using_file_locks(H5P_genplist_t *fapl, hbool_t *use_file_locking) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Make sure the out parameter has a value */ + *use_file_locking = TRUE; + + /* Check the fapl property */ + if(H5P_get(fapl, H5F_ACS_USE_FILE_LOCKING_NAME, use_file_locking) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get use file locking flag") + + /* Check the environment variable */ + if(use_locks_env_g != FAIL) + *use_file_locking = (use_locks_env_g == TRUE) ? TRUE : FALSE; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__check_if_using_file_locks() */ + + +/*------------------------------------------------------------------------- * Function: H5F_open * * Purpose: Opens (or creates) a file. This function understands the @@ -1514,8 +1751,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) hbool_t set_flag = FALSE; /*set the status_flags in the superblock */ hbool_t clear = FALSE; /*clear the status_flags */ hbool_t evict_on_close; /* evict on close value from plist */ - char *lock_env_var = NULL;/*env var pointer */ - hbool_t use_file_locking; /*read from env var */ + hbool_t use_file_locking = TRUE; /* Using file locks? */ hbool_t ci_load = FALSE; /* whether MDC ci load requested */ hbool_t ci_write = FALSE; /* whether MDC CI write requested */ H5F_t *ret_value = NULL; /*actual return value */ @@ -1533,15 +1769,13 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) if(NULL == (drvr = H5FD_get_class(fapl_id))) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to retrieve VFL class") - /* Check the environment variable that determines if we care - * about file locking. File locking should be used unless explicitly - * disabled. - */ - lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); - if(lock_env_var && !HDstrcmp(lock_env_var, "FALSE")) - use_file_locking = FALSE; - else - use_file_locking = TRUE; + /* Get the file access property list, for future queries */ + if(NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list") + + /* Check if we are using file locking */ + if (H5F__check_if_using_file_locks(a_plist, &use_file_locking) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to get file locking flag") /* * Opening a file is a two step process. First we try to open the @@ -1618,8 +1852,8 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) if(H5FD_lock(lf, (hbool_t)((flags & H5F_ACC_RDWR) ? TRUE : FALSE)) < 0) { /* Locking failed - Closing will remove the lock */ if(H5FD_close(lf) < 0) - HDONE_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to close low-level file info") - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to lock the file") + HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, NULL, "unable to close low-level file info") + HGOTO_ERROR(H5E_FILE, H5E_CANTLOCKFILE, NULL, "unable to lock the file") } /* end if */ /* Create the 'top' file structure */ @@ -1651,10 +1885,6 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) shared = file->shared; lf = shared->lf; - /* Get the file access property list, for future queries */ - if(NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list") - /* Check if page buffering is enabled */ if(H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &page_buf_size) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get page buffer size") @@ -1795,7 +2025,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) /* Remove the file lock for SWMR_WRITE */ if(use_file_locking && (H5F_INTENT(file) & H5F_ACC_SWMR_WRITE)) { if(H5FD_unlock(file->shared->lf) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to unlock the file") + HGOTO_ERROR(H5E_FILE, H5E_CANTUNLOCKFILE, NULL, "unable to unlock the file") } /* end if */ } /* end if */ else { /* H5F_ACC_RDONLY: check consistency of status_flags */ diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 46185ff..3a39e67 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -384,6 +384,7 @@ struct H5F_t { unsigned nmounts; /* Number of children mounted to this file */ }; + /*****************************/ /* Package Private Variables */ /*****************************/ @@ -394,6 +395,10 @@ H5FL_EXTERN(H5F_t); /* Declare a free list to manage the H5F_shared_t struct */ H5FL_EXTERN(H5F_shared_t); +/* Whether or not to use file locking (based on the environment variable) + * FAIL means ignore the environment variable. + */ +H5_DLLVAR htri_t use_locks_env_g; /******************************/ /* Package Private Prototypes */ @@ -412,6 +417,7 @@ H5_DLL herr_t H5F__start_swmr_write(H5F_t *f); H5_DLL herr_t H5F__close(H5F_t *f); H5_DLL herr_t H5F__set_libver_bounds(H5F_t *f, H5F_libver_t low, H5F_libver_t high); H5_DLL herr_t H5F__get_cont_info(const H5F_t *f, H5VL_file_cont_info_t *info); +H5_DLL herr_t H5F__parse_file_lock_env_var(htri_t *use_locks); /* File mount related routines */ H5_DLL herr_t H5F__mount(H5G_loc_t *loc, const char *name, H5F_t *child, hid_t plist_id); @@ -472,6 +478,7 @@ H5_DLL herr_t H5F__check_cached_stab_test(hid_t file_id); H5_DLL herr_t H5F__get_maxaddr_test(hid_t file_id, haddr_t *maxaddr); H5_DLL herr_t H5F__get_sbe_addr_test(hid_t file_id, haddr_t *sbe_addr); H5_DLL htri_t H5F__same_file_test(hid_t file_id1, hid_t file_id2); +H5_DLL herr_t H5F__reparse_file_lock_variable_test(void); #endif /* H5F_TESTING */ #endif /* _H5Fpkg_H */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index c5d4c89..5fa4f7f 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -518,6 +518,8 @@ typedef struct H5F_t H5F_t; #define H5F_ACS_PAGE_BUFFER_SIZE_NAME "page_buffer_size" /* the maximum size for the page buffer cache */ #define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME "page_buffer_min_meta_perc" /* the min metadata percentage for the page buffer cache */ #define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME "page_buffer_min_raw_perc" /* the min raw data percentage for the page buffer cache */ +#define H5F_ACS_USE_FILE_LOCKING_NAME "use_file_locking" /* whether or not we use file locks for SWMR control and to prevent multiple writers */ +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME "ignore_disabled_file_locks" /* whether or not we ignore "locks disabled" errors */ #ifdef H5_HAVE_PARALLEL #define H5F_ACS_MPI_PARAMS_COMM_NAME "mpi_params_comm" /* the MPI communicator */ #define H5F_ACS_MPI_PARAMS_INFO_NAME "mpi_params_info" /* the MPI info struct */ diff --git a/src/H5Ftest.c b/src/H5Ftest.c index 49a2a22..f17af2f 100644 --- a/src/H5Ftest.c +++ b/src/H5Ftest.c @@ -267,3 +267,36 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F__same_file_test() */ + +/*------------------------------------------------------------------------- + * Function: H5F__reparse_file_lock_variable_test + * + * Purpose: Re-parse the file locking environment variable. + * + * Since getenv(3) is fairly expensive, we only parse it once, + * when the library opens. This test function is used to + * re-parse the environment variable after we've changed it + * with setnev(3). + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Summer 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F__reparse_file_lock_variable_test(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check the file locking environment variable */ + if(H5F__parse_file_lock_env_var(&use_locks_env_g) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to parse file locking environment variable") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__reparse_file_lock_variable_test() */ + diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 30b590f..7b332d0 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -281,6 +281,29 @@ #define H5F_ACS_VOL_CONN_COPY H5P__facc_vol_copy #define H5F_ACS_VOL_CONN_CMP H5P__facc_vol_cmp #define H5F_ACS_VOL_CONN_CLOSE H5P__facc_vol_close +/* Definition for using file locking or not. The default is set + * via the configure step. + */ +#define H5F_ACS_USE_FILE_LOCKING_SIZE sizeof(hbool_t) +#if defined H5_USE_FILE_LOCKING && H5_USE_FILE_LOCKING +#define H5F_ACS_USE_FILE_LOCKING_DEF TRUE +#else +#define H5F_ACS_USE_FILE_LOCKING_DEF FALSE +#endif +#define H5F_ACS_USE_FILE_LOCKING_ENC H5P__encode_hbool_t +#define H5F_ACS_USE_FILE_LOCKING_DEC H5P__decode_hbool_t +/* Definition for whether we ignore file locking errors when we can + * tell that file locking has been disabled on the file system. + * The default is set via the configure step. + */ +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_SIZE sizeof(hbool_t) +#if defined H5_IGNORE_DISABLED_FILE_LOCKS && H5_IGNORE_DISABLED_FILE_LOCKS +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF TRUE +#else +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF FALSE +#endif +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_ENC H5P__encode_hbool_t +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEC H5P__decode_hbool_t /******************/ @@ -447,6 +470,8 @@ static const H5AC_cache_image_config_t H5F_def_mdc_initCacheImageCfg_g = H5F_ACS static const size_t H5F_def_page_buf_size_g = H5F_ACS_PAGE_BUFFER_SIZE_DEF; /* Default page buffer size */ static const unsigned H5F_def_page_buf_min_meta_perc_g = H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEF; /* Default page buffer minimum metadata size */ static const unsigned H5F_def_page_buf_min_raw_perc_g = H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF; /* Default page buffer mininum raw data size */ +static const hbool_t H5F_def_use_file_locking_g = H5F_ACS_USE_FILE_LOCKING_DEF; /* Default use file locking flag */ +static const hbool_t H5F_def_ignore_disabled_file_locks_g = H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF; /* Default ignore disabled file locks flag */ /*------------------------------------------------------------------------- @@ -705,6 +730,16 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) H5F_ACS_VOL_CONN_DEL, H5F_ACS_VOL_CONN_COPY, H5F_ACS_VOL_CONN_CMP, H5F_ACS_VOL_CONN_CLOSE) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the use file locking flag */ + if(H5P__register_real(pclass, H5F_ACS_USE_FILE_LOCKING_NAME, H5F_ACS_USE_FILE_LOCKING_SIZE, &H5F_def_use_file_locking_g, + NULL, NULL, NULL, H5F_ACS_USE_FILE_LOCKING_ENC, H5F_ACS_USE_FILE_LOCKING_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the ignore disabled file locks flag */ + if(H5P__register_real(pclass, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_SIZE, &H5F_def_ignore_disabled_file_locks_g, + NULL, NULL, NULL, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_ENC, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__facc_reg_prop() */ @@ -4551,6 +4586,98 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Pget_evict_on_close() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_file_locking + * + * Purpose: Sets the file locking property values. + * + * Overrides the default file locking flag setting that was + * set when the library was configured. + * + * Can be overridden by the HDF5_USE_FILE_LOCKING environment + * variable. + * + * File locking is used when creating/opening a file to prevent + * problematic file accesses. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Spring 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_file_locking(hid_t fapl_id, hbool_t use_file_locking, hbool_t ignore_when_disabled) +{ + H5P_genplist_t *plist; /* property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "ibb", fapl_id, use_file_locking, ignore_when_disabled); + + /* Make sure this is a fapl */ + if(TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not a file access plist") + + /* Get the plist structure */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Set values */ + if(H5P_set(plist, H5F_ACS_USE_FILE_LOCKING_NAME, &use_file_locking) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set use file locking property") + if(H5P_set(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &ignore_when_disabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set ignore disabled file locks property") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_file_locking() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_file_locking + * + * Purpose: Gets the file locking property values. + * + * File locking is used when creating/opening a file to prevent + * problematic file accesses. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Spring 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_file_locking(hid_t fapl_id, hbool_t *use_file_locking, hbool_t *ignore_when_disabled) +{ + H5P_genplist_t *plist; /* property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "i*b*b", fapl_id, use_file_locking, ignore_when_disabled); + + /* Make sure this is a fapl */ + if(TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not an access plist") + + /* Get the plist structure */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get values */ + if(H5P_get(plist, H5F_ACS_USE_FILE_LOCKING_NAME, use_file_locking) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get use file locking property") + if(H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, ignore_when_disabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get ignore disabled file locks property") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_file_locking() */ + #ifdef H5_HAVE_PARALLEL /*------------------------------------------------------------------------- diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index bb33561..9dfa050 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -380,6 +380,8 @@ H5_DLL herr_t H5Pset_mdc_log_options(hid_t plist_id, hbool_t is_enabled, const c H5_DLL herr_t H5Pget_mdc_log_options(hid_t plist_id, hbool_t *is_enabled, char *location, size_t *location_size, hbool_t *start_on_access); H5_DLL herr_t H5Pset_evict_on_close(hid_t fapl_id, hbool_t evict_on_close); H5_DLL herr_t H5Pget_evict_on_close(hid_t fapl_id, hbool_t *evict_on_close); +H5_DLL herr_t H5Pset_file_locking(hid_t fapl_id, hbool_t use_file_locking, hbool_t ignore_when_disabled); +H5_DLL herr_t H5Pget_file_locking(hid_t fapl_id, hbool_t *use_file_locking, hbool_t *ignore_when_disabled); #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5Pset_all_coll_metadata_ops(hid_t plist_id, hbool_t is_collective); H5_DLL herr_t H5Pget_all_coll_metadata_ops(hid_t plist_id, hbool_t *is_collective); diff --git a/src/H5err.txt b/src/H5err.txt index 9fec521..24ac2ac 100644 --- a/src/H5err.txt +++ b/src/H5err.txt @@ -136,6 +136,8 @@ MINOR, FILEACC, H5E_BADFILE, Bad file ID accessed MINOR, FILEACC, H5E_TRUNCATED, File has been truncated MINOR, FILEACC, H5E_MOUNT, File mount error MINOR, FILEACC, H5E_CANTDELETEFILE, Unable to delete file +MINOR, FILEACC, H5E_CANTLOCKFILE, Unable to lock file +MINOR, FILEACC, H5E_CANTUNLOCKFILE, Unable to unlock file # Generic low-level file I/O errors MINOR, FILE, H5E_SEEKERROR, Seek failed diff --git a/src/H5private.h b/src/H5private.h index 836d7d5..08aebdf 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -888,8 +888,8 @@ H5_DLL H5_ATTR_CONST int Nflock(int fd, int operation); #ifndef HDflock /* NOTE: flock(2) is not present on all POSIX systems. * If it is not present, we try a flock() equivalent based on - * fcntl(2), then fall back to a function that always fails if - * it is not present at all (Windows uses a separate Wflock() + * fcntl(2), then fall back to a function that always succeeds + * if it is not present at all (Windows uses a separate Wflock() * function). */ #if defined(H5_HAVE_FLOCK) diff --git a/src/H5system.c b/src/H5system.c index b3db39c..a6a68b3 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -689,14 +689,14 @@ Pflock(int fd, int operation) { * Purpose: Wrapper function for systems where no file locking is * available. * - * Return: Failure: -1 (always fails) + * Return: 0 (success) * *------------------------------------------------------------------------- */ int H5_ATTR_CONST Nflock(int H5_ATTR_UNUSED fd, int H5_ATTR_UNUSED operation) { - /* just fail */ - return -1; + /* just succeed */ + return 0; } /* end Nflock() */ diff --git a/src/libhdf5.settings.in b/src/libhdf5.settings.in index fb9deec..98390b8 100644 --- a/src/libhdf5.settings.in +++ b/src/libhdf5.settings.in @@ -89,5 +89,6 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ Using memory checker: @USINGMEMCHECKER@ Memory allocation sanity checks: @MEMORYALLOCSANITYCHECK@ Function stack tracing: @CODESTACK@ + Use file locking: @DESIRED_FILE_LOCKING@ Strict file format checks: @STRICT_FORMAT_CHECKS@ Optimization instrumentation: @INSTRUMENT_LIBRARY@ diff --git a/test/h5test.c b/test/h5test.c index 1b445dd..858f30a 100644 --- a/test/h5test.c +++ b/test/h5test.c @@ -2236,3 +2236,61 @@ done: return ret_value; } /* end h5_duplicate_file_by_bytes() */ +/*------------------------------------------------------------------------- + * Function: h5_check_if_file_locking_enabled + * + * Purpose: Checks if file locking is enabled on this file system. + * + * Return: SUCCEED/FAIL + * are_enabled will be FALSE if file locking is disabled on + * the file system of if there were errors. + * + *------------------------------------------------------------------------- + */ +herr_t +h5_check_if_file_locking_enabled(hbool_t *are_enabled) +{ + const char *filename = "locking_test_file"; + mode_t mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH; + int fd = -1; + + *are_enabled = TRUE; + + if((fd = HDcreat(filename, mode)) < 0) + goto error; + + /* Test HDflock() to see if it works */ + if(HDflock(fd, LOCK_EX | LOCK_NB) < 0) { + if(ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. This is most frequently used on + * Lustre. If we also want to check for disabled NFS locks + * we'll need to check for ENOLCK, too. That isn't done by + * default here since that could also represent an actual + * error condition. + */ + errno = 0; + *are_enabled = FALSE; + } + else + goto error; + } + if(HDflock(fd, LOCK_UN) < 0) + goto error; + + if(HDclose(fd) < 0) + goto error; + if(HDremove(filename) < 0) + goto error; + + return SUCCEED; + +error: + *are_enabled = FALSE; + if (fd > -1) { + HDclose(fd); + HDremove(filename); + } + return FAIL; +} /* end h5_check_if_file_locking_enabled() */ + diff --git a/test/h5test.h b/test/h5test.h index 3eeb1f8..697bde6 100644 --- a/test/h5test.h +++ b/test/h5test.h @@ -211,6 +211,7 @@ H5TEST_DLL H5VL_class_t *h5_get_dummy_vol_class(void); H5TEST_DLL const char *h5_get_version_string(H5F_libver_t libver); H5TEST_DLL int h5_compare_file_bytes(char *fname1, char *fname2); H5TEST_DLL int h5_duplicate_file_by_bytes(const char *orig, const char *dest); +H5TEST_DLL herr_t h5_check_if_file_locking_enabled(hbool_t *are_enabled); /* Functions that will replace components of a FAPL */ H5TEST_DLL herr_t h5_get_vfd_fapl(hid_t fapl_id); diff --git a/test/swmr.c b/test/swmr.c index bab91bd..a839a74 100644 --- a/test/swmr.c +++ b/test/swmr.c @@ -91,7 +91,7 @@ static int test_file_lock_concur(hid_t fapl); static int test_file_lock_swmr_concur(hid_t fapl); /* Test file lock environment variable */ -static int test_file_lock_env_var(hid_t fapl); +static int test_file_locking(hid_t in_fapl, hbool_t turn_locking_on, hbool_t env_var_override); /* Tests for SWMR VFD flag */ static int test_swmr_vfd_flag(void); @@ -4256,8 +4256,11 @@ test_file_lock_same(hid_t in_fapl) /* Output message about test being performed */ TESTING("File open with different combinations of flags--single process access"); + /* Set locking in the fapl */ if((fapl = H5Pcopy(in_fapl)) < 0) FAIL_STACK_ERROR + if(H5Pset_file_locking(fapl, TRUE, TRUE) < 0) + FAIL_STACK_ERROR /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[1], fapl, filename, sizeof(filename)); @@ -4416,9 +4419,11 @@ test_file_lock_swmr_same(hid_t in_fapl) /* Output message about test being performed */ TESTING("File open with different combinations of flags + SWMR flags--single process access"); - /* Get a copy of the parameter in_fapl */ + /* Set locking in the fapl */ if((fapl = H5Pcopy(in_fapl)) < 0) FAIL_STACK_ERROR + if(H5Pset_file_locking(fapl, TRUE, TRUE) < 0) + FAIL_STACK_ERROR /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[1], fapl, filename, sizeof(filename)); @@ -4726,8 +4731,11 @@ test_file_lock_concur(hid_t in_fapl) /* Output message about test being performed */ TESTING("File open with different combinations of flags--concurrent access"); + /* Set locking in the fapl */ if((fapl = H5Pcopy(in_fapl)) < 0) FAIL_STACK_ERROR + if(H5Pset_file_locking(fapl, TRUE, TRUE) < 0) + FAIL_STACK_ERROR /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[1], fapl, filename, sizeof(filename)); @@ -5102,8 +5110,11 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Output message about test being performed */ TESTING("File open with different combintations of flags + SWMR flags--concurrent access"); + /* Set locking in the fapl */ if((fapl = H5Pcopy(in_fapl)) < 0) FAIL_STACK_ERROR + if(H5Pset_file_locking(fapl, TRUE, TRUE) < 0) + FAIL_STACK_ERROR /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[2], fapl, filename, sizeof(filename)); @@ -5134,7 +5145,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5155,7 +5166,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(child_fid == FAIL) HDexit(EXIT_SUCCESS); - /* Close the pipe */ + /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); @@ -5205,13 +5216,13 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Fork child process */ if((childpid = HDfork()) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; - /* Close unused write end for out_pdf */ + /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) HDexit(EXIT_FAILURE); @@ -5230,7 +5241,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(child_fid == FAIL) HDexit(EXIT_SUCCESS); - /* Close the pipe */ + /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); @@ -5243,7 +5254,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Open the test file */ if((fid = H5Fopen(filename, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR /* Notify child process */ notify = 1; @@ -5256,7 +5267,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Wait for child process to complete */ if(HDwaitpid(childpid, &child_status, child_wait_option) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR /* Check if child terminated normally */ if(WIFEXITED(child_status)) { @@ -5284,7 +5295,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5305,7 +5316,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(child_fid == FAIL) HDexit(EXIT_SUCCESS); - /* Close the pipe */ + /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); @@ -5354,11 +5365,11 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Fork child process */ if((childpid = HDfork()) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5379,7 +5390,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(child_fid == FAIL) HDexit(EXIT_SUCCESS); - /* Close the pipe */ + /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); @@ -5392,7 +5403,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Open the test file */ if((fid = H5Fopen(filename, H5F_ACC_RDWR|H5F_ACC_SWMR_WRITE, fapl)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR /* Notify child process */ notify = 1; @@ -5405,7 +5416,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Wait for child process to complete */ if(HDwaitpid(childpid, &child_status, child_wait_option) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR /* Check if child terminated normally */ if(WIFEXITED(child_status)) { @@ -5428,11 +5439,11 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Fork child process */ if((childpid = HDfork()) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5456,7 +5467,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) HDexit(EXIT_SUCCESS); } - /* Close the pipe */ + /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); @@ -5469,7 +5480,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Open the test file */ if((fid = H5Fopen(filename, H5F_ACC_RDWR|H5F_ACC_SWMR_WRITE, fapl)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR /* Notify child process */ notify = 1; @@ -5482,7 +5493,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Wait for child process to complete */ if(HDwaitpid(childpid, &child_status, child_wait_option) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR /* Check if child terminated normally */ if(WIFEXITED(child_status)) { @@ -5509,7 +5520,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5530,7 +5541,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(child_fid == FAIL) HDexit(EXIT_SUCCESS); - /* Close the pipe */ + /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); @@ -5584,7 +5595,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5598,14 +5609,14 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Open the test file */ H5E_BEGIN_TRY { - child_fid = H5Fopen(filename, H5F_ACC_RDWR, H5P_DEFAULT); + child_fid = H5Fopen(filename, H5F_ACC_RDWR, fapl); } H5E_END_TRY; /* Should fail */ if(child_fid == FAIL) HDexit(EXIT_SUCCESS); - /* Close the pipe */ + /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); @@ -5659,7 +5670,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5734,7 +5745,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5812,7 +5823,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5835,7 +5846,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) HDexit(EXIT_SUCCESS); } - /* Close the pipe */ + /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); @@ -5889,7 +5900,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5910,7 +5921,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(child_fid == FAIL) HDexit(EXIT_SUCCESS); - /* Close the pipe */ + /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); @@ -5922,7 +5933,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) FAIL_STACK_ERROR /* Open the test file */ - if((fid = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) + if((fid = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) FAIL_STACK_ERROR /* Notify child process */ @@ -5964,7 +5975,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -6059,7 +6070,7 @@ error: ** *****************************************************************/ static int -test_file_lock_env_var(hid_t in_fapl) +test_file_locking(hid_t in_fapl, hbool_t turn_locking_on, hbool_t env_var_override) { #if !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) SKIPPED(); @@ -6074,18 +6085,40 @@ test_file_lock_env_var(hid_t in_fapl) int child_wait_option=0; /* Options passed to waitpid */ int out_pdf[2]; int notify = 0; + int exit_status = 0; + herr_t ret; + + if (turn_locking_on && env_var_override) + TESTING("File locking: ON w/ env var override") + else if (turn_locking_on && !env_var_override) + TESTING("File locking: ON") + else if (!turn_locking_on && env_var_override) + TESTING("File locking: OFF w/ env var override") + else + TESTING("File locking: OFF") - - TESTING("File locking environment variable"); - - - /* Set the environment variable */ - if(HDsetenv("HDF5_USE_FILE_LOCKING", "FALSE", TRUE) < 0) + /* Copy the incoming fapl */ + if((fapl = H5Pcopy(in_fapl)) < 0) TEST_ERROR - if((fapl = H5Pcopy(in_fapl)) < 0) + /* Set locking in the fapl */ + if(H5Pset_file_locking(fapl, turn_locking_on ? TRUE : FALSE, TRUE) < 0) TEST_ERROR + /* If requested, set the environment variable */ + if (env_var_override) { + if(HDsetenv("HDF5_USE_FILE_LOCKING", turn_locking_on ? "FALSE" : "TRUE", TRUE) < 0) + TEST_ERROR + if(H5F__reparse_file_lock_variable_test() < 0) + TEST_ERROR + } + else { + if(HDsetenv("HDF5_USE_FILE_LOCKING", "", TRUE) < 0) + TEST_ERROR + if(H5F__reparse_file_lock_variable_test() < 0) + TEST_ERROR + } + /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[1], fapl, filename, sizeof(filename)); @@ -6097,10 +6130,8 @@ test_file_lock_env_var(hid_t in_fapl) if(H5Fclose(fid) < 0) TEST_ERROR - /* Open a file for read-only and then read-write. This would - * normally fail due to the file locking scheme but should - * pass when the environment variable is set to disable file - * locking. + /* Open a file for read-only and then read-write. This will fail + * when the locking scheme is turned on. */ /* Create 1 pipe */ @@ -6115,7 +6146,7 @@ test_file_lock_env_var(hid_t in_fapl) /* Child process */ - hid_t child_fid; /* File ID */ + hid_t child_fid = H5I_INVALID_HID; /* File ID */ int child_notify = 0; /* Close unused write end for out_pdf */ @@ -6126,18 +6157,23 @@ test_file_lock_env_var(hid_t in_fapl) while(child_notify != 1) { if(HDread(out_pdf[0], &child_notify, sizeof(int)) < 0) HDexit(EXIT_FAILURE); - } /* end while */ + } - /* Open the test file */ - if((child_fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) - TEST_ERROR + /* Open and close the test file */ + H5E_BEGIN_TRY { + child_fid = H5Fopen(filename, H5F_ACC_RDWR, fapl); + ret = H5Fclose(child_fid); + } H5E_END_TRY; /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); - HDexit(EXIT_SUCCESS); - } /* end if */ + if(H5I_INVALID_HID == child_fid || FAIL == ret) + HDexit(EXIT_FAILURE); + else + HDexit(EXIT_SUCCESS); + } /* end child process work */ /* close unused read end for out_pdf */ if(HDclose(out_pdf[0]) < 0) @@ -6160,15 +6196,28 @@ test_file_lock_env_var(hid_t in_fapl) if(HDwaitpid(childpid, &child_status, child_wait_option) < 0) TEST_ERROR - /* Check if child terminated normally */ - if(WIFEXITED(child_status)) { - /* Check exit status of the child */ - if(WEXITSTATUS(child_status) != 0) - TEST_ERROR - } /* end if */ + /* Check exit status of the child */ + if(WIFEXITED(child_status)) + exit_status = WEXITSTATUS(child_status); else TEST_ERROR + /* The child process should have passed or failed as follows: + * + * locks on: FAIL + * locks off: PASS + * locks on, env var override: PASS + * locks off, env var override: FAIL + */ + if(turn_locking_on && !env_var_override && (0 == exit_status)) + TEST_ERROR + else if(!turn_locking_on && !env_var_override && (0 != exit_status)) + TEST_ERROR + else if(turn_locking_on && env_var_override && (0 != exit_status)) + TEST_ERROR + else if(!turn_locking_on && env_var_override && (0 == exit_status)) + TEST_ERROR + /* Close the file */ if(H5Fclose(fid) < 0) TEST_ERROR @@ -6192,7 +6241,7 @@ error: #endif /* !(defined(H5_HAVE_FORK && defined(H5_HAVE_WAITPID)) */ -} /* end test_file_lock_env_var() */ +} /* end test_file_locking() */ static int @@ -7019,7 +7068,7 @@ error: H5Fclose(fid3); } H5E_END_TRY; - return -1; + return 1; } /* test_multiple_same() */ @@ -7036,6 +7085,7 @@ main(void) char *driver = NULL; /* VFD string (from env variable) */ char *lock_env_var = NULL; /* file locking env var pointer */ hbool_t use_file_locking; /* read from env var */ + hbool_t file_locking_enabled = FALSE; /* Checks if the file system supports locks */ /* Skip this test if SWMR I/O is not supported for the VFD specified * by the environment variable. @@ -7056,6 +7106,13 @@ main(void) else use_file_locking = TRUE; + /* Check if file locking is enabled on this file system */ + if(use_file_locking) + if(h5_check_if_file_locking_enabled(&file_locking_enabled) < 0) { + HDprintf("Error when determining if file locks are enabled\n"); + return EXIT_FAILURE; + } + /* Set up */ h5_reset(); @@ -7097,7 +7154,7 @@ main(void) nerrors += test_append_flush_dataset_fixed(fapl); nerrors += test_append_flush_dataset_multiple(fapl); - if(use_file_locking) { + if(use_file_locking && file_locking_enabled) { /* * Tests for: * file open flags--single process access @@ -7127,7 +7184,12 @@ main(void) /* This test changes the HDF5_USE_FILE_LOCKING environment variable * so it should be run last. */ - nerrors += test_file_lock_env_var(fapl); + if (use_file_locking && file_locking_enabled) { + nerrors += test_file_locking(fapl, TRUE, TRUE); + nerrors += test_file_locking(fapl, TRUE, FALSE); + nerrors += test_file_locking(fapl, FALSE, TRUE); + nerrors += test_file_locking(fapl, FALSE, FALSE); + } if(nerrors) goto error; |