From 1dbb83aabd9404f4b626e19193b7472a653ed8f5 Mon Sep 17 00:00:00 2001 From: Dana Robinson Date: Thu, 25 Mar 2021 00:32:40 -0700 Subject: Brings file locking changes from develop --- config/cmake/ConfigureChecks.cmake | 25 +++ config/cmake/H5pubconf.h.in | 6 + config/cmake/libhdf5.settings.cmake.in | 1 + config/cmake/scripts/HDF5config.cmake | 2 +- configure.ac | 50 +++++ src/H5F.c | 145 -------------- src/H5FDcore.c | 101 ++++++---- src/H5FDdirect.c | 68 +++++-- src/H5FDfamily.c | 40 ++-- src/H5FDlog.c | 66 +++++-- src/H5FDmpi.c | 31 +-- src/H5FDmpio.c | 10 +- src/H5FDmulti.c | 13 +- src/H5FDpkg.h | 8 +- src/H5FDpublic.h | 6 +- src/H5FDros3.c | 2 +- src/H5FDsec2.c | 70 +++++-- src/H5FDstdio.c | 108 ++++++---- src/H5Fint.c | 314 +++++++++++++++++++++++++++--- src/H5Fpkg.h | 16 +- src/H5Fprivate.h | 14 +- src/H5Fquery.c | 34 +++- src/H5Ftest.c | 100 ++++++---- src/H5Pfapl.c | 138 ++++++++++++- src/H5Ppublic.h | 2 + src/H5err.txt | 2 + src/H5private.h | 4 +- src/H5system.c | 6 +- src/libhdf5.settings.in | 1 + test/h5test.c | 60 +++++- test/h5test.h | 7 +- test/swmr.c | 346 +++++++++++++++++++++++++-------- test/tfile.c | 76 +++++--- tools/src/h5repack/h5repack_copy.c | 67 +++++-- 34 files changed, 1372 insertions(+), 567 deletions(-) diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake index 4805eeb..4e31b38 100644 --- a/config/cmake/ConfigureChecks.cmake +++ b/config/cmake/ConfigureChecks.cmake @@ -61,6 +61,31 @@ if (HDF5_ENABLE_CODESTACK) endif () MARK_AS_ADVANCED (HDF5_ENABLE_CODESTACK) +# ---------------------------------------------------------------------- +# Check if they would like to use file locking by default +#----------------------------------------------------------------------------- +option (HDF5_USE_FILE_LOCKING "Use file locking by default (mainly for SWMR)" ON) +if (HDF5_USE_FILE_LOCKING) + set (${HDF_PREFIX}_USE_FILE_LOCKING 1) +endif () + +# ---------------------------------------------------------------------- +# Check if they would like to ignore file locks when disabled on a file system +#----------------------------------------------------------------------------- +option (HDF5_IGNORE_DISABLED_FILE_LOCKS "Ignore file locks when disabled on file system" ON) +if (HDF5_IGNORE_DISABLED_FILE_LOCKS) + set (${HDF_PREFIX}_IGNORE_DISABLED_FILE_LOCKS 1) +endif () + +# Set the libhdf5.settings file variable +if (HDF5_IGNORE_DISABLED_FILE_LOCKS AND HDF5_USE_FILE_LOCKING) + set (HDF5_FILE_LOCKING_SETTING "best-effort") +elseif (HDF5_IGNORE_DISABLED_FILE_LOCKS) + set (HDF5_FILE_LOCKING_SETTING "yes") +else () + set (HDF5_FILE_LOCKING_SETTING "no") +endif () + #----------------------------------------------------------------------------- # Are we going to use HSIZE_T #----------------------------------------------------------------------------- diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index f9e9f10..62cfcab 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -495,6 +495,9 @@ /* Define if the compiler understands __inline__ */ #cmakedefine H5_HAVE___INLINE__ @H5_HAVE___INLINE__@ +/* Define if the library will ignore file locks when disabled */ +#cmakedefine H5_IGNORE_DISABLED_FILE_LOCKS @H5_IGNORE_DISABLED_FILE_LOCKS@ + /* Define if the high-level library headers should be included in hdf5.h */ #cmakedefine H5_INCLUDE_HL @H5_INCLUDE_HL@ @@ -735,6 +738,9 @@ /* Define using v1.14 public API symbols by default */ #cmakedefine H5_USE_114_API_DEFAULT @H5_USE_114_API_DEFAULT@ +/* Define if the library will use file locking */ +#cmakedefine H5_USE_FILE_LOCKING @H5_USE_FILE_LOCKING@ + /* Define if a memory checking tool will be used on the library, to cause library to be very picky about memory operations and also disable the internal free list manager code. */ diff --git a/config/cmake/libhdf5.settings.cmake.in b/config/cmake/libhdf5.settings.cmake.in index b745765..ebcbd61 100644 --- a/config/cmake/libhdf5.settings.cmake.in +++ b/config/cmake/libhdf5.settings.cmake.in @@ -85,5 +85,6 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ Using memory checker: @HDF5_ENABLE_USING_MEMCHECKER@ Memory allocation sanity checks: @HDF5_MEMORY_ALLOC_SANITY_CHECK@ Function Stack Tracing: @HDF5_ENABLE_CODESTACK@ + Use file locking: @HDF5_FILE_LOCKING_SETTING@ Strict File Format Checks: @HDF5_STRICT_FORMAT_CHECKS@ Optimization Instrumentation: @HDF5_Enable_Instrument@ diff --git a/config/cmake/scripts/HDF5config.cmake b/config/cmake/scripts/HDF5config.cmake index bd48588..7005d3a 100644 --- a/config/cmake/scripts/HDF5config.cmake +++ b/config/cmake/scripts/HDF5config.cmake @@ -199,7 +199,7 @@ endif () #set (LOCAL_NO_PACKAGE "TRUE") ##### Following controls source update ##### #set (LOCAL_UPDATE "TRUE") -set (REPOSITORY_URL "https://git@bitbucket.hdfgroup.org/scm/hdffv/hdf5.git") +set (REPOSITORY_URL "https://github.com/HDFGroup/hdf5.git") set (REPOSITORY_BRANCH "develop") #uncomment to use a compressed source file: *.tar on linux or mac *.zip on windows diff --git a/configure.ac b/configure.ac index f1d99c8..2b8bb69 100644 --- a/configure.ac +++ b/configure.ac @@ -2378,6 +2378,56 @@ case "X-$OPTIMIZATION" in esac ## ---------------------------------------------------------------------- +## Check if file locking should be used +## +AC_MSG_CHECKING([enable file locking]) +AC_ARG_ENABLE([file-locking], + [AS_HELP_STRING([--enable-file-locking=(yes|no|best-effort)], + [Sets the default for whether or not to use file + locking when opening files. Can be overridden + with the HDF5_USE_FILE_LOCKING environment variable + and the H5Pset_file_locking() API call. + best-effort attempts to use file locking but does + not fail when file locks have been disabled on + the file system (useful with Lustre). + [default=best-effort] + ])], + [DESIRED_FILE_LOCKING=$enableval]) + +## Set defaults +if test "X-$DESIRED_FILE_LOCKING" = X- ; then + DESIRED_FILE_LOCKING=best-effort +fi + +## Allow this variable to be substituted in +## other files (src/libhdf5.settings.in, etc.) +AC_SUBST([DESIRED_FILE_LOCKING]) +AC_SUBST([USE_FILE_LOCKING]) +AC_SUBST([IGNORE_DISABLED_FILE_LOCKS]) + +case "X-$DESIRED_FILE_LOCKING" in + X-best-effort) + AC_MSG_RESULT([best-effort]) + AC_DEFINE([USE_FILE_LOCKING], [1], + [Define if the library will use file locking]) + AC_DEFINE([IGNORE_DISABLED_FILE_LOCKS], [1], + [Define if the library will ignore file locks when disabled]) + ;; + X-yes) + AC_MSG_RESULT([yes]) + AC_DEFINE([USE_FILE_LOCKING], [1], + [Define if the library will use file locking]) + ;; + X-no) + AC_MSG_RESULT([no]) + ;; + *) + AC_MSG_ERROR([Unrecognized value: $USE_FILE_LOCKING]) + ;; +esac + + +## ---------------------------------------------------------------------- ## Enable/disable internal package-level debugging output ## AC_MSG_CHECKING([for internal debug output]) diff --git a/src/H5F.c b/src/H5F.c index d8638f8..95e218f 100644 --- a/src/H5F.c +++ b/src/H5F.c @@ -68,8 +68,6 @@ typedef struct { /* Local Prototypes */ /********************/ -static herr_t H5F__close_cb(H5VL_object_t *file_vol_obj); - /* Callback for getting object counts in a file */ static int H5F__get_all_count_cb(void H5_ATTR_UNUSED *obj_ptr, hid_t H5_ATTR_UNUSED obj_id, void *key); @@ -80,9 +78,6 @@ static int H5F__get_all_ids_cb(void H5_ATTR_UNUSED *obj_ptr, hid_t obj_id, void /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ /*****************************/ @@ -97,146 +92,6 @@ H5FL_EXTERN(H5VL_t); /* Declare a free list to manage the H5VL_object_t struct */ H5FL_EXTERN(H5VL_object_t); -/* File ID class */ -static const H5I_class_t H5I_FILE_CLS[1] = {{ - H5I_FILE, /* ID class value */ - 0, /* Class flags */ - 0, /* # of reserved IDs for class */ - (H5I_free_t)H5F__close_cb /* Callback routine for closing objects of this class */ -}}; - -/*------------------------------------------------------------------------- - * Function: H5F_init - * - * Purpose: Initialize the interface from some other layer. - * - * Return: Success: non-negative - * - * Failure: negative - *------------------------------------------------------------------------- - */ -herr_t -H5F_init(void) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - /* FUNC_ENTER() does all the work */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5F_init() */ - -/*-------------------------------------------------------------------------- -NAME - H5F__init_package -- Initialize interface-specific information -USAGE - herr_t H5F__init_package() -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. - ---------------------------------------------------------------------------*/ -herr_t -H5F__init_package(void) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* - * Initialize the atom group for the file IDs. - */ - if (H5I_register_type(H5I_FILE_CLS) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to initialize interface") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5F__init_package() */ - -/*------------------------------------------------------------------------- - * Function: H5F_term_package - * - * Purpose: Terminate this interface: free all memory and reset global - * variables to their initial values. Release all ID groups - * associated with this interface. - * - * Return: Success: Positive if anything was done that might - * have affected other interfaces; - * zero otherwise. - * - * Failure: Never fails - * - *------------------------------------------------------------------------- - */ -int -H5F_term_package(void) -{ - int n = 0; - - FUNC_ENTER_NOAPI_NOINIT_NOERR - - if (H5_PKG_INIT_VAR) { - if (H5I_nmembers(H5I_FILE) > 0) { - (void)H5I_clear_type(H5I_FILE, FALSE, FALSE); - n++; /*H5I*/ - } /* end if */ - else { - /* Make certain we've cleaned up all the shared file objects */ - H5F_sfile_assert_num(0); - - /* Destroy the file object id group */ - n += (H5I_dec_type_ref(H5I_FILE) > 0); - - /* Mark closed */ - if (0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end else */ - } /* end if */ - - FUNC_LEAVE_NOAPI(n) -} /* end H5F_term_package() */ - -/*------------------------------------------------------------------------- - * Function: H5F__close_cb - * - * Purpose: Closes a file or causes the close operation to be pended. - * This function is called from the API and gets called - * by H5Fclose->H5I_dec_ref->H5F__close_cb when H5I_dec_ref() - * decrements the file ID reference count to zero. The file ID - * is removed from the H5I_FILE group by H5I_dec_ref() just - * before H5F__close_cb() is called. If there are open object - * headers then the close is pended by moving the file to the - * H5I_FILE_CLOSING ID group (the f->closing contains the ID - * assigned to file). - * - * Return: SUCCEED/FAIL - * - *------------------------------------------------------------------------- - */ -static herr_t -H5F__close_cb(H5VL_object_t *file_vol_obj) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity check */ - HDassert(file_vol_obj); - - /* Close the file */ - if (H5VL_file_close(file_vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close file") - - /* Free the VOL object */ - if (H5VL_free_object(file_vol_obj) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTDEC, FAIL, "unable to free VOL object") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5F__close_cb() */ - /*------------------------------------------------------------------------- * Function: H5Fget_create_plist * diff --git a/src/H5FDcore.c b/src/H5FDcore.c index de8c20c..9d7b025 100644 --- a/src/H5FDcore.c +++ b/src/H5FDcore.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Tuesday, August 10, 1999 * * Purpose: A driver which stores the HDF5 data in main memory using @@ -36,6 +36,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_CORE_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* The skip list node type. Represents a region in the file. */ typedef struct H5FD_core_region_t { haddr_t start; /* Start address of the region */ @@ -56,7 +59,8 @@ typedef struct H5FD_core_t { hbool_t backing_store; /* write to file name on flush */ hbool_t write_tracking; /* Whether to track writes */ size_t bstore_page_size; /* backing store page size */ - int fd; /* backing store file descriptor */ + hbool_t ignore_disabled_file_locks; + int fd; /* backing store file descriptor */ /* Information for determining uniqueness of a file with a backing store */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely @@ -142,8 +146,8 @@ static herr_t H5FD__core_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, h const void *buf); static herr_t H5FD__core_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); static herr_t H5FD__core_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); -static herr_t H5FD_core_lock(H5FD_t *_file, hbool_t rw); -static herr_t H5FD_core_unlock(H5FD_t *_file); +static herr_t H5FD__core_lock(H5FD_t *_file, hbool_t rw); +static herr_t H5FD__core_unlock(H5FD_t *_file); static const H5FD_class_t H5FD_core_g = { "core", /* name */ @@ -175,8 +179,8 @@ static const H5FD_class_t H5FD_core_g = { H5FD__core_write, /* write */ H5FD__core_flush, /* flush */ H5FD__core_truncate, /* truncate */ - H5FD_core_lock, /* lock */ - H5FD_core_unlock, /* unlock */ + H5FD__core_lock, /* lock */ + H5FD__core_unlock, /* unlock */ NULL, /* dedup */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; @@ -278,13 +282,13 @@ H5FD__core_add_dirty_region(H5FD_core_t *file, haddr_t start, haddr_t end) else { /* Store the new item endpoint if it's bigger */ item->end = (item->end < end) ? end : item->end; - } /* end else */ - } /* end if */ + } + } else { /* Update the size of the before region */ if (b_item->end < end) b_item->end = end; - } /* end else */ + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -342,7 +346,7 @@ H5FD__core_write_to_bstore(H5FD_core_t *file, haddr_t addr, size_t size) HDoff_t offset = (HDoff_t)addr; /* Offset to write at */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC HDassert(file); @@ -414,9 +418,19 @@ done: static herr_t H5FD__init_package(void) { - herr_t ret_value = SUCCEED; + char * lock_env_var = NULL; /* Environment variable pointer */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC - FUNC_ENTER_NOAPI_NOINIT + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ if (H5FD_core_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize core VFD") @@ -795,6 +809,16 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr /* Save file image callbacks */ file->fi_callbacks = file_image_info.callbacks; + /* Check the file locking flags in the fapl */ + if (ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if (H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + if (fd >= 0) { /* Retrieve information for determining uniqueness of file */ #ifdef H5_HAVE_WIN32_API @@ -1601,12 +1625,12 @@ done: } /* end H5FD__core_truncate() */ /*------------------------------------------------------------------------- - * Function: H5FD_core_lock + * Function: H5FD__core_lock * * Purpose: To place an advisory lock on a file. - * The lock type to apply depends on the parameter "rw": - * TRUE--opens for write: an exclusive lock - * FALSE--opens for read: a shared lock + * The lock type to apply depends on the parameter "rw": + * TRUE--opens for write: an exclusive lock + * FALSE--opens for read: a shared lock * * Return: SUCCEED/FAIL * @@ -1615,13 +1639,13 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_core_lock(H5FD_t *_file, hbool_t rw) +H5FD__core_lock(H5FD_t *_file, hbool_t rw) { H5FD_core_t *file = (H5FD_core_t *)_file; /* VFD file struct */ int lock_flags; /* file locking flags */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC HDassert(file); @@ -1629,28 +1653,28 @@ H5FD_core_lock(H5FD_t *_file, hbool_t rw) * descriptor, this is a no-op. */ if (file->fd >= 0) { - /* Set exclusive or shared lock based on rw status */ lock_flags = rw ? LOCK_EX : LOCK_SH; /* Place a non-blocking lock on the file */ if (HDflock(file->fd, lock_flags | LOCK_NB) < 0) { - if (ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)") + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") } /* end if */ - - } /* end if */ + } /* end if */ done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_core_lock() */ +} /* end H5FD__core_lock() */ /*------------------------------------------------------------------------- - * Function: H5FD_core_unlock + * Function: H5FD__core_unlock * * Purpose: To remove the existing lock on the file * @@ -1661,28 +1685,27 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_core_unlock(H5FD_t *_file) +H5FD__core_unlock(H5FD_t *_file) { H5FD_core_t *file = (H5FD_core_t *)_file; /* VFD file struct */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC HDassert(file); - if (file->fd >= 0) { - + if (file->fd >= 0) if (HDflock(file->fd, LOCK_UN) < 0) { - if (ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)") + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") - } /* end if */ - - } /* end if */ + } done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_core_unlock() */ +} /* end H5FD__core_unlock() */ diff --git a/src/H5FDdirect.c b/src/H5FDdirect.c index fdbdb5e..1233799 100644 --- a/src/H5FDdirect.c +++ b/src/H5FDdirect.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Raymond Lu + * Programmer: Raymond Lu * Wednesday, 20 September 2006 * * Purpose: The Direct I/O file driver forces the data to be written to @@ -37,6 +37,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_DIRECT_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* File operations */ #define OP_UNKNOWN 0 #define OP_READ 1 @@ -70,6 +73,7 @@ typedef struct H5FD_direct_t { haddr_t pos; /*current file I/O position */ int op; /*last operation */ H5FD_direct_fapl_t fa; /*file access properties */ + hbool_t ignore_disabled_file_locks; #ifndef H5_HAVE_WIN32_API /* * On most systems the combination of device and i-node number uniquely @@ -188,10 +192,20 @@ DESCRIPTION static herr_t H5FD__init_package(void) { - herr_t ret_value = SUCCEED; + char * lock_env_var = NULL; /* Environment variable pointer */ + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if (H5FD_direct_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize direct VFD") @@ -489,7 +503,7 @@ H5FD_direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxadd /* Get the driver specific information */ if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list") - if (NULL == (fa = (H5FD_direct_fapl_t *)H5P_peek_driver_info(plist))) + if (NULL == (fa = (const H5FD_direct_fapl_t *)H5P_peek_driver_info(plist))) HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "bad VFL driver info") file->fd = fd; @@ -509,6 +523,16 @@ H5FD_direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxadd file->fa.fbsize = fa->fbsize; file->fa.cbsize = fa->cbsize; + /* Check the file locking flags in the fapl */ + if (ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if (H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + /* Try to decide if data alignment is required. The reason to check it here * is to handle correctly the case that the file is in a different file system * than the one where the program is running. @@ -529,7 +553,8 @@ H5FD_direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxadd } else { file->fa.must_align = FALSE; - HDftruncate(file->fd, (HDoff_t)0); + if (-1 == HDftruncate(file->fd, (HDoff_t)0)) + HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, NULL, "unable to truncate file") } } else { @@ -1324,17 +1349,28 @@ done: static herr_t H5FD_direct_lock(H5FD_t *_file, hbool_t rw) { - H5FD_direct_t *file = (H5FD_direct_t *)_file; /* VFD file struct */ - const int lock = rw ? LOCK_EX : LOCK_SH; - herr_t ret_value = SUCCEED; /* Return value */ + H5FD_direct_t *file = (H5FD_direct_t *)_file; /* VFD file struct */ + int lock_flags; /* file locking flags */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT HDassert(file); - /* Place the lock with non-blocking */ - if (HDflock(file->fd, lock | LOCK_NB) < 0) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to flock file") + /* Set exclusive or shared lock based on rw status */ + lock_flags = rw ? LOCK_EX : LOCK_SH; + + /* Place a non-blocking lock on the file */ + if (HDflock(file->fd, lock_flags | LOCK_NB) < 0) { + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } + else + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file") + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -1361,8 +1397,16 @@ H5FD_direct_unlock(H5FD_t *_file) HDassert(file); - if (HDflock(file->fd, LOCK_UN) < 0) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to flock (unlock) file") + if (HDflock(file->fd, LOCK_UN) < 0) { + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } + else + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDfamily.c b/src/H5FDfamily.c index 89ff037..d09d25e 100644 --- a/src/H5FDfamily.c +++ b/src/H5FDfamily.c @@ -12,23 +12,23 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke - * Monday, November 10, 1997 - * - * Purpose: Implements a family of files that acts as a single hdf5 - * file. The purpose is to be able to split a huge file on a - * 64-bit platform, transfer all the <2GB members to a 32-bit - * platform, and then access the entire huge file on the 32-bit - * platform. - * - * All family members are logically the same size although their - * physical sizes may vary. The logical member size is - * determined by looking at the physical size of the first member - * when the file is opened. When creating a file family, the - * first member is created with a predefined physical size - * (actually, this happens when the file family is flushed, and - * can be quite time consuming on file systems that don't - * implement holes, like nfs). + * Programmer: Robb Matzke + * Monday, November 10, 1997 + * + * Purpose: Implements a family of files that acts as a single hdf5 + * file. The purpose is to be able to split a huge file on a + * 64-bit platform, transfer all the <2GB members to a 32-bit + * platform, and then access the entire huge file on the 32-bit + * platform. + * + * All family members are logically the same size although their + * physical sizes may vary. The logical member size is + * determined by looking at the physical size of the first member + * when the file is opened. When creating a file family, the + * first member is created with a predefined physical size + * (actually, this happens when the file family is flushed, and + * can be quite time consuming on file systems that don't + * implement holes, like nfs). * */ @@ -1305,9 +1305,9 @@ H5FD_family_lock(H5FD_t *_file, hbool_t rw) for (v = 0; v < u; v++) { if (H5FD_unlock(file->memb[v]) < 0) /* Push error, but keep going */ - HDONE_ERROR(H5E_IO, H5E_CANTUNLOCK, FAIL, "unable to unlock member files") + HDONE_ERROR(H5E_IO, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock member files") } /* end for */ - HGOTO_ERROR(H5E_IO, H5E_CANTLOCK, FAIL, "unable to lock member files") + HGOTO_ERROR(H5E_IO, H5E_CANTLOCKFILE, FAIL, "unable to lock member files") } /* end if */ done: @@ -1338,7 +1338,7 @@ H5FD_family_unlock(H5FD_t *_file) for (u = 0; u < file->nmembs; u++) if (file->memb[u]) if (H5FD_unlock(file->memb[u]) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTUNLOCK, FAIL, "unable to unlock member files") + HGOTO_ERROR(H5E_IO, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock member files") done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDlog.c b/src/H5FDlog.c index d2ba67c..c55ba6b 100644 --- a/src/H5FDlog.c +++ b/src/H5FDlog.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Monday, April 17, 2000 * * Purpose: The POSIX unbuffered file driver using only the HDF5 public @@ -39,6 +39,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_LOG_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* Driver-specific file access properties */ typedef struct H5FD_log_fapl_t { char * logfile; /* Allocated log file name */ @@ -74,6 +77,7 @@ typedef struct H5FD_log_t { haddr_t eof; /* end of file; current file size */ haddr_t pos; /* current file I/O position */ H5FD_file_op_t op; /* last operation */ + hbool_t ignore_disabled_file_locks; char filename[H5FD_MAX_FILENAME_LEN]; /* Copy of file name from open operation */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely @@ -225,10 +229,20 @@ H5FL_DEFINE_STATIC(H5FD_log_t); static herr_t H5FD__init_package(void) { - herr_t ret_value = SUCCEED; + char * lock_env_var = NULL; /* Environment variable pointer */ + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if (H5FD_log_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize log VFD") @@ -471,8 +485,8 @@ H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) #ifdef H5_HAVE_WIN32_API struct _BY_HANDLE_FILE_INFORMATION fileinfo; #endif - H5_timer_t open_timer; /* Timer for open() call */ - H5_timer_t stat_timer; /* Timer for stat() call */ + H5_timer_t open_timer = {{0}, {0}, {0}, FALSE}; /* Timer for open() call */ + H5_timer_t stat_timer = {{0}, {0}, {0}, FALSE}; /* Timer for stat() call */ h5_stat_t sb; H5FD_t * ret_value = NULL; /* Return value */ @@ -612,6 +626,16 @@ H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) } /* end if */ } /* end if */ + /* Check the file locking flags in the fapl */ + if (ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if (H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + /* Check for non-default FAPL */ if (H5P_FILE_ACCESS_DEFAULT != fapl_id) { /* This step is for h5repart tool only. If user wants to change file driver from @@ -654,9 +678,9 @@ done: static herr_t H5FD_log_close(H5FD_t *_file) { - H5FD_log_t *file = (H5FD_log_t *)_file; - H5_timer_t close_timer; /* Timer for close() call */ - herr_t ret_value = SUCCEED; /* Return value */ + H5FD_log_t *file = (H5FD_log_t *)_file; + H5_timer_t close_timer = {{0}, {0}, {0}, FALSE}; /* Timer for close() call */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -1692,13 +1716,15 @@ H5FD_log_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if (HDflock(file->fd, lock_flags | LOCK_NB) < 0) { - if (ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)") + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file") + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -1726,13 +1752,15 @@ H5FD_log_unlock(H5FD_t *_file) HDassert(file); if (HDflock(file->fd, LOCK_UN) < 0) { - if (ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)") + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDmpi.c b/src/H5FDmpi.c index a2b79a2..32c899d 100644 --- a/src/H5FDmpi.c +++ b/src/H5FDmpi.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Friday, January 30, 2004 * * Purpose: Common routines for all MPI-based VFL drivers. @@ -41,8 +41,6 @@ * Programmer: Quincey Koziol * Friday, January 30, 2004 * - * Modifications: - * *------------------------------------------------------------------------- */ int @@ -79,8 +77,6 @@ done: * Programmer: Quincey Koziol * Friday, January 30, 2004 * - * Modifications: - * *------------------------------------------------------------------------- */ int @@ -116,8 +112,6 @@ done: * Programmer: Quincey Koziol * Friday, January 30, 2004 * - * Modifications: - * *------------------------------------------------------------------------- */ MPI_Comm @@ -191,13 +185,6 @@ done: * Programmer: Unknown * January 30, 1998 * - * Modifications: - * Robb Matzke, 1999-04-23 - * An error is reported for address overflows. The ADDR output - * argument is optional. - * - * Robb Matzke, 1999-08-06 - * Modified to work with the virtual file layer. *------------------------------------------------------------------------- */ haddr_t @@ -228,16 +215,6 @@ H5FD_mpi_MPIOff_to_haddr(MPI_Offset mpi_off) * Programmer: Unknown * January 30, 1998 * - * Modifications: - * Robb Matzke, 1999-04-23 - * An error is reported for address overflows. The ADDR output - * argument is optional. - * - * Robb Matzke, 1999-07-28 - * The ADDR argument is passed by value. - * - * Robb Matzke, 1999-08-06 - * Modified to work with the virtual file layer. *------------------------------------------------------------------------- */ herr_t @@ -283,9 +260,6 @@ H5FD_mpi_haddr_to_MPIOff(haddr_t addr, MPI_Offset *mpi_off /*out*/) * Programmer: rky * 19981207 * - * Modifications: - * Robb Matzke, 1999-08-09 - * Modified to work with the virtual file layer. *------------------------------------------------------------------------- */ herr_t @@ -337,9 +311,6 @@ done: * Programmer: rky * 19981207 * - * Modifications: - * Robb Matzke, 1999-08-09 - * Modified to work with the virtual file layer. *------------------------------------------------------------------------- */ herr_t diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index ac57d62..b8bfae8 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Thursday, July 29, 1999 * * Purpose: This is the MPI-2 I/O driver. @@ -1445,6 +1445,7 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, h if (H5FD_mpio_Debug[(int)'w']) HDfprintf(stdout, "%s: doing MPI collective IO\n", FUNC); #endif + /* Perform collective write operation */ if (MPI_SUCCESS != (mpi_code = MPI_File_write_at_all(file->f, mpi_off, buf, size_i, buf_type, &mpi_stat))) HMPI_GOTO_ERROR(FAIL, "MPI_File_write_at_all failed", mpi_code) @@ -1457,6 +1458,7 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, h if (H5FD_mpio_Debug[(int)'w']) HDfprintf(stdout, "%s: doing MPI independent IO\n", FUNC); #endif + /* Perform independent write operation */ if (MPI_SUCCESS != (mpi_code = MPI_File_write_at(file->f, mpi_off, buf, size_i, buf_type, &mpi_stat))) HMPI_GOTO_ERROR(FAIL, "MPI_File_write_at failed", mpi_code) @@ -1506,9 +1508,9 @@ H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, h file->local_eof = addr + (haddr_t)bytes_written; done: - if (derived_type) { + if (derived_type) MPI_Type_free(&buf_type); - } + #ifdef H5FDmpio_DEBUG if (H5FD_mpio_Debug[(int)'t']) HDfprintf(stdout, "%s: Leaving, proc %d: ret_value = %d\n", FUNC, file->mpi_rank, ret_value); @@ -1636,7 +1638,7 @@ H5FD__mpio_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR if (H5FD_mpi_haddr_to_MPIOff(file->eoa, &needed_eof) < 0) HGOTO_ERROR(H5E_INTERNAL, H5E_BADRANGE, FAIL, "cannot convert from haddr_t to MPI_Offset") - /* eoa != eof. Set eof to eoa */ + /* EOA != EOF. Set EOF to EOA */ if (size != needed_eof) { /* Extend the file's size */ if (MPI_SUCCESS != (mpi_code = MPI_File_set_size(file->f, needed_eof))) diff --git a/src/H5FDmulti.c b/src/H5FDmulti.c index 965ba56..7c6dc67 100644 --- a/src/H5FDmulti.c +++ b/src/H5FDmulti.c @@ -12,8 +12,8 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke - * Monday, November 10, 1997 + * Programmer: Robb Matzke + * Monday, November 10, 1997 * * Purpose: Implements a file driver which dispatches I/O requests to * other file drivers depending on the purpose of the address @@ -1829,8 +1829,8 @@ H5FD_multi_lock(H5FD_t *_file, hbool_t rw) { H5FD_multi_t * file = (H5FD_multi_t *)_file; int nerrors = 0; - H5FD_mem_t out_mt; - static const char *func = "H5FD_multi_unlock"; /* Function Name for error reporting */ + H5FD_mem_t out_mt = H5FD_MEM_DEFAULT; + static const char *func = "H5FD_multi_unlock"; /* Function Name for error reporting */ /* Clear the error stack */ H5Eclear2(H5E_DEFAULT); @@ -1866,7 +1866,8 @@ H5FD_multi_lock(H5FD_t *_file, hbool_t rw) } /* end if */ if (nerrors) - H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "error locking member files", -1) return 0; + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTLOCKFILE, "error locking member files", -1); + return 0; } /* H5FD_multi_lock() */ @@ -1902,7 +1903,7 @@ H5FD_multi_unlock(H5FD_t *_file) END_MEMBERS; if (nerrors) - H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "error unlocking member files", -1) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTUNLOCKFILE, "error unlocking member files", -1); return 0; } /* H5FD_multi_unlock() */ diff --git a/src/H5FDpkg.h b/src/H5FDpkg.h index 957f559..5031c8e 100644 --- a/src/H5FDpkg.h +++ b/src/H5FDpkg.h @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Thursday, January 3, 2008 * * Purpose: This file contains declarations which are visible only within @@ -23,8 +23,8 @@ #error "Do not include this file outside the H5FD package!" #endif -#ifndef _H5FDpkg_H -#define _H5FDpkg_H +#ifndef H5FDpkg_H +#define H5FDpkg_H /* Get package's private header */ #include "H5FDprivate.h" /* File drivers */ @@ -57,4 +57,4 @@ H5_DLL herr_t H5FD__vfd_swmr_reader_md_test(H5FD_t *file, unsigned num_entries, H5FD_vfd_swmr_idx_entry_t index[]); #endif /* H5FD_TESTING */ -#endif /* _H5FDpkg_H */ +#endif /* H5FDpkg_H */ diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h index b100117..8862362 100644 --- a/src/H5FDpublic.h +++ b/src/H5FDpublic.h @@ -12,11 +12,11 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Monday, July 26, 1999 */ -#ifndef _H5FDpublic_H -#define _H5FDpublic_H +#ifndef H5FDpublic_H +#define H5FDpublic_H #include "H5public.h" #include "H5Fpublic.h" /*for H5F_close_degree_t */ diff --git a/src/H5FDros3.c b/src/H5FDros3.c index d5237ba..52f5134 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -13,7 +13,7 @@ /* * Read-Only S3 Virtual File Driver (VFD) * - * Programmer: Jacob Smith + * Programmer: Jacob Smith * 2017-10-13 * * Purpose: diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c index 79b7428..cc774d9 100644 --- a/src/H5FDsec2.c +++ b/src/H5FDsec2.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Thursday, July 29, 1999 * * Purpose: The POSIX unbuffered file driver using only the HDF5 public @@ -38,6 +38,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_SEC2_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* The description of a file belonging to this driver. The 'eoa' and 'eof' * determine the amount of hdf5 address space in use and the high-water mark * of the file (the current size of the underlying filesystem file). The @@ -56,6 +59,7 @@ typedef struct H5FD_sec2_t { haddr_t eof; /* end of file; current file size */ haddr_t pos; /* current file I/O position */ H5FD_file_op_t op; /* last operation */ + hbool_t ignore_disabled_file_locks; char filename[H5FD_MAX_FILENAME_LEN]; /* Copy of file name from open operation */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely @@ -187,10 +191,20 @@ H5FL_DEFINE_STATIC(H5FD_sec2_t); static herr_t H5FD__init_package(void) { - herr_t ret_value = SUCCEED; + char * lock_env_var = NULL; /* Environment variable pointer */ + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if (H5FD_sec2_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize sec2 VFD") @@ -308,8 +322,9 @@ H5FD_sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) #ifdef H5_HAVE_WIN32_API struct _BY_HANDLE_FILE_INFORMATION fileinfo; #endif - h5_stat_t sb; - H5FD_t * ret_value = NULL; /* Return value */ + h5_stat_t sb; + H5P_genplist_t *plist; /* Property list pointer */ + H5FD_t * ret_value = NULL; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -369,17 +384,26 @@ H5FD_sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) file->inode = sb.st_ino; #endif /* H5_HAVE_WIN32_API */ + /* Get the FAPL */ + if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, NULL, "not a file access property list") + + /* Check the file locking flags in the fapl */ + if (ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if (H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + /* Retain a copy of the name used to open the file, for possible error reporting */ HDstrncpy(file->filename, name, sizeof(file->filename)); file->filename[sizeof(file->filename) - 1] = '\0'; /* Check for non-default FAPL */ if (H5P_FILE_ACCESS_DEFAULT != fapl_id) { - H5P_genplist_t *plist; /* Property list pointer */ - - /* Get the FAPL */ - if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) - HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, NULL, "not a file access property list") /* This step is for h5repart tool only. If user wants to change file driver from * family to one that uses single files (sec2, etc.) while using h5repart, this @@ -974,13 +998,15 @@ H5FD_sec2_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if (HDflock(file->fd, lock_flags | LOCK_NB) < 0) { - if (ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)") + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file") + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -1008,13 +1034,15 @@ H5FD_sec2_unlock(H5FD_t *_file) HDassert(file); if (HDflock(file->fd, LOCK_UN) < 0) { - if (ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)") + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDstdio.c b/src/H5FDstdio.c index 6ea296c..aaa84ee 100644 --- a/src/H5FDstdio.c +++ b/src/H5FDstdio.c @@ -11,7 +11,7 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Robb Matzke +/* Programmer: Robb Matzke * Wednesday, October 22, 1997 * * Purpose: The C STDIO virtual file driver which only uses calls from stdio.h. @@ -52,6 +52,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_STDIO_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = -1; + /* The maximum number of bytes which can be written in a single I/O operation */ static size_t H5_STDIO_MAX_IO_BYTES_g = (size_t)-1; @@ -82,7 +85,8 @@ typedef struct H5FD_stdio_t { haddr_t eof; /* end of file; current file size */ haddr_t pos; /* current file I/O position */ unsigned write_access; /* Flag to indicate the file was opened with write access */ - H5FD_stdio_file_op op; /* last operation */ + hbool_t ignore_disabled_file_locks; + H5FD_stdio_file_op op; /* last operation */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely * identify a file. Note that Cygwin, MinGW and other Windows POSIX @@ -231,11 +235,23 @@ static const H5FD_class_t H5FD_stdio_g = { hid_t H5FD_stdio_init(void) { + char *lock_env_var = NULL; /* Environment variable pointer */ + /* Clear the error stack */ H5Eclear2(H5E_DEFAULT); + /* Check the use disabled file locks environment variable */ + lock_env_var = getenv("HDF5_USE_FILE_LOCKING"); + if (lock_env_var && !strcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = 1; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!strcmp(lock_env_var, "TRUE") || !strcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = 0; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = -1; /* Environment variable not set, or not set correctly */ + if (H5I_VFL != H5Iget_type(H5FD_STDIO_g)) H5FD_STDIO_g = H5FDregister(&H5FD_stdio_g); + return H5FD_STDIO_g; } /* end H5FD_stdio_init() */ @@ -285,9 +301,9 @@ H5Pset_fapl_stdio(hid_t fapl_id) H5Eclear2(H5E_DEFAULT); if (0 == H5Pisa_class(fapl_id, H5P_FILE_ACCESS)) - H5Epush_ret(func, H5E_ERR_CLS, H5E_PLIST, H5E_BADTYPE, "not a file access property list", -1) + H5Epush_ret(func, H5E_ERR_CLS, H5E_PLIST, H5E_BADTYPE, "not a file access property list", -1); - return H5Pset_driver(fapl_id, H5FD_STDIO, NULL); + return H5Pset_driver(fapl_id, H5FD_STDIO, NULL); } /* end H5Pset_fapl_stdio() */ /*------------------------------------------------------------------------- @@ -397,6 +413,22 @@ H5FD_stdio_open(const char *name, unsigned flags, hid_t /*UNUSED*/ fapl_id, hadd file->eof = (haddr_t)x; } + /* Check the file locking flags in the fapl */ + if (ignore_disabled_file_locks_s != -1) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + hbool_t unused; + + /* Use the value in the property list */ + if (H5Pget_file_locking(fapl_id, &unused, &file->ignore_disabled_file_locks) < 0) { + free(file); + fclose(f); + H5Epush_ret(func, H5E_ERR_CLS, H5E_FILE, H5E_CANTGET, + "unable to get use disabled file locks property", NULL); + } + } + /* Get the file descriptor (needed for truncate and some Windows information) */ #ifdef H5_HAVE_WIN32_API file->fd = _fileno(file->fp); @@ -556,7 +588,7 @@ static herr_t H5FD_stdio_query(const H5FD_t *_f, unsigned long /*OUT*/ *flags) { /* Quiet the compiler */ - _f = _f; + (void)_f; /* Set the VFL feature flags that this driver supports. * @@ -601,8 +633,8 @@ H5FD_stdio_alloc(H5FD_t *_file, H5FD_mem_t /*UNUSED*/ type, hid_t /*UNUSED*/ dxp haddr_t addr; /* Quiet compiler */ - type = type; - dxpl_id = dxpl_id; + (void)type; + (void)dxpl_id; /* Clear the error stack */ H5Eclear2(H5E_DEFAULT); @@ -640,7 +672,7 @@ H5FD_stdio_get_eoa(const H5FD_t *_file, H5FD_mem_t /*UNUSED*/ type) H5Eclear2(H5E_DEFAULT); /* Quiet compiler */ - type = type; + (void)type; return file->eoa; } /* end H5FD_stdio_get_eoa() */ @@ -670,7 +702,7 @@ H5FD_stdio_set_eoa(H5FD_t *_file, H5FD_mem_t /*UNUSED*/ type, haddr_t addr) H5Eclear2(H5E_DEFAULT); /* Quiet the compiler */ - type = type; + (void)type; file->eoa = addr; @@ -701,13 +733,13 @@ H5FD_stdio_get_eof(const H5FD_t *_file, H5FD_mem_t /*UNUSED*/ type) const H5FD_stdio_t *file = (const H5FD_stdio_t *)_file; /* Quiet the compiler */ - type = type; + (void)type; /* Clear the error stack */ H5Eclear2(H5E_DEFAULT); /* Quiet the compiler */ - type = type; + (void)type; return (file->eof); } /* end H5FD_stdio_get_eof() */ @@ -731,7 +763,7 @@ H5FD_stdio_get_handle(H5FD_t *_file, hid_t /*UNUSED*/ fapl, void **file_handle) static const char *func = "H5FD_stdio_get_handle"; /* Function Name for error reporting */ /* Quiet the compiler */ - fapl = fapl; + (void)fapl; /* Clear the error stack */ H5Eclear2(H5E_DEFAULT); @@ -769,8 +801,8 @@ H5FD_stdio_read(H5FD_t *_file, H5FD_mem_t /*UNUSED*/ type, hid_t /*UNUSED*/ dxpl static const char *func = "H5FD_stdio_read"; /* Function Name for error reporting */ /* Quiet the compiler */ - type = type; - dxpl_id = dxpl_id; + (void)type; + (void)dxpl_id; /* Clear the error stack */ H5Eclear2(H5E_DEFAULT); @@ -871,8 +903,8 @@ H5FD_stdio_write(H5FD_t *_file, H5FD_mem_t /*UNUSED*/ type, hid_t /*UNUSED*/ dxp static const char *func = "H5FD_stdio_write"; /* Function Name for error reporting */ /* Quiet the compiler */ - dxpl_id = dxpl_id; - type = type; + (void)dxpl_id; + (void)type; /* Clear the error stack */ H5Eclear2(H5E_DEFAULT); @@ -959,7 +991,7 @@ H5FD_stdio_flush(H5FD_t *_file, hid_t /*UNUSED*/ dxpl_id, hbool_t closing) static const char *func = "H5FD_stdio_flush"; /* Function Name for error reporting */ /* Quiet the compiler */ - dxpl_id = dxpl_id; + (void)dxpl_id; /* Clear the error stack */ H5Eclear2(H5E_DEFAULT); @@ -1003,8 +1035,8 @@ H5FD_stdio_truncate(H5FD_t *_file, hid_t /*UNUSED*/ dxpl_id, hbool_t /*UNUSED*/ static const char *func = "H5FD_stdio_truncate"; /* Function Name for error reporting */ /* Quiet the compiler */ - dxpl_id = dxpl_id; - closing = closing; + (void)dxpl_id; + (void)closing; /* Clear the error stack */ H5Eclear2(H5E_DEFAULT); @@ -1104,20 +1136,22 @@ H5FD_stdio_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if (flock(file->fd, lock_flags | LOCK_NB) < 0) { - if (ENOSYS == errno) - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment " - "variable to override)", - -1) else H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file lock failed", -1) + if (file->ignore_disabled_file_locks && ENOSYS == errno) + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + else + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTLOCKFILE, "file lock failed", -1); } /* end if */ /* Flush the stream */ if (fflush(file->fp) < 0) - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_WRITEERROR, "fflush failed", -1) + H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_WRITEERROR, "fflush failed", -1); #endif /* H5_HAVE_FLOCK */ - return 0; + return 0; } /* end H5FD_stdio_lock() */ /*------------------------------------------------------------------------- @@ -1149,18 +1183,18 @@ H5FD_stdio_unlock(H5FD_t *_file) /* Flush the stream */ if (fflush(file->fp) < 0) - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_WRITEERROR, "fflush failed", -1) + H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_WRITEERROR, "fflush failed", -1); - /* Place a non-blocking lock on the file */ - if (flock(file->fd, LOCK_UN) < 0) - { - if (ENOSYS == errno) - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)", - -1) else H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file unlock failed", - -1) - } /* end if */ + /* Place a non-blocking lock on the file */ + if (flock(file->fd, LOCK_UN) < 0) { + if (file->ignore_disabled_file_locks && ENOSYS == errno) + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + else + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTUNLOCKFILE, "file unlock failed", -1); + } /* end if */ #endif /* H5_HAVE_FLOCK */ diff --git a/src/H5Fint.c b/src/H5Fint.c index a78f92b..57176ce 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -73,6 +73,7 @@ typedef struct H5F_olist_t { /* Local Prototypes */ /********************/ +static herr_t H5F__close_cb(H5VL_object_t *file_vol_obj, void **request); static herr_t H5F__set_vol_conn(H5F_t *file); static herr_t H5F__get_objects(const H5F_t *f, unsigned types, size_t max_index, hid_t *obj_id_list, hbool_t app_ref, size_t *obj_id_count_ptr); @@ -89,6 +90,15 @@ static herr_t H5F__flush_phase2(H5F_t *f, hbool_t closing); /* Package Variables */ /*********************/ +/* Package initialization variable */ +hbool_t H5_PKG_INIT_VAR = FALSE; + +/* Based on the value of the HDF5_USE_FILE_LOCKING environment variable. + * TRUE/FALSE have obvious meanings. FAIL means the environment variable was + * not set, so the code should ignore it and use the fapl value instead. + */ +htri_t use_locks_env_g = FAIL; + /*****************************/ /* Library Private Variables */ /*****************************/ @@ -103,6 +113,181 @@ H5FL_DEFINE(H5F_t); /* Declare a free list to manage the H5F_shared_t struct */ H5FL_DEFINE(H5F_shared_t); +/* File ID class */ +static const H5I_class_t H5I_FILE_CLS[1] = {{ + H5I_FILE, /* ID class value */ + 0, /* Class flags */ + 0, /* # of reserved IDs for class */ + (H5I_free_t)H5F__close_cb /* Callback routine for closing objects of this class */ +}}; + +/*------------------------------------------------------------------------- + * Function: H5F_init + * + * Purpose: Initialize the interface from some other layer. + * + * Return: Success: non-negative + * + * Failure: negative + *------------------------------------------------------------------------- + */ +herr_t +H5F_init(void) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + /* FUNC_ENTER() does all the work */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F_init() */ + +/*-------------------------------------------------------------------------- +NAME + H5F__init_package -- Initialize interface-specific information +USAGE + herr_t H5F__init_package() +RETURNS + Non-negative on success/Negative on failure +DESCRIPTION + Initializes any interface-specific data or routines. + +--------------------------------------------------------------------------*/ +herr_t +H5F__init_package(void) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Initialize the ID group for the file IDs */ + if (H5I_register_type(H5I_FILE_CLS) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to initialize interface") + + /* Check the file locking environment variable */ + if (H5F__parse_file_lock_env_var(&use_locks_env_g) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to parse file locking environment variable") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5F__init_package() */ + +/*------------------------------------------------------------------------- + * Function: H5F_term_package + * + * Purpose: Terminate this interface: free all memory and reset global + * variables to their initial values. Release all ID groups + * associated with this interface. + * + * Return: Success: Positive if anything was done that might + * have affected other interfaces; + * zero otherwise. + * + * Failure: Never fails + * + *------------------------------------------------------------------------- + */ +int +H5F_term_package(void) +{ + int n = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + if (H5_PKG_INIT_VAR) { + if (H5I_nmembers(H5I_FILE) > 0) { + (void)H5I_clear_type(H5I_FILE, FALSE, FALSE); + n++; /*H5I*/ + } /* end if */ + else { + /* Make certain we've cleaned up all the shared file objects */ + H5F_sfile_assert_num(0); + + /* Destroy the file object id group */ + n += (H5I_dec_type_ref(H5I_FILE) > 0); + + /* Mark closed */ + if (0 == n) + H5_PKG_INIT_VAR = FALSE; + } /* end else */ + } /* end if */ + + FUNC_LEAVE_NOAPI(n) +} /* end H5F_term_package() */ + +/*------------------------------------------------------------------------- + * Function: H5F__close_cb + * + * Purpose: Closes a file or causes the close operation to be pended. + * This function is called from the API and gets called + * by H5Fclose->H5I_dec_ref->H5F__close_cb when H5I_dec_ref() + * decrements the file ID reference count to zero. The file ID + * is removed from the H5I_FILE group by H5I_dec_ref() just + * before H5F__close_cb() is called. If there are open object + * headers then the close is pended by moving the file to the + * H5I_FILE_CLOSING ID group (the f->closing contains the ID + * assigned to file). + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F__close_cb(H5VL_object_t *file_vol_obj, void **request) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(file_vol_obj); + + /* Close the file */ + if (H5VL_file_close(file_vol_obj, H5P_DATASET_XFER_DEFAULT, request) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close file") + + /* Free the VOL object; it is unnecessary to unwrap the VOL + * object before freeing it, as the object was not wrapped */ + if (H5VL_free_object(file_vol_obj) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDEC, FAIL, "unable to free VOL object") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__close_cb() */ + +/*------------------------------------------------------------------------- + * Function: H5F__parse_file_lock_env_var + * + * Purpose: Parses the HDF5_USE_FILE_LOCKING environment variable. + * + * NOTE: This is done in a separate function so we can call it from + * the test code. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5F__parse_file_lock_env_var(htri_t *use_locks) +{ + char *lock_env_var = NULL; /* Environment variable pointer */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Check the file locking environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if (lock_env_var && (!HDstrcmp(lock_env_var, "FALSE") || !HDstrcmp(lock_env_var, "0"))) + *use_locks = FALSE; /* Override: Never use locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "BEST_EFFORT") || + !HDstrcmp(lock_env_var, "1"))) + *use_locks = TRUE; /* Override: Always use locks */ + else + *use_locks = FAIL; /* Environment variable not set, or not set correctly */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5F__parse_file_lock_env_var() */ + /*------------------------------------------------------------------------- * Function: H5F__set_vol_conn * @@ -857,9 +1042,10 @@ done: htri_t H5F__is_hdf5(const char *name, hid_t fapl_id) { - H5FD_t *file = NULL; /* Low-level file struct */ - haddr_t sig_addr = HADDR_UNDEF; /* Addess of hdf5 file signature */ - htri_t ret_value = FAIL; /* Return value */ + H5FD_t * file = NULL; /* Low-level file struct */ + H5F_shared_t *shared = NULL; /* Shared part of file */ + haddr_t sig_addr = HADDR_UNDEF; /* Addess of hdf5 file signature */ + htri_t ret_value = FAIL; /* Return value */ FUNC_ENTER_PACKAGE @@ -870,10 +1056,20 @@ H5F__is_hdf5(const char *name, hid_t fapl_id) if (NULL == (file = H5FD_open(name, H5F_ACC_RDONLY, fapl_id, HADDR_UNDEF))) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to open file") - /* The file is an hdf5 file if the hdf5 file signature can be found */ - if (H5FD_locate_signature(file, &sig_addr) < 0) - HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "error while trying to locate file signature") - ret_value = (HADDR_UNDEF != sig_addr); + /* If the file is already open, it's an HDF5 file + * + * If the file is open with an exclusive lock on an operating system that enforces + * mandatory file locks (like Windows), creating a new file handle and attempting + * to read through it will fail so we have to try this first. + */ + if ((shared = H5F__sfile_search(file)) != NULL) + ret_value = TRUE; + else { + /* The file is an HDF5 file if the HDF5 file signature can be found */ + if (H5FD_locate_signature(file, &sig_addr) < 0) + HGOTO_ERROR(H5E_FILE, H5E_NOTHDF5, FAIL, "error while trying to locate file signature") + ret_value = (HADDR_UNDEF != sig_addr); + } done: /* Close the file */ @@ -1490,6 +1686,51 @@ H5F__dest(H5F_t *f, hbool_t flush) } /* end H5F__dest() */ /*------------------------------------------------------------------------- + * Function: H5F__check_if_using_file_locks + * + * Purpose: Determines if this file will use file locks. + * + * There are three ways that file locking can be controlled: + * + * 1) The configure/cmake option that sets the H5_USE_FILE_LOCKING + * symbol (which is used as the default fapl value). + * + * 2) The H5Pset_file_locking() API call, which will override + * the configuration default. + * + * 3) The HDF5_USE_FILE_LOCKING environment variable, which overrides + * everything above. + * + * The main reason to disable file locking is to prevent errors on file + * systems where locking is not supported or has been disabled (as is + * often the case in parallel file systems). + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5F__check_if_using_file_locks(H5P_genplist_t *fapl, hbool_t *use_file_locking) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Make sure the out parameter has a value */ + *use_file_locking = TRUE; + + /* Check the fapl property */ + if (H5P_get(fapl, H5F_ACS_USE_FILE_LOCKING_NAME, use_file_locking) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get use file locking flag") + + /* Check the environment variable */ + if (use_locks_env_g != FAIL) + *use_file_locking = (use_locks_env_g == TRUE) ? TRUE : FALSE; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__check_if_using_file_locks() */ + +/*------------------------------------------------------------------------- * Function: H5F_open * * Purpose: Opens (or creates) a file. This function understands the @@ -1578,8 +1819,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) hbool_t set_flag = FALSE; /*set the status_flags in the superblock */ hbool_t clear = FALSE; /*clear the status_flags */ hbool_t evict_on_close; /* evict on close value from plist */ - char * lock_env_var = NULL; /*env var pointer */ - hbool_t use_file_locking; /*read from env var */ + hbool_t use_file_locking = TRUE; /* Using file locks? */ hbool_t ci_load = FALSE; /* whether MDC ci load requested */ hbool_t ci_write = FALSE; /* whether MDC CI write requested */ hbool_t file_create = FALSE; /* creating a new file or not */ @@ -1620,15 +1860,13 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) if (NULL == (drvr = H5FD_get_class(fapl_id))) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to retrieve VFL class") - /* Check the environment variable that determines if we care - * about file locking. File locking should be used unless explicitly - * disabled. - */ - lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); - if (lock_env_var && !HDstrcmp(lock_env_var, "FALSE")) - use_file_locking = FALSE; - else - use_file_locking = TRUE; + /* Get the file access property list, for future queries */ + if (NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list") + + /* Check if we are using file locking */ + if (H5F__check_if_using_file_locks(a_plist, &use_file_locking) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to get file locking flag") /* * Opening a file is a two step process. First we try to open the @@ -1716,8 +1954,8 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) if (H5FD_lock(lf, (hbool_t)((flags & H5F_ACC_RDWR) ? TRUE : FALSE)) < 0) { /* Locking failed - Closing will remove the lock */ if (H5FD_close(lf) < 0) - HDONE_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to close low-level file info") - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to lock the file") + HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, NULL, "unable to close low-level file info") + HGOTO_ERROR(H5E_FILE, H5E_CANTLOCKFILE, NULL, "unable to lock the file") } /* end if */ /* Create the 'top' file structure */ @@ -1749,6 +1987,15 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) shared = file->shared; lf = shared->lf; + /* Set the file locking flag. If the file is already open, the file + * requested file locking flag must match that of the open file. + */ + if (shared->nrefs == 1) + file->shared->use_file_locking = use_file_locking; + else if (shared->nrefs > 1) + if (file->shared->use_file_locking != use_file_locking) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "file locking flag values don't match") + /* Check if page buffering is enabled */ if (H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &page_buf_size) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get page buffer size") @@ -1813,7 +2060,6 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) /* Open the root group */ if (H5G_mkroot(file, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to read root group") - } /* end if */ /* Checked if configured for VFD SWMR */ @@ -1910,7 +2156,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) /* Remove the file lock for SWMR_WRITE */ if (use_file_locking && ((H5F_INTENT(file) & H5F_ACC_SWMR_WRITE) || H5F_USE_VFD_SWMR(file))) { if (H5FD_unlock(file->shared->lf) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to unlock the file") + HGOTO_ERROR(H5E_FILE, H5E_CANTUNLOCKFILE, NULL, "unable to unlock the file") } /* end if */ } /* end if */ else { /* H5F_ACC_RDONLY: check consistency of status_flags */ @@ -1979,7 +2225,7 @@ H5F__post_open(H5F_t *f) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5F__flush() */ +} /* end H5F__post_open() */ /*------------------------------------------------------------------------- * Function: H5F_flush_phase1 @@ -3573,6 +3819,19 @@ H5F__start_swmr_write(H5F_t *f) setup = TRUE; + /* Place an advisory lock on the file */ + if (H5F_USE_FILE_LOCKING(f)) { + /* Have to unlock on Windows as Win32 doesn't support changing the lock + * type (exclusive vs shared) with a second call. + */ + if (H5FD_unlock(f->shared->lf) < 0) { + HGOTO_ERROR(H5E_FILE, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock the file") + } + if (H5FD_lock(f->shared->lf, TRUE) < 0) { + HGOTO_ERROR(H5E_FILE, H5E_CANTLOCKFILE, FAIL, "unable to lock the file") + } + } + /* Mark superblock as dirty */ if (H5F_super_dirty(f) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty") @@ -3599,10 +3858,6 @@ H5F__start_swmr_write(H5F_t *f) if (H5O_refresh_metadata_reopen(obj_ids[u], &obj_glocs[u], NULL, vol_connector, TRUE) < 0) HGOTO_ERROR(H5E_ATOM, H5E_CLOSEERROR, FAIL, "can't refresh-close object") - /* Unlock the file */ - if (H5FD_unlock(f->shared->lf) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to unlock the file") - done: if (ret_value < 0 && setup) { @@ -3631,6 +3886,11 @@ done: HDONE_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush superblock") } /* end if */ + /* Unlock the file */ + if (H5F_USE_FILE_LOCKING(f)) + if (H5FD_unlock(f->shared->lf) < 0) + HDONE_ERROR(H5E_FILE, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock the file") + /* Free memory */ if (obj_ids) H5MM_xfree(obj_ids); diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 3270a03..14a4761 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Thursday, September 28, 2000 * * Purpose: This file contains declarations which are visible only within @@ -23,8 +23,8 @@ #error "Do not include this file outside the H5F package!" #endif -#ifndef _H5Fpkg_H -#define _H5Fpkg_H +#ifndef H5Fpkg_H +#define H5Fpkg_H /* Get package's private header */ #include "H5Fprivate.h" @@ -340,6 +340,7 @@ struct H5F_shared_t { struct H5G_t * root_grp; /* Open root group */ H5FO_t * open_objs; /* Open objects in file */ H5UC_t * grp_btree_shared; /* Ref-counted group B-tree node info */ + hbool_t use_file_locking; /* Whether or not to use file locking */ hbool_t closing; /* File is in the process of being closed */ /* Cached VOL connector ID & info */ @@ -507,6 +508,11 @@ H5FL_EXTERN(H5F_t); /* Declare a free list to manage the H5F_shared_t struct */ H5FL_EXTERN(H5F_shared_t); +/* Whether or not to use file locking (based on the environment variable) + * FAIL means ignore the environment variable. + */ +H5_DLLVAR htri_t use_locks_env_g; + /******************************/ /* Package Private Prototypes */ /******************************/ @@ -524,6 +530,7 @@ H5_DLL herr_t H5F__start_swmr_write(H5F_t *f); H5_DLL herr_t H5F__close(H5F_t *f); H5_DLL herr_t H5F__set_libver_bounds(H5F_t *f, H5F_libver_t low, H5F_libver_t high); H5_DLL herr_t H5F__get_cont_info(const H5F_t *f, H5VL_file_cont_info_t *info); +H5_DLL herr_t H5F__parse_file_lock_env_var(htri_t *use_locks); H5_DLL herr_t H5F__vfd_swmr_end_tick(H5F_t *f); H5_DLL herr_t H5F__vfd_swmr_disable_end_of_tick(H5F_t *f); H5_DLL herr_t H5F__vfd_swmr_enable_end_of_tick(H5F_t *f); @@ -594,6 +601,7 @@ H5_DLL herr_t H5F__vfd_swmr_writer_create_open_flush_test(hid_t file_id, hbool_t H5_DLL herr_t H5F__vfd_swmr_writer_md_test(hid_t, unsigned, struct H5FD_vfd_swmr_idx_entry_t *, unsigned); H5_DLL htri_t H5F__same_file_test(hid_t file_id1, hid_t file_id2); +H5_DLL herr_t H5F__reparse_file_lock_variable_test(void); #endif /* H5F_TESTING */ -#endif /* _H5Fpkg_H */ +#endif /* H5Fpkg_H */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 5d7cfca..19a91ec 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -15,8 +15,8 @@ * This file contains macros & information for file access */ -#ifndef _H5Fprivate_H -#define _H5Fprivate_H +#ifndef H5Fprivate_H +#define H5Fprivate_H /* Early typedefs to avoid circular dependencies */ typedef struct H5F_t H5F_t; @@ -386,6 +386,7 @@ uint64_decode(uint8_t **pp) #define H5F_SET_MIN_DSET_OHDR(F, V) ((F)->shared->crt_dset_min_ohdr_flag = (V)) #define H5F_VOL_CLS(F) ((F)->shared->vol_cls) #define H5F_VOL_OBJ(F) ((F)->vol_obj) +#define H5F_USE_FILE_LOCKING(F) ((F)->shared->use_file_locking) #else /* H5F_MODULE */ #define H5F_LOW_BOUND(F) (H5F_get_low_bound(F)) #define H5F_HIGH_BOUND(F) (H5F_get_high_bound(F)) @@ -451,6 +452,7 @@ uint64_decode(uint8_t **pp) #define H5F_SET_MIN_DSET_OHDR(F, V) (H5F_set_min_dset_ohdr((F), (V))) #define H5F_VOL_CLS(F) (H5F_get_vol_cls(F)) #define H5F_VOL_OBJ(F) (H5F_get_vol_obj(F)) +#define H5F_USE_FILE_LOCKING(F) (H5F_get_use_file_locking(F)) #endif /* H5F_MODULE */ /* Macros to encode/decode offset/length's for storing in the file */ @@ -615,6 +617,11 @@ uint64_decode(uint8_t **pp) "page_buffer_min_meta_perc" /* the min metadata percentage for the page buffer cache */ #define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME \ "page_buffer_min_raw_perc" /* the min raw data percentage for the page buffer cache */ +#define H5F_ACS_USE_FILE_LOCKING_NAME \ + "use_file_locking" /* whether or not we use file locks for SWMR control and to prevent multiple writers \ + */ +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME \ + "ignore_disabled_file_locks" /* whether or not we ignore "locks disabled" errors */ #ifdef H5_HAVE_PARALLEL #define H5F_ACS_MPI_PARAMS_COMM_NAME "mpi_params_comm" /* the MPI communicator */ #define H5F_ACS_MPI_PARAMS_INFO_NAME "mpi_params_info" /* the MPI info struct */ @@ -875,6 +882,7 @@ H5_DLL hbool_t H5F_get_min_dset_ohdr(const H5F_t *f); H5_DLL herr_t H5F_set_min_dset_ohdr(H5F_t *f, hbool_t minimize); H5_DLL const H5VL_class_t *H5F_get_vol_cls(const H5F_t *f); H5_DLL H5VL_object_t *H5F_get_vol_obj(const H5F_t *f); +H5_DLL hbool_t H5F_get_file_locking(const H5F_t *f); /* Functions than retrieve values set/cached from the superblock/FCPL */ H5_DLL haddr_t H5F_get_base_addr(const H5F_t *f); @@ -999,4 +1007,4 @@ H5_DLL herr_t H5F_debug(H5F_t *f, FILE *stream, int indent, int fwidth); H5_DLL hbool_t H5F_use_vfd_swmr(const H5F_t *f); -#endif /* _H5Fprivate_H */ +#endif /* H5Fprivate_H */ diff --git a/src/H5Fquery.c b/src/H5Fquery.c index 146f41b..c2fce7e 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -15,7 +15,7 @@ * * Created: H5Fquery.c * Jan 10 2008 - * Quincey Koziol + * Quincey Koziol * * Purpose: File structure query routines. * @@ -200,8 +200,8 @@ H5F_get_actual_name(const H5F_t *f) * Function: H5F_get_extpath * * Purpose: Retrieve the file's 'extpath' flags - * This is used by H5L_extern_traverse() and H5D_build_file_prefix() to retrieve the main file's - *location when searching the target file. + * This is used by H5L_extern_traverse() and H5D_build_file_prefix() + * to retrieve the main file's location when searching the target file. * * Return: 'extpath' on success/abort on failure (shouldn't fail) *------------------------------------------------------------------------- @@ -1229,8 +1229,8 @@ H5F_get_null_fsm_addr(const H5F_t *f) * * Return: VOL class pointer for file, can't fail * - * Programmer: Quincey Koziol - * Saturday, August 17, 2019 + * Programmer: Quincey Koziol + * Saturday, August 17, 2019 * *------------------------------------------------------------------------- */ @@ -1272,8 +1272,8 @@ H5F_get_vol_obj(const H5F_t *f) * Return: Success: Non-negative * Failure: Negative * - * Programmer: Quincey Koziol - * Saturday, August 17, 2019 + * Programmer: Quincey Koziol + * Saturday, August 17, 2019 * *------------------------------------------------------------------------- */ @@ -1302,6 +1302,26 @@ done: } /* end H5F_get_cont_info */ /*------------------------------------------------------------------------- + * Function: H5F_get_file_locking + * + * Purpose: Get the file locking flag for the file + * + * Return: TRUE/FALSE + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_get_file_locking(const H5F_t *f) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->use_file_locking) +} /* end H5F_get_file_locking */ + +/*------------------------------------------------------------------------- * Function: H5F_use_vfd_swmr * * Purpose: Quick and dirty routine to determine if VFD SWMR is diff --git a/src/H5Ftest.c b/src/H5Ftest.c index 63e0b25..03b613c 100644 --- a/src/H5Ftest.c +++ b/src/H5Ftest.c @@ -15,7 +15,7 @@ * * Created: H5Ftest.c * Jan 3 2007 - * Quincey Koziol + * Quincey Koziol * * Purpose: File testing routines. * @@ -231,6 +231,71 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F__get_sbe_addr_test() */ +/*------------------------------------------------------------------------- + * Function: H5F__same_file_test + * + * Purpose: Check if two file IDs refer to the same underlying file. + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * Oct 13, 2018 + * + *------------------------------------------------------------------------- + */ +htri_t +H5F__same_file_test(hid_t file_id1, hid_t file_id2) +{ + H5F_t *file1, *file2; /* File info */ + htri_t ret_value = FAIL; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check arguments */ + if (NULL == (file1 = (H5F_t *)H5VL_object_verify(file_id1, H5I_FILE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file") + if (NULL == (file2 = (H5F_t *)H5VL_object_verify(file_id2, H5I_FILE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file") + + /* If they are using the same underlying "shared" file struct, they are the same file */ + ret_value = (file1->shared == file2->shared); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__same_file_test() */ + +/*------------------------------------------------------------------------- + * Function: H5F__reparse_file_lock_variable_test + * + * Purpose: Re-parse the file locking environment variable. + * + * Since getenv(3) is fairly expensive, we only parse it once, + * when the library opens. This test function is used to + * re-parse the environment variable after we've changed it + * with setnev(3). + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Summer 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F__reparse_file_lock_variable_test(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check the file locking environment variable */ + if (H5F__parse_file_lock_env_var(&use_locks_env_g) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to parse file locking environment variable") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__reparse_file_lock_variable_test() */ + /* * VFD SWMR tests */ @@ -595,36 +660,3 @@ done: } FUNC_LEAVE_NOAPI(ret_value) } /* H5F__vfd_swmr_writer_md_test() */ - -/*------------------------------------------------------------------------- - * Function: H5F__same_file_test - * - * Purpose: Check if two file IDs refer to the same underlying file. - * - * Return: SUCCEED/FAIL - * - * Programmer: Quincey Koziol - * Oct 13, 2018 - * - *------------------------------------------------------------------------- - */ -htri_t -H5F__same_file_test(hid_t file_id1, hid_t file_id2) -{ - H5F_t *file1, *file2; /* File info */ - htri_t ret_value = FAIL; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Check arguments */ - if (NULL == (file1 = (H5F_t *)H5VL_object_verify(file_id1, H5I_FILE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file") - if (NULL == (file2 = (H5F_t *)H5VL_object_verify(file_id2, H5I_FILE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file") - - /* If they are using the same underlying "shared" file struct, they are the same file */ - ret_value = (file1->shared == file2->shared); - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5F__same_file_test() */ diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 89aabeb..55a4624 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -289,7 +289,29 @@ #define H5F_ACS_VOL_CONN_COPY H5P__facc_vol_copy #define H5F_ACS_VOL_CONN_CMP H5P__facc_vol_cmp #define H5F_ACS_VOL_CONN_CLOSE H5P__facc_vol_close - +/* Definition for using file locking or not. The default is set + * via the configure step. + */ +#define H5F_ACS_USE_FILE_LOCKING_SIZE sizeof(hbool_t) +#if defined H5_USE_FILE_LOCKING && H5_USE_FILE_LOCKING +#define H5F_ACS_USE_FILE_LOCKING_DEF TRUE +#else +#define H5F_ACS_USE_FILE_LOCKING_DEF FALSE +#endif +#define H5F_ACS_USE_FILE_LOCKING_ENC H5P__encode_hbool_t +#define H5F_ACS_USE_FILE_LOCKING_DEC H5P__decode_hbool_t +/* Definition for whether we ignore file locking errors when we can + * tell that file locking has been disabled on the file system. + * The default is set via the configure step. + */ +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_SIZE sizeof(hbool_t) +#if defined H5_IGNORE_DISABLED_FILE_LOCKS && H5_IGNORE_DISABLED_FILE_LOCKS +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF TRUE +#else +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF FALSE +#endif +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_ENC H5P__encode_hbool_t +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEC H5P__decode_hbool_t /* Definitions for the VFD SWMR configuration */ #define H5F_ACS_VFD_SWMR_CONFIG_SIZE sizeof(H5F_vfd_swmr_config_t) #define H5F_ACS_VFD_SWMR_CONFIG_DEF H5F__DEFAULT_VFD_SWMR_CONFIG @@ -487,7 +509,10 @@ static const unsigned H5F_def_page_buf_min_meta_perc_g = H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEF; /* Default page buffer minimum metadata size */ static const unsigned H5F_def_page_buf_min_raw_perc_g = H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF; /* Default page buffer mininum raw data size */ - +static const hbool_t H5F_def_use_file_locking_g = + H5F_ACS_USE_FILE_LOCKING_DEF; /* Default use file locking flag */ +static const hbool_t H5F_def_ignore_disabled_file_locks_g = + H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF; /* Default ignore disabled file locks flag */ static const H5F_vfd_swmr_config_t H5F_def_vfd_swmr_config_g = H5F_ACS_VFD_SWMR_CONFIG_DEF; /* Default vfd swmr configuration */ @@ -782,6 +807,19 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) H5F_ACS_VOL_CONN_CLOSE) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the use file locking flag */ + if (H5P__register_real(pclass, H5F_ACS_USE_FILE_LOCKING_NAME, H5F_ACS_USE_FILE_LOCKING_SIZE, + &H5F_def_use_file_locking_g, NULL, NULL, NULL, H5F_ACS_USE_FILE_LOCKING_ENC, + H5F_ACS_USE_FILE_LOCKING_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the ignore disabled file locks flag */ + if (H5P__register_real(pclass, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, + H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_SIZE, &H5F_def_ignore_disabled_file_locks_g, + NULL, NULL, NULL, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_ENC, + H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + if (H5P_LST_FILE_ACCESS_ANY_VFD_g == H5I_INVALID_HID) { H5P_LST_FILE_ACCESS_ANY_VFD_g = H5P_create_id(pclass, false); if (H5P_LST_FILE_ACCESS_ANY_VFD_g == H5I_INVALID_HID) { @@ -814,12 +852,6 @@ done: * Programmer: Robb Matzke * Tuesday, June 9, 1998 * - * Modifications: - * - * Raymond Lu - * Tuesday, Oct 23, 2001 - * Changed file access property list mechanism to the new - * generic property list. *------------------------------------------------------------------------- */ herr_t @@ -4750,6 +4782,96 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Pget_evict_on_close() */ +/*------------------------------------------------------------------------- + * Function: H5Pset_file_locking + * + * Purpose: Sets the file locking property values. + * + * Overrides the default file locking flag setting that was + * set when the library was configured. + * + * Can be overridden by the HDF5_USE_FILE_LOCKING environment + * variable. + * + * File locking is used when creating/opening a file to prevent + * problematic file accesses. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Spring 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_file_locking(hid_t fapl_id, hbool_t use_file_locking, hbool_t ignore_when_disabled) +{ + H5P_genplist_t *plist; /* property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "ibb", fapl_id, use_file_locking, ignore_when_disabled); + + /* Make sure this is a fapl */ + if (TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not a file access plist") + + /* Get the plist structure */ + if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Set values */ + if (H5P_set(plist, H5F_ACS_USE_FILE_LOCKING_NAME, &use_file_locking) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set use file locking property") + if (H5P_set(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &ignore_when_disabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set ignore disabled file locks property") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_file_locking() */ + +/*------------------------------------------------------------------------- + * Function: H5Pget_file_locking + * + * Purpose: Gets the file locking property values. + * + * File locking is used when creating/opening a file to prevent + * problematic file accesses. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Spring 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_file_locking(hid_t fapl_id, hbool_t *use_file_locking /*out*/, hbool_t *ignore_when_disabled /*out*/) +{ + H5P_genplist_t *plist; /* property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "ixx", fapl_id, use_file_locking, ignore_when_disabled); + + /* Make sure this is a fapl */ + if (TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not an access plist") + + /* Get the plist structure */ + if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get values */ + if (H5P_get(plist, H5F_ACS_USE_FILE_LOCKING_NAME, use_file_locking) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get use file locking property") + if (H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, ignore_when_disabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get ignore disabled file locks property") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_file_locking() */ + #ifdef H5_HAVE_PARALLEL /*------------------------------------------------------------------------- diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 0031835..64476db 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -351,6 +351,8 @@ H5_DLL herr_t H5Pset_file_image(hid_t fapl_id, void *buf_ptr, size_t buf_le H5_DLL herr_t H5Pget_file_image(hid_t fapl_id, void **buf_ptr_ptr, size_t *buf_len_ptr); H5_DLL herr_t H5Pset_file_image_callbacks(hid_t fapl_id, H5FD_file_image_callbacks_t *callbacks_ptr); H5_DLL herr_t H5Pget_file_image_callbacks(hid_t fapl_id, H5FD_file_image_callbacks_t *callbacks_ptr); +H5_DLL herr_t H5Pset_file_locking(hid_t fapl_id, hbool_t use_file_locking, hbool_t ignore_when_disabled); +H5_DLL herr_t H5Pget_file_locking(hid_t fapl_id, hbool_t *use_file_locking, hbool_t *ignore_when_disabled); H5_DLL herr_t H5Pset_core_write_tracking(hid_t fapl_id, hbool_t is_enabled, size_t page_size); H5_DLL herr_t H5Pget_core_write_tracking(hid_t fapl_id, hbool_t *is_enabled, size_t *page_size); H5_DLL herr_t H5Pset_metadata_read_attempts(hid_t plist_id, unsigned attempts); diff --git a/src/H5err.txt b/src/H5err.txt index 05309ff..a902991 100644 --- a/src/H5err.txt +++ b/src/H5err.txt @@ -136,6 +136,8 @@ MINOR, FILEACC, H5E_BADFILE, Bad file ID accessed MINOR, FILEACC, H5E_TRUNCATED, File has been truncated MINOR, FILEACC, H5E_MOUNT, File mount error MINOR, FILEACC, H5E_CANTDELETEFILE, Unable to delete file +MINOR, FILEACC, H5E_CANTLOCKFILE, Unable to lock file +MINOR, FILEACC, H5E_CANTUNLOCKFILE, Unable to unlock file # Generic low-level file I/O errors MINOR, FILE, H5E_SEEKERROR, Seek failed diff --git a/src/H5private.h b/src/H5private.h index d481123..49ffbcb 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -893,8 +893,8 @@ H5_DLL H5_ATTR_CONST int Nflock(int fd, int operation); #ifndef HDflock /* NOTE: flock(2) is not present on all POSIX systems. * If it is not present, we try a flock() equivalent based on - * fcntl(2), then fall back to a function that always fails if - * it is not present at all (Windows uses a separate Wflock() + * fcntl(2), then fall back to a function that always succeeds + * if it is not present at all (Windows uses a separate Wflock() * function). */ #if defined(H5_HAVE_FLOCK) diff --git a/src/H5system.c b/src/H5system.c index eef621e..8577912 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -677,15 +677,15 @@ Pflock(int fd, int operation) * Purpose: Wrapper function for systems where no file locking is * available. * - * Return: Failure: -1 (always fails) + * Return: 0 (success) * *------------------------------------------------------------------------- */ int H5_ATTR_CONST Nflock(int H5_ATTR_UNUSED fd, int H5_ATTR_UNUSED operation) { - /* just fail */ - return -1; + /* just succeed */ + return 0; } /* end Nflock() */ /*------------------------------------------------------------------------- diff --git a/src/libhdf5.settings.in b/src/libhdf5.settings.in index df7ddf2..44c1540 100644 --- a/src/libhdf5.settings.in +++ b/src/libhdf5.settings.in @@ -89,5 +89,6 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ Using memory checker: @USINGMEMCHECKER@ Memory allocation sanity checks: @MEMORYALLOCSANITYCHECK@ Function stack tracing: @CODESTACK@ + Use file locking: @DESIRED_FILE_LOCKING@ Strict file format checks: @STRICT_FORMAT_CHECKS@ Optimization instrumentation: @INSTRUMENT_LIBRARY@ diff --git a/test/h5test.c b/test/h5test.c index 591cd2b..0d9fccd 100644 --- a/test/h5test.c +++ b/test/h5test.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Thursday, November 19, 1998 * * Purpose: Provides support functions for most of the hdf5 tests cases. @@ -2287,3 +2287,61 @@ done: HDfree(dup_buf); return ret_value; } /* end h5_duplicate_file_by_bytes() */ + +/*------------------------------------------------------------------------- + * Function: h5_check_if_file_locking_enabled + * + * Purpose: Checks if file locking is enabled on this file system. + * + * Return: SUCCEED/FAIL + * are_enabled will be FALSE if file locking is disabled on + * the file system of if there were errors. + * + *------------------------------------------------------------------------- + */ +herr_t +h5_check_if_file_locking_enabled(hbool_t *is_enabled) +{ + const char *filename = "locking_test_file"; + int pmode = O_RDWR | O_CREAT | O_TRUNC; + int fd = -1; + + *is_enabled = TRUE; + + if ((fd = HDopen(filename, pmode, H5_POSIX_CREATE_MODE_RW)) < 0) + goto error; + + /* Test HDflock() to see if it works */ + if (HDflock(fd, LOCK_EX | LOCK_NB) < 0) { + if (ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. This is most frequently used on + * Lustre. If we also want to check for disabled NFS locks + * we'll need to check for ENOLCK, too. That isn't done by + * default here since that could also represent an actual + * error condition. + */ + errno = 0; + *is_enabled = FALSE; + } + else + goto error; + } + if (HDflock(fd, LOCK_UN) < 0) + goto error; + + if (HDclose(fd) < 0) + goto error; + if (HDremove(filename) < 0) + goto error; + + return SUCCEED; + +error: + *is_enabled = FALSE; + if (fd > -1) { + HDclose(fd); + HDremove(filename); + } + return FAIL; +} /* end h5_check_if_file_locking_enabled() */ diff --git a/test/h5test.h b/test/h5test.h index 5e4c69f..81925b9 100644 --- a/test/h5test.h +++ b/test/h5test.h @@ -12,13 +12,13 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Friday, November 20, 1998 * * Purpose: Test support stuff. */ -#ifndef _H5TEST_H -#define _H5TEST_H +#ifndef H5TEST_H +#define H5TEST_H /* * Include required headers. This file tests internal library functions, @@ -262,6 +262,7 @@ H5TEST_DLL H5VL_class_t *h5_get_dummy_vol_class(void); H5TEST_DLL const char * h5_get_version_string(H5F_libver_t libver); H5TEST_DLL int h5_compare_file_bytes(char *fname1, char *fname2); H5TEST_DLL int h5_duplicate_file_by_bytes(const char *orig, const char *dest); +H5TEST_DLL herr_t h5_check_if_file_locking_enabled(hbool_t *are_enabled); /* Functions that will replace components of a FAPL */ H5TEST_DLL herr_t h5_get_vfd_fapl(hid_t fapl_id); diff --git a/test/swmr.c b/test/swmr.c index 7f531e2..24a0b7c 100644 --- a/test/swmr.c +++ b/test/swmr.c @@ -87,7 +87,7 @@ static int test_file_lock_concur(hid_t fapl); static int test_file_lock_swmr_concur(hid_t fapl); /* Test file lock environment variable */ -static int test_file_lock_env_var(hid_t fapl); +static int test_file_locking(hid_t in_fapl, hbool_t turn_locking_on, hbool_t env_var_override); /* Tests for SWMR VFD flag */ static int test_swmr_vfd_flag(void); @@ -847,14 +847,17 @@ error: static int test_metadata_read_retry_info(hid_t in_fapl) { - hid_t fapl, new_fapl; /* File access property list */ - hid_t fid, fid1; /* File IDs */ + hid_t fapl = H5I_INVALID_HID; /* File access property list */ + hid_t new_fapl = H5I_INVALID_HID; /* File access property list */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t fid1 = H5I_INVALID_HID; /* File ID */ H5F_retry_info_t info, info1; /* The collection of metadata retries */ H5F_t * f = NULL, *f1 = NULL; /* Internal file object pointers */ unsigned i, j, n; /* Local index variables */ - hid_t did1, did2; /* Dataset IDs */ - hid_t sid; /* Dataspace ID */ - hid_t dcpl; /* Dataset creation property list */ + hid_t did1 = H5I_INVALID_HID; /* Dataset ID */ + hid_t did2 = H5I_INVALID_HID; /* Dataset ID */ + hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ + hid_t dcpl = H5I_INVALID_HID; /* Dataset creation property list */ hsize_t dims[2] = {6, 10}; /* Dataset dimensions */ char filename[NAME_BUF_SIZE]; /* File name */ int buf[6][10], chkbuf1[6][10], chkbuf2[6][10]; /* Buffers for data */ @@ -1623,7 +1626,7 @@ test_start_swmr_write(hid_t in_fapl, hbool_t new_format) FAIL_STACK_ERROR /* Should be 100 */ - if (attempts != (new_format ? H5F_METADATA_READ_ATTEMPTS : H5F_SWMR_METADATA_READ_ATTEMPTS)) + if (attempts != (unsigned int)(new_format ? H5F_METADATA_READ_ATTEMPTS : H5F_SWMR_METADATA_READ_ATTEMPTS)) TEST_ERROR; /* Close the property list */ @@ -2392,17 +2395,24 @@ error: * (5) Parent: open a file with write access; enable SWMR writing mode * Child: concurrent open of the file with write and SWMR write access (fail) */ -#if !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) +#ifndef H5_HAVE_UNISTD_H static int -test_start_swmr_write_concur(hid_t H5_ATTR_UNUSED in_fapl, hbool_t H5_ATTR_UNUSED new_format) +test_start_swmr_write_concur(hid_t H5_ATTR_UNUSED in_fapl, hbool_t new_format) { + if (new_format) { + TESTING("H5Fstart_swmr_write()--concurrent access for latest format"); + } + else { + TESTING("H5Fstart_swmr_write()--concurrent access for non-latest-format"); + } + SKIPPED(); - HDputs(" Test skipped due to fork or waitpid not defined."); + HDputs(" Test skipped due to a lack of unistd.h functionality."); return 0; } /* test_start_swmr_write_concur() */ -#else /* defined(H5_HAVE_FORK && defined(H5_HAVE_WAITPID) */ +#else /* H5_HAVE_UNISTD_H */ static int test_start_swmr_write_concur(hid_t in_fapl, hbool_t new_format) @@ -2434,7 +2444,7 @@ test_start_swmr_write_concur(hid_t in_fapl, hbool_t new_format) } else { TESTING("H5Fstart_swmr_write()--concurrent access for non-latest-format"); - } /* end if */ + } if ((fapl = H5Pcopy(in_fapl)) < 0) FAIL_STACK_ERROR @@ -3004,7 +3014,7 @@ error: return -1; } /* test_start_swmr_write_concur() */ -#endif /* !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) */ +#endif /* H5_HAVE_UNISTD_H */ /* * test_start_swmr_write_stress_ohdr(): @@ -4332,8 +4342,11 @@ test_file_lock_same(hid_t in_fapl) /* Output message about test being performed */ TESTING("File open with different combinations of flags--single process access"); + /* Set locking in the fapl */ if ((fapl = H5Pcopy(in_fapl)) < 0) FAIL_STACK_ERROR + if (H5Pset_file_locking(fapl, TRUE, TRUE) < 0) + FAIL_STACK_ERROR /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[1], fapl, filename, sizeof(filename)); @@ -4489,14 +4502,15 @@ error: static int test_file_lock_swmr_same(hid_t in_fapl) { - hid_t fid, fid2; /* File IDs */ - hid_t fapl; /* File access property list */ + hid_t fid = H5I_INVALID_HID; /* File IDs */ + hid_t fid2 = H5I_INVALID_HID; + hid_t fapl = H5I_INVALID_HID; /* File access property list */ char filename[NAME_BUF_SIZE]; /* file name */ /* Output message about test being performed */ TESTING("File open with different combinations of flags + SWMR flags--single process access"); - /* Get a copy of the parameter in_fapl */ + /* Set locking in the fapl */ if ((fapl = H5Pcopy(in_fapl)) < 0) FAIL_STACK_ERROR @@ -4795,7 +4809,7 @@ error: ** This is for concurrent access. ** *****************************************************************/ -#if !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID) && defined(H5_HAVE_FLOCK)) +#ifndef H5_HAVE_UNISTD_H static int test_file_lock_concur(hid_t H5_ATTR_UNUSED in_fapl) @@ -4803,12 +4817,12 @@ test_file_lock_concur(hid_t H5_ATTR_UNUSED in_fapl) /* Output message about test being performed */ TESTING("File open with different combinations of flags--concurrent access"); SKIPPED(); - HDputs(" Test skipped due to fork, waitpid, or flock not defined."); + HDputs(" Test skipped due to a lack of unistd.h functionality."); return 0; } /* end test_file_lock_concur() */ -#else +#else /* H5_HAVE_UNISTD_H */ static int test_file_lock_concur(hid_t in_fapl) @@ -4825,8 +4839,11 @@ test_file_lock_concur(hid_t in_fapl) /* Output message about test being performed */ TESTING("File open with different combinations of flags--concurrent access"); + /* Set locking in the fapl */ if ((fapl = H5Pcopy(in_fapl)) < 0) FAIL_STACK_ERROR + if (H5Pset_file_locking(fapl, TRUE, TRUE) < 0) + FAIL_STACK_ERROR /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[1], fapl, filename, sizeof(filename)); @@ -5175,7 +5192,7 @@ error: } /* end test_file_lock_concur() */ -#endif /* !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID) && defined(H5_HAVE_FLOCK)) */ +#endif /* H5_HAVE_UNISTD_H */ /**************************************************************** ** @@ -5185,7 +5202,7 @@ error: ** This is for concurrent access. ** *****************************************************************/ -#if !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) +#ifndef H5_HAVE_UNISTD_H static int test_file_lock_swmr_concur(hid_t H5_ATTR_UNUSED in_fapl) @@ -5193,12 +5210,12 @@ test_file_lock_swmr_concur(hid_t H5_ATTR_UNUSED in_fapl) /* Output message about test being performed */ TESTING("File open with different combintations of flags + SWMR flags--concurrent access"); SKIPPED(); - HDputs(" Test skipped due to fork or waitpid not defined."); + HDputs(" Test skipped due to a lack of unistd.h functionality."); return 0; } /* end test_file_lock_swmr_concur() */ -#else +#else /* H5_HAVE_UNISTD_H */ static int test_file_lock_swmr_concur(hid_t in_fapl) @@ -5215,8 +5232,11 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Output message about test being performed */ TESTING("File open with different combintations of flags + SWMR flags--concurrent access"); + /* Set locking in the fapl */ if ((fapl = H5Pcopy(in_fapl)) < 0) FAIL_STACK_ERROR + if (H5Pset_file_locking(fapl, TRUE, TRUE) < 0) + FAIL_STACK_ERROR /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[2], fapl, filename, sizeof(filename)); @@ -5730,7 +5750,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Open the test file */ H5E_BEGIN_TRY { - child_fid = H5Fopen(filename, H5F_ACC_RDWR, H5P_DEFAULT); + child_fid = H5Fopen(filename, H5F_ACC_RDWR, fapl); } H5E_END_TRY; @@ -6065,7 +6085,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) FAIL_STACK_ERROR /* Open the test file */ - if ((fid = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) + if ((fid = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) FAIL_STACK_ERROR /* Notify child process */ @@ -6195,42 +6215,73 @@ error: } /* end test_file_lock_swmr_concur() */ -#endif /* !(defined(H5_HAVE_FORK && defined(H5_HAVE_WAITPID)) */ +#endif /* H5_HAVE_UNISTD_H */ /**************************************************************** ** -** test_file_lock_swmr_concur(): low-level file test routine. -** With the implementation of file locking, this test checks file -** open with different combinations of flags + SWMR flags. -** This is for concurrent access. +** test_file_locking(): +** Tests various combinations of file locking flags and +** and environment variables. ** *****************************************************************/ static int -test_file_lock_env_var(hid_t in_fapl) +test_file_locking(hid_t in_fapl, hbool_t turn_locking_on, hbool_t env_var_override) { -#if !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) +#ifndef H5_HAVE_UNISTD_H + if (turn_locking_on && env_var_override) + TESTING("File locking: ON w/ env var override") + else if (turn_locking_on && !env_var_override) + TESTING("File locking: ON") + else if (!turn_locking_on && env_var_override) + TESTING("File locking: OFF w/ env var override") + else + TESTING("File locking: OFF") SKIPPED(); - HDputs(" Test skipped due to fork or waitpid not defined."); + HDputs(" Test skipped due to a lack of unistd.h functionality."); return 0; -#else - hid_t fid = -1; /* File ID */ - hid_t fapl = -1; /* File access property list */ - char filename[NAME_BUF_SIZE]; /* file name */ - pid_t childpid = 0; /* Child process ID */ - int child_status; /* Status passed to waitpid */ - int child_wait_option = 0; /* Options passed to waitpid */ - int out_pdf[2]; - int notify = 0; - - TESTING("File locking environment variable"); +#else /* H5_HAVE_UNISTD_H */ + hid_t fid = -1; /* File ID */ + hid_t fapl = -1; /* File access property list */ + char filename[NAME_BUF_SIZE]; /* file name */ + pid_t childpid = 0; /* Child process ID */ + int child_status; /* Status passed to waitpid */ + int child_wait_option = 0; /* Options passed to waitpid */ + int out_pdf[2]; + int notify = 0; + int exit_status = 0; + herr_t ret; + + if (turn_locking_on && env_var_override) + TESTING("File locking: ON w/ env var override") + else if (turn_locking_on && !env_var_override) + TESTING("File locking: ON") + else if (!turn_locking_on && env_var_override) + TESTING("File locking: OFF w/ env var override") + else + TESTING("File locking: OFF") - /* Set the environment variable */ - if (HDsetenv("HDF5_USE_FILE_LOCKING", "FALSE", TRUE) < 0) + /* Copy the incoming fapl */ + if ((fapl = H5Pcopy(in_fapl)) < 0) TEST_ERROR - if ((fapl = H5Pcopy(in_fapl)) < 0) + /* Set locking in the fapl */ + if (H5Pset_file_locking(fapl, turn_locking_on ? TRUE : FALSE, TRUE) < 0) TEST_ERROR + /* If requested, set the environment variable */ + if (env_var_override) { + if (HDsetenv("HDF5_USE_FILE_LOCKING", turn_locking_on ? "FALSE" : "TRUE", TRUE) < 0) + TEST_ERROR + if (H5F__reparse_file_lock_variable_test() < 0) + TEST_ERROR + } + else { + if (HDsetenv("HDF5_USE_FILE_LOCKING", "", TRUE) < 0) + TEST_ERROR + if (H5F__reparse_file_lock_variable_test() < 0) + TEST_ERROR + } + /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[1], fapl, filename, sizeof(filename)); @@ -6242,10 +6293,8 @@ test_file_lock_env_var(hid_t in_fapl) if (H5Fclose(fid) < 0) TEST_ERROR - /* Open a file for read-only and then read-write. This would - * normally fail due to the file locking scheme but should - * pass when the environment variable is set to disable file - * locking. + /* Open a file for read-only and then read-write. This will fail + * when the locking scheme is turned on. */ /* Create 1 pipe */ @@ -6260,7 +6309,7 @@ test_file_lock_env_var(hid_t in_fapl) /* Child process */ - hid_t child_fid; /* File ID */ + hid_t child_fid = H5I_INVALID_HID; /* File ID */ int child_notify = 0; /* Close unused write end for out_pdf */ @@ -6271,18 +6320,25 @@ test_file_lock_env_var(hid_t in_fapl) while (child_notify != 1) { if (HDread(out_pdf[0], &child_notify, sizeof(int)) < 0) HDexit(EXIT_FAILURE); - } /* end while */ + } - /* Open the test file */ - if ((child_fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) - TEST_ERROR + /* Open and close the test file */ + H5E_BEGIN_TRY + { + child_fid = H5Fopen(filename, H5F_ACC_RDWR, fapl); + ret = H5Fclose(child_fid); + } + H5E_END_TRY; /* Close the pipe */ if (HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); - HDexit(EXIT_SUCCESS); - } /* end if */ + if (H5I_INVALID_HID == child_fid || FAIL == ret) + HDexit(EXIT_FAILURE); + else + HDexit(EXIT_SUCCESS); + } /* end child process work */ /* close unused read end for out_pdf */ if (HDclose(out_pdf[0]) < 0) @@ -6305,15 +6361,28 @@ test_file_lock_env_var(hid_t in_fapl) if (HDwaitpid(childpid, &child_status, child_wait_option) < 0) TEST_ERROR - /* Check if child terminated normally */ - if (WIFEXITED(child_status)) { - /* Check exit status of the child */ - if (WEXITSTATUS(child_status) != 0) - TEST_ERROR - } /* end if */ + /* Check exit status of the child */ + if (WIFEXITED(child_status)) + exit_status = WEXITSTATUS(child_status); else TEST_ERROR + /* The child process should have passed or failed as follows: + * + * locks on: FAIL + * locks off: PASS + * locks on, env var override: PASS + * locks off, env var override: FAIL + */ + if (turn_locking_on && !env_var_override && (0 == exit_status)) + TEST_ERROR + else if (!turn_locking_on && !env_var_override && (0 != exit_status)) + TEST_ERROR + else if (turn_locking_on && env_var_override && (0 != exit_status)) + TEST_ERROR + else if (!turn_locking_on && env_var_override && (0 == exit_status)) + TEST_ERROR + /* Close the file */ if (H5Fclose(fid) < 0) TEST_ERROR @@ -6336,9 +6405,86 @@ error: return -1; -#endif /* !(defined(H5_HAVE_FORK && defined(H5_HAVE_WAITPID)) */ +#endif /* H5_HAVE_UNISTD_H */ -} /* end test_file_lock_env_var() */ +} /* end test_file_locking() */ + +/**************************************************************** +** +** test_different_lock_flags(): +** Tests opening a file multiple times with different lock +** flags. +** +*****************************************************************/ +static int +test_different_lock_flags(hid_t in_fapl) +{ + hid_t fid1 = H5I_INVALID_HID; /* File ID */ + hid_t fid2 = H5I_INVALID_HID; /* File ID */ + hid_t fid3 = H5I_INVALID_HID; /* File ID */ + hid_t fapl_id = H5I_INVALID_HID; /* File access property list */ + char filename[NAME_BUF_SIZE]; /* File name */ + + TESTING("Using different lock flags") + + /* Copy the incoming fapl */ + if ((fapl_id = H5Pcopy(in_fapl)) < 0) + TEST_ERROR + + /* Set locking in the fapl */ + if (H5Pset_file_locking(fapl_id, TRUE, TRUE) < 0) + TEST_ERROR + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[1], fapl_id, filename, sizeof(filename)); + + /* Create the test file */ + if ((fid1 = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0) + TEST_ERROR + + /* Open the test file with the same flags (should pass) */ + if ((fid2 = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR + + /* Unset locking in the fapl */ + if (H5Pset_file_locking(fapl_id, FALSE, FALSE) < 0) + TEST_ERROR + + /* Open the test file with different flags (should FAIL) */ + H5E_BEGIN_TRY + { + fid3 = H5Fopen(filename, H5F_ACC_RDWR, fapl_id); + } + H5E_END_TRY; + if (H5I_INVALID_HID != fid3) + FAIL_PUTS_ERROR("Should not have been able to open a file with different locking flags") + + /* Close the files */ + if (H5Fclose(fid1) < 0) + TEST_ERROR + if (H5Fclose(fid2) < 0) + TEST_ERROR + + /* Close the copied property list */ + if (H5Pclose(fapl_id) < 0) + TEST_ERROR + + PASSED(); + + return 0; + +error: + H5E_BEGIN_TRY + { + H5Pclose(fapl_id); + H5Fclose(fid1); + H5Fclose(fid2); + H5Fclose(fid3); + } + H5E_END_TRY; + + return -1; +} /* end test_different_lock_flags() */ static int test_swmr_vfd_flag(void) @@ -6414,10 +6560,18 @@ error: static int test_bug_refresh(hid_t in_fapl) { - hid_t fid = -1; /* File ID */ - hid_t fapl; + hid_t fid = H5I_INVALID_HID; + hid_t fapl = H5I_INVALID_HID; H5F_t *f; - hid_t gid1, gid2, gid3, gid4, gid5, gid6, gid7, gid8, gid9; + hid_t gid1 = H5I_INVALID_HID; + hid_t gid2 = H5I_INVALID_HID; + hid_t gid3 = H5I_INVALID_HID; + hid_t gid4 = H5I_INVALID_HID; + hid_t gid5 = H5I_INVALID_HID; + hid_t gid6 = H5I_INVALID_HID; + hid_t gid7 = H5I_INVALID_HID; + hid_t gid8 = H5I_INVALID_HID; + hid_t gid9 = H5I_INVALID_HID; char filename[NAME_BUF_SIZE]; /* File name */ /* Create a copy of the input parameter in_fapl */ @@ -6545,17 +6699,24 @@ error: * (7) Refresh the dataset * (8) Verify the dataset's dimension and data are correct */ -#if !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) +#ifndef H5_HAVE_UNISTD_H static int -test_refresh_concur(hid_t H5_ATTR_UNUSED in_fapl, hbool_t H5_ATTR_UNUSED new_format) +test_refresh_concur(hid_t H5_ATTR_UNUSED in_fapl, hbool_t new_format) { + if (new_format) { + TESTING("H5Drefresh()--concurrent access for latest format"); + } + else { + TESTING("H5Drefresh()--concurrent access for non-latest-format"); + } + SKIPPED(); - HDputs(" Test skipped due to fork or waitpid not defined."); + HDputs(" Test skipped due to a lack of unistd.h functionality."); return 0; } /* test_refresh_concur() */ -#else /* defined(H5_HAVE_FORK && defined(H5_HAVE_WAITPID) */ +#else /* H5_HAVE_UNISTD_H */ static int test_refresh_concur(hid_t in_fapl, hbool_t new_format) @@ -6855,7 +7016,7 @@ error: return -1; } /* test_refresh_concur() */ -#endif /* !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) */ +#endif /* H5_HAVE_UNISTD_H */ /* * test_multiple_same(): @@ -7184,7 +7345,7 @@ error: } H5E_END_TRY; - return -1; + return 1; } /* test_multiple_same() */ @@ -7196,11 +7357,12 @@ error: int main(void) { - int nerrors = 0; /* The # of errors */ - hid_t fapl = -1; /* File access property list ID */ - char * driver = NULL; /* VFD string (from env variable) */ - char * lock_env_var = NULL; /* file locking env var pointer */ - hbool_t use_file_locking; /* read from env var */ + int nerrors = 0; /* The # of errors */ + hid_t fapl = -1; /* File access property list ID */ + char * driver = NULL; /* VFD string (from env variable) */ + char * lock_env_var = NULL; /* file locking env var pointer */ + hbool_t use_file_locking; /* read from env var */ + hbool_t file_locking_enabled = FALSE; /* Checks if the file system supports locks */ /* Skip this test if SWMR I/O is not supported for the VFD specified * by the environment variable. @@ -7221,6 +7383,13 @@ main(void) else use_file_locking = TRUE; + /* Check if file locking is enabled on this file system */ + if (use_file_locking) + if (h5_check_if_file_locking_enabled(&file_locking_enabled) < 0) { + HDprintf("Error when determining if file locks are enabled\n"); + return EXIT_FAILURE; + } + /* Set up */ h5_reset(); @@ -7262,7 +7431,7 @@ main(void) nerrors += test_append_flush_dataset_fixed(fapl); nerrors += test_append_flush_dataset_multiple(fapl); - if (use_file_locking) { + if (use_file_locking && file_locking_enabled) { /* * Tests for: * file open flags--single process access @@ -7289,10 +7458,19 @@ main(void) if (NULL == driver || !HDstrcmp(driver, "") || !HDstrcmp(driver, "sec2")) nerrors += test_swmr_vfd_flag(); - /* This test changes the HDF5_USE_FILE_LOCKING environment variable - * so it should be run last. + /* Test multiple opens via different locking flags */ + if (use_file_locking && file_locking_enabled) + nerrors += test_different_lock_flags(fapl); + + /* These tests change the HDF5_USE_FILE_LOCKING environment variable + * so they should be run last. */ - nerrors += test_file_lock_env_var(fapl); + if (use_file_locking && file_locking_enabled) { + nerrors += test_file_locking(fapl, TRUE, TRUE); + nerrors += test_file_locking(fapl, TRUE, FALSE); + nerrors += test_file_locking(fapl, FALSE, TRUE); + nerrors += test_file_locking(fapl, FALSE, FALSE); + } if (nerrors) goto error; diff --git a/test/tfile.c b/test/tfile.c index b4048b4..34f3389 100644 --- a/test/tfile.c +++ b/test/tfile.c @@ -1666,6 +1666,29 @@ test_file_is_accessible(const char *env_h5_drvr) is_hdf5 = H5Fis_accessible(filename, fapl_id); VERIFY(is_hdf5, TRUE, "H5Fis_accessible"); + /*****************************************/ + /* Newly created file that is still open */ + /*****************************************/ + + /* On Windows, file locking is mandatory so this check ensures that + * H5Fis_accessible() works on files that have an exclusive lock. + * Previous versions of this API call created an additional file handle + * and attempted to read through it, which will not work when locks + * are enforced by the OS. + */ + + /* Create a file and hold it open */ + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + CHECK(fid, H5I_INVALID_HID, "H5Fcreate"); + + /* Verify that the file is an HDF5 file */ + is_hdf5 = H5Fis_accessible(filename, fapl_id); + VERIFY(is_hdf5, TRUE, "H5Fis_accessible"); + + /* Close file */ + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + /*******************************/ /* Non-default user block size */ /*******************************/ @@ -3134,7 +3157,6 @@ test_rw_noupdate(void) ** test_userblock_alignment() test, to handle common testing ** ** Programmer: Quincey Koziol -** koziol@hdfgroup.org ** Septmber 10, 2009 ** *****************************************************************/ @@ -3196,7 +3218,6 @@ test_userblock_alignment_helper1(hid_t fcpl, hid_t fapl) ** test_userblock_alignment() test, to handle common testing ** ** Programmer: Quincey Koziol -** koziol@hdfgroup.org ** Septmber 10, 2009 ** *****************************************************************/ @@ -3266,7 +3287,6 @@ test_userblock_alignment_helper2(hid_t fapl, hbool_t open_rw) ** object [allocation] alignment size set interact properly. ** ** Programmer: Quincey Koziol -** koziol@hdfgroup.org ** Septmber 8, 2009 ** *****************************************************************/ @@ -4300,7 +4320,7 @@ set_multi_split(hid_t fapl, hsize_t pagesize, hbool_t split) /* Free memb_name */ for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) - free(memb_name[mt]); + HDfree(memb_name[mt]); return 0; @@ -5799,7 +5819,7 @@ test_libver_bounds_super_create(hid_t fapl, hid_t fcpl, htri_t is_swmr, htri_t n /* Get the internal file pointer if the create succeeds */ if (fid >= 0) { f = (H5F_t *)H5VL_object(fid); - CHECK(f, NULL, "H5VL_object"); + CHECK_PTR(f, "H5VL_object"); } /* Retrieve the low/high bounds */ @@ -5975,7 +5995,7 @@ test_libver_bounds_super_open(hid_t fapl, hid_t fcpl, htri_t is_swmr, htri_t non /* Get the internal file pointer */ f = (H5F_t *)H5VL_object(fid); - CHECK(f, NULL, "H5VL_object"); + CHECK_PTR(f, "H5VL_object"); /* The file's superblock version */ super_vers = f->shared->sblock->super_vers; @@ -6016,7 +6036,7 @@ test_libver_bounds_super_open(hid_t fapl, hid_t fcpl, htri_t is_swmr, htri_t non /* Get the internal file pointer if the open succeeds */ if (fid >= 0) { f = (H5F_t *)H5VL_object(fid); - CHECK(f, NULL, "H5VL_object"); + CHECK_PTR(f, "H5VL_object"); } /* Verify the file open succeeds or fails */ @@ -6194,7 +6214,7 @@ test_libver_bounds_obj(hid_t fapl) /* Get the internal file pointer */ f = (H5F_t *)H5VL_object(fid); - CHECK(f, NULL, "H5VL_object"); + CHECK_PTR(f, "H5VL_object"); /* Create a group in the file */ gid = H5Gcreate2(fid, GRP_NAME, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); @@ -6302,7 +6322,7 @@ test_libver_bounds_dataset(hid_t fapl) /* Get the internal dataset pointer */ dset = (H5D_t *)H5VL_object(did); - CHECK(dset, NULL, "H5VL_object"); + CHECK_PTR(dset, "H5VL_object"); /* Verify version for layout and fill value messages */ if (low == H5F_LIBVER_EARLIEST) { @@ -6350,7 +6370,7 @@ test_libver_bounds_dataset(hid_t fapl) /* Get the internal dataset pointer */ dset = (H5D_t *)H5VL_object(did); - CHECK(dset, NULL, "H5VL_object"); + CHECK_PTR(dset, "H5VL_object"); /* Verify layout message version and chunk indexing type */ VERIFY(dset->shared->layout.version, H5O_LAYOUT_VERSION_4, "H5O_layout_ver_bounds"); @@ -6412,7 +6432,7 @@ test_libver_bounds_dataset(hid_t fapl) /* Get the internal file pointer */ f = (H5F_t *)H5VL_object(fid); - CHECK(f, NULL, "H5VL_object"); + CHECK_PTR(f, "H5VL_object"); /* Create the chunked dataset */ did = H5Dcreate2(fid, DSETC, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); @@ -6420,7 +6440,7 @@ test_libver_bounds_dataset(hid_t fapl) /* Get the internal file pointer */ dset = (H5D_t *)H5VL_object(did); - CHECK(dset, NULL, "H5VL_object"); + CHECK_PTR(dset, "H5VL_object"); /* Verify the dataset's layout, fill value and filter pipeline message versions */ /* Also verify the chunk indexing type */ @@ -6541,7 +6561,7 @@ test_libver_bounds_dataspace(hid_t fapl) sid = H5Dget_space(did); CHECK(sid, H5I_INVALID_HID, "H5Dget_space"); space = (H5S_t *)H5I_object(sid); - CHECK(space, NULL, "H5I_object"); + CHECK_PTR(space, "H5I_object"); /* Verify the dataspace version */ VERIFY(space->extent.version, H5O_sdspace_ver_bounds[low], "H5O_sdspace_ver_bounds"); @@ -6558,7 +6578,7 @@ test_libver_bounds_dataspace(hid_t fapl) sid_null = H5Dget_space(did_null); CHECK(sid_null, H5I_INVALID_HID, "H5Dget_space"); space_null = (H5S_t *)H5I_object(sid_null); - CHECK(space_null, NULL, "H5I_object"); + CHECK_PTR(space_null, "H5I_object"); /* Verify the dataspace version */ VERIFY(space_null->extent.version, H5O_SDSPACE_VERSION_2, "H5O_sdspace_ver_bounds"); @@ -6636,7 +6656,7 @@ test_libver_bounds_dataspace(hid_t fapl) /* Get the internal file pointer */ f = (H5F_t *)H5VL_object(fid); - CHECK(f, NULL, "H5VL_object"); + CHECK_PTR(f, "H5VL_object"); /* Create the chunked dataset */ did = H5Dcreate2(fid, DSETA, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); @@ -6646,7 +6666,7 @@ test_libver_bounds_dataspace(hid_t fapl) tmp_sid = H5Dget_space(did); CHECK(tmp_sid, H5I_INVALID_HID, "H5Dget_space"); tmp_space = (H5S_t *)H5I_object(tmp_sid); - CHECK(tmp_space, NULL, "H5I_object"); + CHECK_PTR(tmp_space, "H5I_object"); /* Create the compact dataset */ did_compact = H5Dcreate2(fid, DSETB, H5T_NATIVE_INT, sid_compact, H5P_DEFAULT, dcpl_compact, @@ -6657,7 +6677,7 @@ test_libver_bounds_dataspace(hid_t fapl) tmp_sid_compact = H5Dget_space(did_compact); CHECK(tmp_sid_compact, H5I_INVALID_HID, "H5Dget_space"); tmp_space_compact = (H5S_t *)H5I_object(tmp_sid_compact); - CHECK(tmp_space_compact, NULL, "H5I_object"); + CHECK_PTR(tmp_space_compact, "H5I_object"); /* Create the contiguous dataset */ did_contig = @@ -6668,7 +6688,7 @@ test_libver_bounds_dataspace(hid_t fapl) tmp_sid_contig = H5Dget_space(did_contig); CHECK(tmp_sid_contig, H5I_INVALID_HID, "H5Dget_space"); tmp_space_contig = (H5S_t *)H5I_object(tmp_sid_contig); - CHECK(tmp_space_contig, NULL, "H5I_object"); + CHECK_PTR(tmp_space_contig, "H5I_object"); /* Verify versions for the three dataspaces */ VERIFY(tmp_space->extent.version, H5O_sdspace_ver_bounds[f->shared->low_bound], @@ -6892,7 +6912,7 @@ test_libver_bounds_datatype_check(hid_t fapl, hid_t tid) /* Get the internal datatype pointer */ dtype = (H5T_t *)H5I_object(dtid); - CHECK(dtype, NULL, "H5I_object"); + CHECK_PTR(dtype, "H5I_object"); /* Verify the datatype message version */ /* H5T_COMPOUND, H5T_ENUM, H5T_ARRAY: @@ -6966,13 +6986,13 @@ test_libver_bounds_datatype_check(hid_t fapl, hid_t tid) /* Get the internal file pointer */ f = (H5F_t *)H5VL_object(fid); - CHECK(f, NULL, "H5VL_object"); + CHECK_PTR(f, "H5VL_object"); /* Open the committed datatype */ str_tid = H5Topen2(fid, "datatype", H5P_DEFAULT); CHECK(str_tid, FAIL, "H5Topen2"); str_dtype = (H5T_t *)H5VL_object(str_tid); - CHECK(str_dtype, NULL, "H5VL_object"); + CHECK_PTR(str_dtype, "H5VL_object"); /* Verify the committed datatype message version */ VERIFY(str_dtype->shared->version, H5O_dtype_ver_bounds[H5F_LIBVER_EARLIEST], @@ -6992,7 +7012,7 @@ test_libver_bounds_datatype_check(hid_t fapl, hid_t tid) /* Get the internal datatype pointer */ dtype = (H5T_t *)H5I_object(dtid); - CHECK(dtype, NULL, "H5I_object"); + CHECK_PTR(dtype, "H5I_object"); /* Verify the dataset's datatype message version */ /* H5T_COMPOUND, H5T_ENUM, H5T_ARRAY: @@ -7125,7 +7145,7 @@ test_libver_bounds_attributes(hid_t fapl) /* Get the internal attribute pointer */ attr = (H5A_t *)H5VL_object(aid); - CHECK(attr, NULL, "H5VL_object"); + CHECK_PTR(attr, "H5VL_object"); /* Verify the attribute version */ if (low == H5F_LIBVER_EARLIEST) @@ -7144,7 +7164,7 @@ test_libver_bounds_attributes(hid_t fapl) /* Get the internal attribute pointer */ attr = (H5A_t *)H5VL_object(aid); - CHECK(attr, NULL, "H5VL_object"); + CHECK_PTR(attr, "H5VL_object"); /* Verify attribute version */ VERIFY(attr->shared->version, H5O_attr_ver_bounds[low], "H5O_attr_ver_bounds"); @@ -7165,7 +7185,7 @@ test_libver_bounds_attributes(hid_t fapl) /* Get internal attribute pointer */ attr = (H5A_t *)H5VL_object(aid); - CHECK(attr, NULL, "H5VL_object"); + CHECK_PTR(attr, "H5VL_object"); /* Verify attribute version */ if (low == H5F_LIBVER_EARLIEST) @@ -7230,7 +7250,7 @@ test_libver_bounds_attributes(hid_t fapl) /* Get the internal attribute pointer */ attr = (H5A_t *)H5VL_object(aid); - CHECK(attr, NULL, "H5VL_object"); + CHECK_PTR(attr, "H5VL_object"); /* Verify the attribute version */ if (low == H5F_LIBVER_EARLIEST) @@ -7292,7 +7312,7 @@ test_libver_bounds_attributes(hid_t fapl) /* Get the internal file pointer */ f = (H5F_t *)H5VL_object(fid); - CHECK(f, NULL, "H5VL_object"); + CHECK_PTR(f, "H5VL_object"); /* Open the group */ gid = H5Gopen2(fid, GRP_NAME, H5P_DEFAULT); @@ -7304,7 +7324,7 @@ test_libver_bounds_attributes(hid_t fapl) /* Get the internal attribute pointer */ attr = (H5A_t *)H5VL_object(aid); - CHECK(attr, NULL, "H5VL_object"); + CHECK_PTR(attr, "H5VL_object"); /* Verify the attribute message version */ VERIFY(attr->shared->version, H5O_attr_ver_bounds[f->shared->low_bound], diff --git a/tools/src/h5repack/h5repack_copy.c b/tools/src/h5repack/h5repack_copy.c index 9a62f31..55fb566 100644 --- a/tools/src/h5repack/h5repack_copy.c +++ b/tools/src/h5repack/h5repack_copy.c @@ -305,14 +305,6 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options) H5TOOLS_GOTO_ERROR((-1), "H5Fcreate could not create file <%s>:", fnameout); /*------------------------------------------------------------------------- - * write a new user block if requested - *------------------------------------------------------------------------- - */ - if (options->ublock_size > 0) - if (copy_user_block(options->ublock_filename, fnameout, options->ublock_size) < 0) - H5TOOLS_GOTO_ERROR((-1), "Could not copy user block. Exiting..."); - - /*------------------------------------------------------------------------- * get list of objects *------------------------------------------------------------------------- */ @@ -346,27 +338,60 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options) } /*------------------------------------------------------------------------- - * write only the input file user block if there is no user block file input + * Close the file and everything in it so the lock is removed + *------------------------------------------------------------------------- + */ + if (H5Pclose(fcpl) < 0) + H5TOOLS_GOTO_ERROR((-1), "could not close fcpl"); + if (H5Pclose(options->fout_fapl) < 0) + H5TOOLS_GOTO_ERROR((-1), "could not close fcpl"); + options->fout_fapl = H5P_DEFAULT; + if (H5Pclose(gcpl_in) < 0) + H5TOOLS_GOTO_ERROR((-1), "could not close fcpl"); + if (H5Gclose(grp_in) < 0) + H5TOOLS_GOTO_ERROR((-1), "could not close fcpl"); + if (H5Fclose(fidout) < 0) + H5TOOLS_GOTO_ERROR((-1), "could not close fcpl"); + if (H5Fclose(fidin) < 0) + H5TOOLS_GOTO_ERROR((-1), "could not close fcpl"); + + /*------------------------------------------------------------------------- + * NOTE: The userblock MUST be written out AFTER the file is closed or + * the file locking will cause failures on Windows, where file locks + * are mandatory, not advisory. + *------------------------------------------------------------------------- + */ + + /*------------------------------------------------------------------------- + * Write a new user block if requested, using the input file user block if + * there is no separate user block file input *------------------------------------------------------------------------- */ - if (ub_size > 0 && options->ublock_size == 0) + if (options->ublock_size > 0) { + if (copy_user_block(options->ublock_filename, fnameout, options->ublock_size) < 0) + H5TOOLS_GOTO_ERROR((-1), "Could not copy user block. Exiting..."); + } + else if (ub_size > 0 && options->ublock_size == 0) { if (copy_user_block(fnamein, fnameout, ub_size) < 0) H5TOOLS_GOTO_ERROR((-1), "Could not copy user block. Exiting..."); + } done: - H5E_BEGIN_TRY - { - H5Pclose(fcpl_in); - H5Pclose(gcpl_in); - H5Pclose(fcpl); - H5Gclose(grp_in); - H5Fclose(fidin); - H5Fclose(fidout); - H5Fclose(fidin); - H5Fclose(fidout); + if (-1 == ret_value) { + H5E_BEGIN_TRY + { + H5Pclose(fcpl); + H5Pclose(options->fout_fapl); + options->fout_fapl = H5P_DEFAULT; + H5Pclose(gcpl_in); + H5Gclose(grp_in); + H5Pclose(fcpl_in); + H5Fclose(fidout); + H5Fclose(fidin); + } + H5E_END_TRY; } - H5E_END_TRY; if (travt) trav_table_free(travt); -- cgit v0.12