diff options
Diffstat (limited to 'src')
51 files changed, 5873 insertions, 1135 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9ff16c9..34afd42 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -230,6 +230,7 @@ set (H5FD_SOURCES ${HDF5_SRC_DIR}/H5FDs3comms.c ${HDF5_SRC_DIR}/H5FDsec2.c ${HDF5_SRC_DIR}/H5FDspace.c + ${HDF5_SRC_DIR}/H5FDsplitter.c ${HDF5_SRC_DIR}/H5FDstdio.c ${HDF5_SRC_DIR}/H5FDtest.c ${HDF5_SRC_DIR}/H5FDwindows.c @@ -248,6 +249,7 @@ set (H5FD_HDRS ${HDF5_SRC_DIR}/H5FDros3.h ${HDF5_SRC_DIR}/H5FDs3comms.h ${HDF5_SRC_DIR}/H5FDsec2.h + ${HDF5_SRC_DIR}/H5FDsplitter.h ${HDF5_SRC_DIR}/H5FDstdio.h ${HDF5_SRC_DIR}/H5FDwindows.h ) diff --git a/src/H5EAdblock.c b/src/H5EAdblock.c index d926fd5..d540a3c 100644 --- a/src/H5EAdblock.c +++ b/src/H5EAdblock.c @@ -15,7 +15,7 @@ * * Created: H5EAdblock.c * Sep 11 2008 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * * Purpose: Data block routines for extensible arrays. * @@ -92,7 +92,6 @@ H5FL_DEFINE_STATIC(H5EA_dblock_t); * Return: Non-NULL pointer to data block on success/NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 11 2008 * *------------------------------------------------------------------------- @@ -153,7 +152,6 @@ END_FUNC(PKG) /* end H5EA__dblock_alloc() */ * Return: Valid file address on success/HADDR_UNDEF on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 9 2008 * *------------------------------------------------------------------------- @@ -248,7 +246,6 @@ END_FUNC(PKG) /* end H5EA__dblock_create() */ * Return: Super block index on success/Can't fail * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 11 2008 * *------------------------------------------------------------------------- @@ -285,7 +282,6 @@ END_FUNC(PKG) /* end H5EA__dblock_sblk_idx() */ * Return: Non-NULL pointer to data block on success/NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 18 2008 * *------------------------------------------------------------------------- @@ -348,7 +344,6 @@ END_FUNC(PKG) /* end H5EA__dblock_protect() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 11 2008 * *------------------------------------------------------------------------- @@ -379,7 +374,6 @@ END_FUNC(PKG) /* end H5EA__dblock_unprotect() */ * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 22 2008 * *------------------------------------------------------------------------- @@ -442,7 +436,6 @@ END_FUNC(PKG) /* end H5EA__dblock_delete() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 11 2008 * *------------------------------------------------------------------------- diff --git a/src/H5EAhdr.c b/src/H5EAhdr.c index ec40298..8b4d5f7 100644 --- a/src/H5EAhdr.c +++ b/src/H5EAhdr.c @@ -15,7 +15,7 @@ * * Created: H5EAhdr.c * Aug 26 2008 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * * Purpose: Array header routines for extensible arrays. * @@ -110,7 +110,6 @@ H5FL_SEQ_DEFINE_STATIC(H5EA_sblk_info_t); * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Aug 26 2008 * *------------------------------------------------------------------------- @@ -180,7 +179,6 @@ END_FUNC(PKG) /* end H5EA__hdr_alloc() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 18 2008 * *------------------------------------------------------------------------- @@ -245,7 +243,6 @@ END_FUNC(PKG) /* end H5EA__hdr_init() */ * Return: Non-NULL pointer to buffer for elements on success/NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 16 2008 * *------------------------------------------------------------------------- @@ -312,7 +309,6 @@ END_FUNC(PKG) /* end H5EA__hdr_alloc_elmts() */ * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 18 2008 * *------------------------------------------------------------------------- @@ -349,7 +345,6 @@ END_FUNC(PKG) /* end H5EA__hdr_free_elmts() */ * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Jun 17 2008 * *------------------------------------------------------------------------- @@ -464,7 +459,6 @@ END_FUNC(PKG) /* end H5EA__hdr_create() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Aug 26 2008 * *------------------------------------------------------------------------- @@ -497,7 +491,6 @@ END_FUNC(PKG) /* end H5EA__hdr_incr() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Aug 26 2008 * *------------------------------------------------------------------------- @@ -533,7 +526,6 @@ END_FUNC(PKG) /* end H5EA__hdr_decr() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Aug 26 2008 * *------------------------------------------------------------------------- @@ -559,7 +551,6 @@ END_FUNC(PKG) /* end H5EA__hdr_fuse_incr() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Aug 26 2008 * *------------------------------------------------------------------------- @@ -589,7 +580,6 @@ END_FUNC(PKG) /* end H5EA__hdr_fuse_decr() */ * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 9 2008 * *------------------------------------------------------------------------- @@ -619,7 +609,6 @@ END_FUNC(PKG) /* end H5EA__hdr_modified() */ * Return: Non-NULL pointer to header on success/NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Jul 31 2013 * *------------------------------------------------------------------------- @@ -677,7 +666,6 @@ END_FUNC(PKG) /* end H5EA__hdr_protect() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Aug 1 2013 * *------------------------------------------------------------------------- @@ -708,7 +696,6 @@ END_FUNC(PKG) /* end H5EA__hdr_unprotect() */ * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Aug 26 2008 * *------------------------------------------------------------------------- @@ -765,7 +752,6 @@ END_FUNC(PKG) /* end H5EA__hdr_delete() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 11 2008 * *------------------------------------------------------------------------- diff --git a/src/H5EAiblock.c b/src/H5EAiblock.c index 1b5957a..c45d15a 100644 --- a/src/H5EAiblock.c +++ b/src/H5EAiblock.c @@ -15,7 +15,7 @@ * * Created: H5EAiblock.c * Sep 9 2008 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * * Purpose: Index block routines for extensible arrays. * @@ -42,7 +42,7 @@ #include "H5EApkg.h" /* Extensible Arrays */ #include "H5FLprivate.h" /* Free Lists */ #include "H5MFprivate.h" /* File memory management */ -#include "H5VMprivate.h" /* Vectors and arrays */ +#include "H5VMprivate.h" /* Vectors and arrays */ /****************/ @@ -98,7 +98,6 @@ H5FL_SEQ_DEFINE_STATIC(haddr_t); * Return: Non-NULL pointer to index block on success/NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 9 2008 * *------------------------------------------------------------------------- @@ -129,11 +128,6 @@ H5EA__iblock_alloc(H5EA_hdr_t *hdr)) iblock->nsblks = H5EA_SBLK_FIRST_IDX(hdr->cparam.sup_blk_min_data_ptrs); iblock->ndblk_addrs = 2 * ((size_t)hdr->cparam.sup_blk_min_data_ptrs - 1); iblock->nsblk_addrs = hdr->nsblks - iblock->nsblks; -#ifdef QAK -HDfprintf(stderr, "%s: iblock->nsblks = %u\n", FUNC, iblock->nsblks); -HDfprintf(stderr, "%s: iblock->ndblk_addrs = %Zu\n", FUNC, iblock->ndblk_addrs); -HDfprintf(stderr, "%s: iblock->nsblk_addrs = %Zu\n", FUNC, iblock->nsblk_addrs); -#endif /* QAK */ /* Allocate buffer for elements in index block */ if(hdr->cparam.idx_blk_elmts > 0) @@ -169,7 +163,6 @@ END_FUNC(PKG) /* end H5EA__iblock_alloc() */ * Return: Valid file address on success/HADDR_UNDEF on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 9 2008 * *------------------------------------------------------------------------- @@ -183,10 +176,6 @@ H5EA__iblock_create(H5EA_hdr_t *hdr, hbool_t *stats_changed)) haddr_t iblock_addr; /* Extensible array index block address */ hbool_t inserted = FALSE; /* Whether the header was inserted into cache */ -#ifdef QAK -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* QAK */ - /* Sanity check */ HDassert(hdr); HDassert(stats_changed); @@ -197,9 +186,6 @@ HDfprintf(stderr, "%s: Called\n", FUNC); /* Set size of index block on disk */ iblock->size = H5EA_IBLOCK_SIZE(iblock); -#ifdef QAK -HDfprintf(stderr, "%s: iblock->size = %Zu\n", FUNC, iblock->size); -#endif /* QAK */ /* Allocate space for the index block on disk */ if(HADDR_UNDEF == (iblock_addr = H5MF_alloc(hdr->f, H5FD_MEM_EARRAY_IBLOCK, (hsize_t)iblock->size))) @@ -284,7 +270,6 @@ END_FUNC(PKG) /* end H5EA__iblock_create() */ * Return: Non-NULL pointer to index block on success/NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 9 2008 * *------------------------------------------------------------------------- @@ -296,10 +281,6 @@ H5EA__iblock_protect(H5EA_hdr_t *hdr, unsigned flags)) /* Local variables */ H5EA_iblock_t *iblock = NULL; /* Pointer to index block */ -#ifdef QAK -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* QAK */ - /* Sanity check */ HDassert(hdr); @@ -340,7 +321,6 @@ END_FUNC(PKG) /* end H5EA__iblock_protect() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 9 2008 * *------------------------------------------------------------------------- @@ -351,10 +331,6 @@ H5EA__iblock_unprotect(H5EA_iblock_t *iblock, unsigned cache_flags)) /* Local variables */ -#ifdef QAK -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* QAK */ - /* Sanity check */ HDassert(iblock); @@ -375,7 +351,6 @@ END_FUNC(PKG) /* end H5EA__iblock_unprotect() */ * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 9 2008 * *------------------------------------------------------------------------- @@ -387,10 +362,6 @@ H5EA__iblock_delete(H5EA_hdr_t *hdr)) /* Local variables */ H5EA_iblock_t *iblock = NULL; /* Pointer to index block */ -#ifdef QAK -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* QAK */ - /* Sanity check */ HDassert(hdr); HDassert(H5F_addr_defined(hdr->idx_blk_addr)); @@ -459,7 +430,6 @@ END_FUNC(PKG) /* end H5EA__iblock_delete() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 11 2008 * *------------------------------------------------------------------------- diff --git a/src/H5EAint.c b/src/H5EAint.c index 2baf1f4..ef8cd7a 100644 --- a/src/H5EAint.c +++ b/src/H5EAint.c @@ -15,7 +15,7 @@ * * Created: H5EAint.c * Jun 17 2008 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * * Purpose: Internal routines for extnsible arrays. * @@ -86,7 +86,6 @@ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Mar 26 2009 * *------------------------------------------------------------------------- @@ -116,7 +115,6 @@ END_FUNC(PKG) /* end H5EA__create_flush_depend() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Mar 26 2009 * *------------------------------------------------------------------------- diff --git a/src/H5EAsblock.c b/src/H5EAsblock.c index fb7c458..b5b9d94 100644 --- a/src/H5EAsblock.c +++ b/src/H5EAsblock.c @@ -42,7 +42,7 @@ #include "H5EApkg.h" /* Extensible Arrays */ #include "H5FLprivate.h" /* Free Lists */ #include "H5MFprivate.h" /* File memory management */ -#include "H5VMprivate.h" /* Vectors and arrays */ +#include "H5VMprivate.h" /* Vectors and arrays */ /****************/ diff --git a/src/H5EAstat.c b/src/H5EAstat.c index 509d3f8..68e0b1e 100644 --- a/src/H5EAstat.c +++ b/src/H5EAstat.c @@ -15,7 +15,7 @@ * * Created: H5EAstat.c * Sep 11 2008 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * * Purpose: Extensible array metadata statistics functions. * @@ -87,7 +87,6 @@ * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Aug 21 2008 * *------------------------------------------------------------------------- @@ -98,10 +97,6 @@ H5EA_get_stats(const H5EA_t *ea, H5EA_stat_t *stats)) /* Local variables */ -#ifdef QAK -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* QAK */ - /* * Check arguments. */ diff --git a/src/H5EAtest.c b/src/H5EAtest.c index 8926d6a..a0802bc 100644 --- a/src/H5EAtest.c +++ b/src/H5EAtest.c @@ -11,7 +11,7 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Quincey Koziol <koziol@hdfgroup.org> +/* Programmer: Quincey Koziol * Thursday, August 28, 2008 * * Purpose: Extensible array testing functions. @@ -38,7 +38,7 @@ #include "H5Eprivate.h" /* Error handling */ #include "H5EApkg.h" /* Extensible Arrays */ #include "H5FLprivate.h" /* Free Lists */ -#include "H5VMprivate.h" /* Vector functions */ +#include "H5VMprivate.h" /* Vector functions */ /****************/ @@ -60,9 +60,6 @@ /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ @@ -74,86 +71,6 @@ hbool_t H5_PKG_INIT_VAR = FALSE; /*******************/ -/* File ID class */ -static const H5I_class_t H5I_FILE_CLS[1] = {{ - H5I_FILE, /* ID class value */ - 0, /* Class flags */ - 0, /* # of reserved IDs for class */ - (H5I_free_t)H5F__close_cb /* Callback routine for closing objects of this class */ -}}; - - -/*-------------------------------------------------------------------------- -NAME - H5F__init_package -- Initialize interface-specific information -USAGE - herr_t H5F__init_package() -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. - ---------------------------------------------------------------------------*/ -herr_t -H5F__init_package(void) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* - * Initialize the atom group for the file IDs. - */ - if(H5I_register_type(H5I_FILE_CLS) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to initialize interface") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5F__init_package() */ - - -/*------------------------------------------------------------------------- - * Function: H5F_term_package - * - * Purpose: Terminate this interface: free all memory and reset global - * variables to their initial values. Release all ID groups - * associated with this interface. - * - * Return: Success: Positive if anything was done that might - * have affected other interfaces; - * zero otherwise. - * - * Failure: Never fails - *------------------------------------------------------------------------- - */ -int -H5F_term_package(void) -{ - int n = 0; - - FUNC_ENTER_NOAPI_NOINIT_NOERR - - if(H5_PKG_INIT_VAR) { - if(H5I_nmembers(H5I_FILE) > 0) { - (void)H5I_clear_type(H5I_FILE, FALSE, FALSE); - n++; /*H5I*/ - } /* end if */ - else { - /* Make certain we've cleaned up all the shared file objects */ - H5F_sfile_assert_num(0); - - /* Destroy the file object id group */ - n += (H5I_dec_type_ref(H5I_FILE) > 0); - - /* Mark closed */ - if(0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end else */ - } /* end if */ - - FUNC_LEAVE_NOAPI(n) -} /* end H5F_term_package() */ - /*------------------------------------------------------------------------- * Function: H5Fget_create_plist @@ -1436,7 +1436,7 @@ done: * constant H5P_DEFAULT). The bytes to be written come from the * buffer BUF. * - * Return: SNon-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1642,7 +1642,7 @@ H5FDlock(H5FD_t *file, hbool_t rw) /* Call private function */ if(H5FD_lock(file, rw) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "file lock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "file lock request failed") done: FUNC_LEAVE_API(ret_value) @@ -1672,7 +1672,7 @@ H5FD_lock(H5FD_t *file, hbool_t rw) /* Dispatch to driver */ if(file->cls->lock && (file->cls->lock)(file, rw) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "driver lock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "driver lock request failed") done: FUNC_LEAVE_NOAPI(ret_value) @@ -1704,7 +1704,7 @@ H5FDunlock(H5FD_t *file) /* Call private function */ if(H5FD_unlock(file) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "file unlock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "file unlock request failed") done: FUNC_LEAVE_API(ret_value) @@ -1733,7 +1733,7 @@ H5FD_unlock(H5FD_t *file) /* Dispatch to driver */ if(file->cls->unlock && (file->cls->unlock)(file) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "driver unlock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "driver unlock request failed") done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDcore.c b/src/H5FDcore.c index e4a3207..40d838c 100644 --- a/src/H5FDcore.c +++ b/src/H5FDcore.c @@ -36,6 +36,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_CORE_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* The skip list node type. Represents a region in the file. */ typedef struct H5FD_core_region_t { haddr_t start; /* Start address of the region */ @@ -55,6 +58,7 @@ typedef struct H5FD_core_t { size_t increment; /* multiples for mem allocation */ hbool_t backing_store; /* write to file name on flush */ size_t bstore_page_size; /* backing store page size */ + hbool_t ignore_disabled_file_locks; int fd; /* backing store file descriptor */ /* Information for determining uniqueness of a file with a backing store */ #ifndef H5_HAVE_WIN32_API @@ -407,9 +411,19 @@ done: static herr_t H5FD__init_package(void) { + char *lock_env_var = NULL; /* Environment variable pointer */ herr_t ret_value = SUCCEED; - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC + + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if(lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ if(H5FD_core_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize core VFD") @@ -691,6 +705,16 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr /* Save file image callbacks */ file->fi_callbacks = file_image_info.callbacks; + /* Check the file locking flags in the fapl */ + if(ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if(H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + if(fd >= 0) { /* Retrieve information for determining uniqueness of file */ #ifdef H5_HAVE_WIN32_API @@ -1509,18 +1533,20 @@ H5FD_core_lock(H5FD_t *_file, hbool_t rw) * descriptor, this is a no-op. */ if(file->fd >= 0) { - /* Set exclusive or shared lock based on rw status */ lock_flags = rw ? LOCK_EX : LOCK_SH; /* Place a non-blocking lock on the file */ if(HDflock(file->fd, lock_flags | LOCK_NB) < 0) { - if(ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") } /* end if */ - } /* end if */ done: @@ -1549,16 +1575,17 @@ H5FD_core_unlock(H5FD_t *_file) HDassert(file); - if(file->fd >= 0) { - + if(file->fd >= 0) if(HDflock(file->fd, LOCK_UN) < 0) { - if(ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") - } /* end if */ - - } /* end if */ + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDdirect.c b/src/H5FDdirect.c index 34c4346..4046050 100644 --- a/src/H5FDdirect.c +++ b/src/H5FDdirect.c @@ -38,6 +38,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_DIRECT_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* File operations */ #define OP_UNKNOWN 0 #define OP_READ 1 @@ -71,6 +74,7 @@ typedef struct H5FD_direct_t { haddr_t pos; /*current file I/O position */ int op; /*last operation */ H5FD_direct_fapl_t fa; /*file access properties */ + hbool_t ignore_disabled_file_locks; #ifndef H5_HAVE_WIN32_API /* * On most systems the combination of device and i-node number uniquely @@ -193,10 +197,20 @@ DESCRIPTION static herr_t H5FD__init_package(void) { + char *lock_env_var = NULL; /* Environment variable pointer */ herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if(lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if(H5FD_direct_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize direct VFD") @@ -518,6 +532,16 @@ H5FD_direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxadd file->fa.fbsize = fa->fbsize; file->fa.cbsize = fa->cbsize; + /* Check the file locking flags in the fapl */ + if(ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if(H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + /* Try to decide if data alignment is required. The reason to check it here * is to handle correctly the case that the file is in a different file system * than the one where the program is running. @@ -1335,16 +1359,27 @@ static herr_t H5FD_direct_lock(H5FD_t *_file, hbool_t rw) { H5FD_direct_t *file = (H5FD_direct_t*)_file; /* VFD file struct */ - const int lock = rw ? LOCK_EX : LOCK_SH; + int lock_flags; /* file locking flags */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT HDassert(file); - /* Place the lock with non-blocking */ - if(HDflock(file->fd, lock | LOCK_NB) < 0) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to flock file") + /* Set exclusive or shared lock based on rw status */ + lock_flags = rw ? LOCK_EX : LOCK_SH; + + /* Place a non-blocking lock on the file */ + if(HDflock(file->fd, lock_flags | LOCK_NB) < 0) { + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } + else + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file") + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -1372,8 +1407,16 @@ H5FD_direct_unlock(H5FD_t *_file) HDassert(file); - if(HDflock(file->fd, LOCK_UN) < 0) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to flock (unlock) file") + if(HDflock(file->fd, LOCK_UN) < 0) { + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } + else + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDfamily.c b/src/H5FDfamily.c index d110ef7..154878e 100644 --- a/src/H5FDfamily.c +++ b/src/H5FDfamily.c @@ -1372,9 +1372,9 @@ H5FD_family_lock(H5FD_t *_file, hbool_t rw) for(v = 0; v < u; v++) { if(H5FD_unlock(file->memb[v]) < 0) /* Push error, but keep going */ - HDONE_ERROR(H5E_IO, H5E_CANTUNLOCK, FAIL, "unable to unlock member files") + HDONE_ERROR(H5E_IO, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock member files") } /* end for */ - HGOTO_ERROR(H5E_IO, H5E_CANTLOCK, FAIL, "unable to lock member files") + HGOTO_ERROR(H5E_IO, H5E_CANTLOCKFILE, FAIL, "unable to lock member files") } /* end if */ done: @@ -1406,7 +1406,7 @@ H5FD_family_unlock(H5FD_t *_file) for(u = 0; u < file->nmembs; u++) if(file->memb[u]) if(H5FD_unlock(file->memb[u]) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTUNLOCK, FAIL, "unable to unlock member files") + HGOTO_ERROR(H5E_IO, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock member files") done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDlog.c b/src/H5FDlog.c index f649bc4..5d1b536 100644 --- a/src/H5FDlog.c +++ b/src/H5FDlog.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@hdfgroup.org> + * Programmer: Quincey Koziol * Monday, April 17, 2000 * * Purpose: The POSIX unbuffered file driver using only the HDF5 public @@ -40,6 +40,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_LOG_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* Driver-specific file access properties */ typedef struct H5FD_log_fapl_t { char *logfile; /* Allocated log file name */ @@ -80,6 +83,7 @@ typedef struct H5FD_log_t { haddr_t eof; /* end of file; current file size */ haddr_t pos; /* current file I/O position */ H5FD_file_op_t op; /* last operation */ + hbool_t ignore_disabled_file_locks; char filename[H5FD_MAX_FILENAME_LEN]; /* Copy of file name from open operation */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely @@ -151,71 +155,71 @@ typedef struct H5FD_log_t { * which can be addressed entirely by the second * argument of the file seek function. */ -#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) -#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR)) +#define MAXADDR (((haddr_t)1 << (8 * sizeof(HDoff_t) - 1)) - 1) +#define ADDR_OVERFLOW(A) (HADDR_UNDEF == (A) || ((A) & ~(haddr_t)MAXADDR)) #define SIZE_OVERFLOW(Z) ((Z) & ~(hsize_t)MAXADDR) #define REGION_OVERFLOW(A,Z) (ADDR_OVERFLOW(A) || SIZE_OVERFLOW(Z) || \ - HADDR_UNDEF==(A)+(Z) || \ - (HDoff_t)((A)+(Z))<(HDoff_t)(A)) + HADDR_UNDEF == (A) + (Z) || \ + (HDoff_t)((A) + (Z)) < (HDoff_t)(A)) /* Prototypes */ -static herr_t H5FD_log_term(void); -static void *H5FD_log_fapl_get(H5FD_t *file); -static void *H5FD_log_fapl_copy(const void *_old_fa); -static herr_t H5FD_log_fapl_free(void *_fa); -static H5FD_t *H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, +static herr_t H5FD__log_term(void); +static void *H5FD__log_fapl_get(H5FD_t *file); +static void *H5FD__log_fapl_copy(const void *_old_fa); +static herr_t H5FD__log_fapl_free(void *_fa); +static H5FD_t *H5FD__log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); -static herr_t H5FD_log_close(H5FD_t *_file); -static int H5FD_log_cmp(const H5FD_t *_f1, const H5FD_t *_f2); -static herr_t H5FD_log_query(const H5FD_t *_f1, unsigned long *flags); -static haddr_t H5FD_log_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size); +static herr_t H5FD__log_close(H5FD_t *_file); +static int H5FD__log_cmp(const H5FD_t *_f1, const H5FD_t *_f2); +static herr_t H5FD__log_query(const H5FD_t *_f1, unsigned long *flags); +static haddr_t H5FD__log_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size); static herr_t H5FD__log_free(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t size); -static haddr_t H5FD_log_get_eoa(const H5FD_t *_file, H5FD_mem_t type); -static herr_t H5FD_log_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); -static haddr_t H5FD_log_get_eof(const H5FD_t *_file, H5FD_mem_t type); -static herr_t H5FD_log_get_handle(H5FD_t *_file, hid_t fapl, void** file_handle); -static herr_t H5FD_log_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, +static haddr_t H5FD__log_get_eoa(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD__log_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); +static haddr_t H5FD__log_get_eof(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD__log_get_handle(H5FD_t *_file, hid_t fapl, void** file_handle); +static herr_t H5FD__log_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, size_t size, void *buf); -static herr_t H5FD_log_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, +static herr_t H5FD__log_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, size_t size, const void *buf); -static herr_t H5FD_log_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); -static herr_t H5FD_log_lock(H5FD_t *_file, hbool_t rw); -static herr_t H5FD_log_unlock(H5FD_t *_file); +static herr_t H5FD__log_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); +static herr_t H5FD__log_lock(H5FD_t *_file, hbool_t rw); +static herr_t H5FD__log_unlock(H5FD_t *_file); static const H5FD_class_t H5FD_log_g = { - "log", /*name */ - MAXADDR, /*maxaddr */ - H5F_CLOSE_WEAK, /* fc_degree */ - H5FD_log_term, /*terminate */ - NULL, /*sb_size */ - NULL, /*sb_encode */ - NULL, /*sb_decode */ - sizeof(H5FD_log_fapl_t), /*fapl_size */ - H5FD_log_fapl_get, /*fapl_get */ - H5FD_log_fapl_copy, /*fapl_copy */ - H5FD_log_fapl_free, /*fapl_free */ - 0, /*dxpl_size */ - NULL, /*dxpl_copy */ - NULL, /*dxpl_free */ - H5FD_log_open, /*open */ - H5FD_log_close, /*close */ - H5FD_log_cmp, /*cmp */ - H5FD_log_query, /*query */ - NULL, /*get_type_map */ - H5FD_log_alloc, /*alloc */ - H5FD__log_free, /*free */ - H5FD_log_get_eoa, /*get_eoa */ - H5FD_log_set_eoa, /*set_eoa */ - H5FD_log_get_eof, /*get_eof */ - H5FD_log_get_handle, /*get_handle */ - H5FD_log_read, /*read */ - H5FD_log_write, /*write */ - NULL, /*flush */ - H5FD_log_truncate, /*truncate */ - H5FD_log_lock, /*lock */ - H5FD_log_unlock, /*unlock */ - H5FD_FLMAP_DICHOTOMY /*fl_map */ + "log", /* name */ + MAXADDR, /* maxaddr */ + H5F_CLOSE_WEAK, /* fc_degree */ + H5FD__log_term, /* terminate */ + NULL, /* sb_size */ + NULL, /* sb_encode */ + NULL, /* sb_decode */ + sizeof(H5FD_log_fapl_t), /* fapl_size */ + H5FD__log_fapl_get, /* fapl_get */ + H5FD__log_fapl_copy, /* fapl_copy */ + H5FD__log_fapl_free, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD__log_open, /* open */ + H5FD__log_close, /* close */ + H5FD__log_cmp, /* cmp */ + H5FD__log_query, /* query */ + NULL, /* get_type_map */ + H5FD__log_alloc, /* alloc */ + H5FD__log_free, /* free */ + H5FD__log_get_eoa, /* get_eoa */ + H5FD__log_set_eoa, /* set_eoa */ + H5FD__log_get_eof, /* get_eof */ + H5FD__log_get_handle, /* get_handle */ + H5FD__log_read, /* read */ + H5FD__log_write, /* write */ + NULL, /* flush */ + H5FD__log_truncate, /* truncate */ + H5FD__log_lock, /* lock */ + H5FD__log_unlock, /* unlock */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ }; /* Declare a free list to manage the H5FD_log_t struct */ @@ -234,10 +238,20 @@ H5FL_DEFINE_STATIC(H5FD_log_t); static herr_t H5FD__init_package(void) { + char *lock_env_var = NULL; /* Environment variable pointer */ herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if(lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if(H5FD_log_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize log VFD") @@ -279,7 +293,7 @@ done: /*--------------------------------------------------------------------------- - * Function: H5FD_log_term + * Function: H5FD__log_term * * Purpose: Shut down the VFD * @@ -291,15 +305,15 @@ done: *--------------------------------------------------------------------------- */ static herr_t -H5FD_log_term(void) +H5FD__log_term(void) { - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR /* Reset VFL ID */ H5FD_LOG_g = 0; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5FD_log_term() */ +} /* end H5FD__log_term() */ /*------------------------------------------------------------------------- @@ -352,7 +366,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5FD_log_fapl_get + * Function: H5FD__log_fapl_get * * Purpose: Returns a file access property list which indicates how the * specified file is being accessed. The return list could be @@ -368,22 +382,22 @@ done: *------------------------------------------------------------------------- */ static void * -H5FD_log_fapl_get(H5FD_t *_file) +H5FD__log_fapl_get(H5FD_t *_file) { H5FD_log_t *file = (H5FD_log_t *)_file; void *ret_value = NULL; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR /* Set return value */ - ret_value = H5FD_log_fapl_copy(&(file->fa)); + ret_value = H5FD__log_fapl_copy(&(file->fa)); FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_fapl_get() */ +} /* end H5FD__log_fapl_get() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_fapl_copy + * Function: H5FD__log_fapl_copy * * Purpose: Copies the log-specific file access properties. * @@ -396,13 +410,13 @@ H5FD_log_fapl_get(H5FD_t *_file) *------------------------------------------------------------------------- */ static void * -H5FD_log_fapl_copy(const void *_old_fa) +H5FD__log_fapl_copy(const void *_old_fa) { const H5FD_log_fapl_t *old_fa = (const H5FD_log_fapl_t*)_old_fa; H5FD_log_fapl_t *new_fa = NULL; /* New FAPL info */ void *ret_value = NULL; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC HDassert(old_fa); @@ -430,11 +444,11 @@ done: } /* end if */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_fapl_copy() */ +} /* end H5FD__log_fapl_copy() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_fapl_free + * Function: H5FD__log_fapl_free * * Purpose: Frees the log-specific file access properties. * @@ -446,11 +460,11 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_log_fapl_free(void *_fa) +H5FD__log_fapl_free(void *_fa) { H5FD_log_fapl_t *fa = (H5FD_log_fapl_t*)_fa; - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR /* Free the fapl information */ if(fa->logfile) @@ -458,11 +472,11 @@ H5FD_log_fapl_free(void *_fa) H5MM_xfree(fa); FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5FD_log_fapl_free() */ +} /* end H5FD__log_fapl_free() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_open + * Function: H5FD__log_open * * Purpose: Create and/or opens a file as an HDF5 file. * @@ -477,7 +491,7 @@ H5FD_log_fapl_free(void *_fa) *------------------------------------------------------------------------- */ static H5FD_t * -H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) +H5FD__log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) { H5FD_log_t *file = NULL; H5P_genplist_t *plist; /* Property list */ @@ -487,15 +501,12 @@ H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) #ifdef H5_HAVE_WIN32_API struct _BY_HANDLE_FILE_INFORMATION fileinfo; #endif -#ifdef H5_HAVE_GETTIMEOFDAY - struct timeval timeval_start; - struct timeval open_timeval_diff; - struct timeval stat_timeval_diff; -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_t open_timer; /* Timer for open() call */ + H5_timer_t stat_timer; /* Timer for stat() call */ h5_stat_t sb; H5FD_t *ret_value = NULL; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC /* Sanity check on file offsets */ HDcompile_assert(sizeof(HDoff_t) >= sizeof(size_t)); @@ -523,54 +534,36 @@ H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) if(NULL == (fa = (const H5FD_log_fapl_t *)H5P_peek_driver_info(plist))) HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "bad VFL driver info") -#ifdef H5_HAVE_GETTIMEOFDAY - if(fa->flags & H5FD_LOG_TIME_OPEN) - HDgettimeofday(&timeval_start, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Start timer for open() call */ + if(fa->flags & H5FD_LOG_TIME_OPEN) { + H5_timer_init(&open_timer); + H5_timer_start(&open_timer); + } /* end if */ + /* Open the file */ if((fd = HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW)) < 0) { int myerrno = errno; HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file: name = '%s', errno = %d, error message = '%s', flags = %x, o_flags = %x", name, myerrno, HDstrerror(myerrno), flags, (unsigned)o_flags); } /* end if */ -#ifdef H5_HAVE_GETTIMEOFDAY - if(fa->flags & H5FD_LOG_TIME_OPEN) { - struct timeval timeval_stop; - HDgettimeofday(&timeval_stop, NULL); + /* Stop timer for open() call */ + if(fa->flags & H5FD_LOG_TIME_OPEN) + H5_timer_stop(&open_timer); - /* Calculate the elapsed gettimeofday time */ - open_timeval_diff.tv_usec = timeval_stop.tv_usec - timeval_start.tv_usec; - open_timeval_diff.tv_sec = timeval_stop.tv_sec - timeval_start.tv_sec; - if(open_timeval_diff.tv_usec < 0) { - open_timeval_diff.tv_usec += 1000000; - open_timeval_diff.tv_sec--; - } /* end if */ + /* Start timer for stat() call */ + if(fa->flags & H5FD_LOG_TIME_STAT) { + H5_timer_init(&stat_timer); + H5_timer_start(&stat_timer); } /* end if */ -#endif /* H5_HAVE_GETTIMEOFDAY */ -#ifdef H5_HAVE_GETTIMEOFDAY - if(fa->flags & H5FD_LOG_TIME_STAT) - HDgettimeofday(&timeval_start, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ /* Get the file stats */ if(HDfstat(fd, &sb) < 0) HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, NULL, "unable to fstat file") -#ifdef H5_HAVE_GETTIMEOFDAY - if(fa->flags & H5FD_LOG_TIME_STAT) { - struct timeval timeval_stop; - HDgettimeofday(&timeval_stop, NULL); - - /* Calculate the elapsed gettimeofday time */ - stat_timeval_diff.tv_usec = timeval_stop.tv_usec - timeval_start.tv_usec; - stat_timeval_diff.tv_sec = timeval_stop.tv_sec - timeval_start.tv_sec; - if(stat_timeval_diff.tv_usec < 0) { - stat_timeval_diff.tv_usec += 1000000; - stat_timeval_diff.tv_sec--; - } /* end if */ - } /* end if */ -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Stop timer for stat() call */ + if(fa->flags & H5FD_LOG_TIME_STAT) + H5_timer_stop(&stat_timer); /* Create the new file struct */ if(NULL == (file = H5FL_CALLOC(H5FD_log_t))) @@ -631,15 +624,31 @@ H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) else file->logfp = stderr; -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_OPEN) - HDfprintf(file->logfp, "Open took: (%f s)\n", (double)open_timeval_diff.tv_sec + ((double)open_timeval_diff.tv_usec / (double)1000000.0f)); - if(file->fa.flags & H5FD_LOG_TIME_STAT) - HDfprintf(file->logfp, "Stat took: (%f s)\n", (double)stat_timeval_diff.tv_sec + ((double)stat_timeval_diff.tv_usec / (double)1000000.0f)); -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Log the timer values */ + if(file->fa.flags & H5FD_LOG_TIME_OPEN) { + H5_timevals_t open_times; /* Elapsed time for open() call */ + + H5_timer_get_times(open_timer, &open_times); + HDfprintf(file->logfp, "Open took: (%f s)\n", open_times.elapsed); + } /* end if */ + if(file->fa.flags & H5FD_LOG_TIME_STAT) { + H5_timevals_t stat_times; /* Elapsed time for stat() call */ + H5_timer_get_times(stat_timer, &stat_times); + HDfprintf(file->logfp, "Stat took: (%f s)\n", stat_times.elapsed); + } /* end if */ } /* end if */ + /* Check the file locking flags in the fapl */ + if(ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if(H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + /* Check for non-default FAPL */ if(H5P_FILE_ACCESS_DEFAULT != fapl_id) { /* This step is for h5repart tool only. If user wants to change file driver from @@ -664,11 +673,11 @@ done: } /* end if */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_open() */ +} /* end H5FD__log_open() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_close + * Function: H5FD__log_close * * Purpose: Closes an HDF5 file. * @@ -681,30 +690,30 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_log_close(H5FD_t *_file) +H5FD__log_close(H5FD_t *_file) { H5FD_log_t *file = (H5FD_log_t *)_file; -#ifdef H5_HAVE_GETTIMEOFDAY - struct timeval timeval_start, timeval_stop; -#endif /* H5_HAVE_GETTIMEOFDAY */ - herr_t ret_value = SUCCEED; /* Return value */ + H5_timer_t close_timer; /* Timer for close() call */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC /* Sanity check */ HDassert(file); -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags&H5FD_LOG_TIME_CLOSE) - HDgettimeofday(&timeval_start, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Start timer for close() call */ + if(file->fa.flags & H5FD_LOG_TIME_CLOSE) { + H5_timer_init(&close_timer); + H5_timer_start(&close_timer); + } /* end if */ + /* Close the underlying file */ if(HDclose(file->fd) < 0) HSYS_GOTO_ERROR(H5E_IO, H5E_CANTCLOSEFILE, FAIL, "unable to close file") -#ifdef H5_HAVE_GETTIMEOFDAY + + /* Stop timer for close() call */ if(file->fa.flags&H5FD_LOG_TIME_CLOSE) - HDgettimeofday(&timeval_stop, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_stop(&close_timer); /* Dump I/O information */ if(file->fa.flags != 0) { @@ -712,20 +721,12 @@ H5FD_log_close(H5FD_t *_file) haddr_t last_addr; unsigned char last_val; -#ifdef H5_HAVE_GETTIMEOFDAY if(file->fa.flags & H5FD_LOG_TIME_CLOSE) { - struct timeval timeval_diff; - - /* Calculate the elapsed gettimeofday time */ - timeval_diff.tv_usec = timeval_stop.tv_usec - timeval_start.tv_usec; - timeval_diff.tv_sec = timeval_stop.tv_sec - timeval_start.tv_sec; - if(timeval_diff.tv_usec < 0) { - timeval_diff.tv_usec += 1000000; - timeval_diff.tv_sec--; - } /* end if */ - HDfprintf(file->logfp, "Close took: (%f s)\n", (double)timeval_diff.tv_sec + ((double)timeval_diff.tv_usec / (double)1000000.0f)); + H5_timevals_t close_times; /* Elapsed time for close() call */ + + H5_timer_get_times(close_timer, &close_times); + HDfprintf(file->logfp, "Close took: (%f s)\n", close_times.elapsed); } /* end if */ -#endif /* H5_HAVE_GETTIMEOFDAY */ /* Dump the total number of seek/read/write operations */ if(file->fa.flags & H5FD_LOG_NUM_READ) @@ -817,11 +818,11 @@ H5FD_log_close(H5FD_t *_file) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_close() */ +} /* end H5FD__log_close() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_cmp + * Function: H5FD__log_cmp * * Purpose: Compares two files belonging to this driver using an * arbitrary (but consistent) ordering. @@ -836,13 +837,13 @@ done: *------------------------------------------------------------------------- */ static int -H5FD_log_cmp(const H5FD_t *_f1, const H5FD_t *_f2) +H5FD__log_cmp(const H5FD_t *_f1, const H5FD_t *_f2) { const H5FD_log_t *f1 = (const H5FD_log_t *)_f1; const H5FD_log_t *f2 = (const H5FD_log_t *)_f2; int ret_value = 0; - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR #ifdef H5_HAVE_WIN32_API if(f1->dwVolumeSerialNumber < f2->dwVolumeSerialNumber) HGOTO_DONE(-1) @@ -873,11 +874,11 @@ H5FD_log_cmp(const H5FD_t *_f1, const H5FD_t *_f2) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_cmp() */ +} /* end H5FD__log_cmp() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_query + * Function: H5FD__log_query * * Purpose: Set the flags that this VFL driver is capable of supporting. * (listed in H5FDpublic.h) @@ -890,11 +891,11 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_log_query(const H5FD_t *_file, unsigned long *flags /* out */) +H5FD__log_query(const H5FD_t *_file, unsigned long *flags /* out */) { const H5FD_log_t *file = (const H5FD_log_t *)_file; - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR /* Set the VFL feature flags that this driver supports */ if(flags) { @@ -913,11 +914,11 @@ H5FD_log_query(const H5FD_t *_file, unsigned long *flags /* out */) } /* end if */ FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5FD_log_query() */ +} /* end H5FD__log_query() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_alloc + * Function: H5FD__log_alloc * * Purpose: Allocate file memory. * @@ -930,13 +931,13 @@ H5FD_log_query(const H5FD_t *_file, unsigned long *flags /* out */) *------------------------------------------------------------------------- */ static haddr_t -H5FD_log_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hsize_t size) +H5FD__log_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hsize_t size) { H5FD_log_t *file = (H5FD_log_t *)_file; haddr_t addr; haddr_t ret_value = HADDR_UNDEF; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR /* Compute the address for the block to allocate */ addr = file->eoa; @@ -960,7 +961,7 @@ H5FD_log_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hsi ret_value = addr; FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_alloc() */ +} /* end H5FD__log_alloc() */ /*------------------------------------------------------------------------- @@ -1001,7 +1002,7 @@ H5FD__log_free(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, /*------------------------------------------------------------------------- - * Function: H5FD_log_get_eoa + * Function: H5FD__log_get_eoa * * Purpose: Gets the end-of-address marker for the file. The EOA marker * is the first address past the last byte allocated in the @@ -1016,18 +1017,18 @@ H5FD__log_free(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, *------------------------------------------------------------------------- */ static haddr_t -H5FD_log_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +H5FD__log_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) { const H5FD_log_t *file = (const H5FD_log_t *)_file; - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR FUNC_LEAVE_NOAPI(file->eoa) -} /* end H5FD_log_get_eoa() */ +} /* end H5FD__log_get_eoa() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_set_eoa + * Function: H5FD__log_set_eoa * * Purpose: Set the end-of-address marker for the file. This function is * called shortly after an existing HDF5 file is opened in order @@ -1041,11 +1042,11 @@ H5FD_log_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) *------------------------------------------------------------------------- */ static herr_t -H5FD_log_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr) +H5FD__log_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr) { H5FD_log_t *file = (H5FD_log_t *)_file; - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR if(file->fa.flags != 0) { /* Check for increasing file size */ @@ -1084,11 +1085,11 @@ H5FD_log_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr) file->eoa = addr; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5FD_log_set_eoa() */ +} /* end H5FD__log_set_eoa() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_get_eof + * Function: H5FD__log_get_eof * * Purpose: Returns the end-of-file marker, which is the greater of * either the filesystem end-of-file or the HDF5 end-of-address @@ -1105,18 +1106,18 @@ H5FD_log_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr) *------------------------------------------------------------------------- */ static haddr_t -H5FD_log_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +H5FD__log_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) { const H5FD_log_t *file = (const H5FD_log_t *)_file; - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR FUNC_LEAVE_NOAPI(file->eof) -} /* end H5FD_log_get_eof() */ +} /* end H5FD__log_get_eof() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_get_handle + * Function: H5FD__log_get_handle * * Purpose: Returns the file handle of LOG file driver. * @@ -1128,12 +1129,12 @@ H5FD_log_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) *------------------------------------------------------------------------- */ static herr_t -H5FD_log_get_handle(H5FD_t *_file, hid_t H5_ATTR_UNUSED fapl, void **file_handle) +H5FD__log_get_handle(H5FD_t *_file, hid_t H5_ATTR_UNUSED fapl, void **file_handle) { H5FD_log_t *file = (H5FD_log_t *)_file; herr_t ret_value = SUCCEED; - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC if(!file_handle) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file handle not valid") @@ -1142,11 +1143,11 @@ H5FD_log_get_handle(H5FD_t *_file, hid_t H5_ATTR_UNUSED fapl, void **file_handle done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_get_handle() */ +} /* end H5FD__log_get_handle() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_read + * Function: H5FD__log_read * * Purpose: Reads SIZE bytes of data from FILE beginning at address ADDR * into buffer BUF according to data transfer properties in @@ -1162,19 +1163,22 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_log_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, +H5FD__log_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, size_t size, void *buf/*out*/) { H5FD_log_t *file = (H5FD_log_t *)_file; size_t orig_size = size; /* Save the original size for later */ haddr_t orig_addr = addr; -#ifdef H5_HAVE_GETTIMEOFDAY - struct timeval timeval_start, timeval_stop; -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_t read_timer; /* Timer for read operation */ + H5_timevals_t read_times; /* Elapsed time for read operation */ +#ifndef H5_HAVE_PREADWRITE + H5_timer_t seek_timer; /* Timer for seek operation */ + H5_timevals_t seek_times; /* Elapsed time for seek operation */ +#endif /* H5_HAVE_PREADWRITE */ HDoff_t offset = (HDoff_t)addr; - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC HDassert(file && file->pub.cls); HDassert(buf); @@ -1201,59 +1205,56 @@ H5FD_log_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hadd #ifndef H5_HAVE_PREADWRITE /* Seek to the correct location (if we don't have pread) */ if(addr != file->pos || OP_READ != file->op) { -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_SEEK) - HDgettimeofday(&timeval_start, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Start timer for seek() call */ + if(file->fa.flags & H5FD_LOG_TIME_SEEK) { + H5_timer_init(&seek_timer); + H5_timer_start(&seek_timer); + } /* end if */ + if(HDlseek(file->fd, (HDoff_t)addr, SEEK_SET) < 0) HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to seek to proper position") -#ifdef H5_HAVE_GETTIMEOFDAY + + /* Stop timer for seek() call */ if(file->fa.flags & H5FD_LOG_TIME_SEEK) - HDgettimeofday(&timeval_stop, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_stop(&seek_timer); - /* Log information about the seek */ + /* Add to the number of seeks, when tracking that */ if(file->fa.flags & H5FD_LOG_NUM_SEEK) file->total_seek_ops++; + + /* Add to the total seek time, when tracking that */ + if(file->fa.flags & H5FD_LOG_TIME_SEEK) { + H5_timer_get_times(seek_timer, &seek_times); + file->total_seek_time += seek_times.elapsed; + } /* end if */ + + /* Emit log string if we're tracking individual seek events. */ if(file->fa.flags & H5FD_LOG_LOC_SEEK) { HDfprintf(file->logfp, "Seek: From %10a To %10a", file->pos, addr); -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_SEEK) { - struct timeval timeval_diff; - double time_diff; - - /* Calculate the elapsed gettimeofday time */ - timeval_diff.tv_usec = timeval_stop.tv_usec - timeval_start.tv_usec; - timeval_diff.tv_sec = timeval_stop.tv_sec - timeval_start.tv_sec; - if(timeval_diff.tv_usec < 0) { - timeval_diff.tv_usec += 1000000; - timeval_diff.tv_sec--; - } /* end if */ - time_diff = (double)timeval_diff.tv_sec + ((double)timeval_diff.tv_usec / (double)1000000.0f); - HDfprintf(file->logfp, " (%fs @ %.6lu.%.6llu)\n", time_diff, (unsigned long long)timeval_start.tv_sec, (unsigned long long)timeval_start.tv_usec); - /* Add to total seek time */ - file->total_seek_time += time_diff; - } /* end if */ - else - HDfprintf(file->logfp, "\n"); -#else /* H5_HAVE_GETTIMEOFDAY */ - HDfprintf(file->logfp, "\n"); -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Add the seek time, if we're tracking that. + * Note that the seek time is NOT emitted for when just H5FD_LOG_TIME_SEEK + * is set. + */ + if(file->fa.flags & H5FD_LOG_TIME_SEEK) + HDfprintf(file->logfp, " (%fs @ %f)\n", seek_times.elapsed, seek_timer.initial.elapsed); + else + HDfprintf(file->logfp, "\n"); } /* end if */ } /* end if */ #endif /* H5_HAVE_PREADWRITE */ + /* Start timer for read operation */ + if(file->fa.flags & H5FD_LOG_TIME_READ) { + H5_timer_init(&read_timer); + H5_timer_start(&read_timer); + } /* end if */ + /* * Read data, being careful of interrupted system calls, partial results, * and the end of the file. */ -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_READ) - HDgettimeofday(&timeval_start, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ while(size > 0) { - h5_posix_io_t bytes_in = 0; /* # of bytes to read */ h5_posix_io_ret_t bytes_read = -1; /* # of bytes actually read */ @@ -1301,14 +1302,22 @@ H5FD_log_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hadd buf = (char *)buf + bytes_read; } /* end while */ -#ifdef H5_HAVE_GETTIMEOFDAY + + /* Stop timer for read operation */ if(file->fa.flags & H5FD_LOG_TIME_READ) - HDgettimeofday(&timeval_stop, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_stop(&read_timer); - /* Log information about the read */ + /* Add to the number of reads, when tracking that */ if(file->fa.flags & H5FD_LOG_NUM_READ) file->total_read_ops++; + + /* Add to the total read time, when tracking that */ + if(file->fa.flags & H5FD_LOG_TIME_READ) { + H5_timer_get_times(read_timer, &read_times); + file->total_read_time += read_times.elapsed; + } /* end if */ + + /* Log information about the read */ if(file->fa.flags & H5FD_LOG_LOC_READ) { HDfprintf(file->logfp, "%10a-%10a (%10Zu bytes) (%s) Read", orig_addr, (orig_addr + orig_size) - 1, orig_size, flavors[type]); @@ -1318,30 +1327,14 @@ H5FD_log_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hadd HDassert(type == H5FD_MEM_DEFAULT || type == (H5FD_mem_t)file->flavor[(orig_addr + orig_size) - 1] || (H5FD_mem_t)file->flavor[(orig_addr + orig_size) - 1] == H5FD_MEM_DEFAULT); } /* end if */ - -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_READ) { - struct timeval timeval_diff; - double time_diff; - - /* Calculate the elapsed gettimeofday time */ - timeval_diff.tv_usec = timeval_stop.tv_usec - timeval_start.tv_usec; - timeval_diff.tv_sec = timeval_stop.tv_sec - timeval_start.tv_sec; - if(timeval_diff.tv_usec < 0) { - timeval_diff.tv_usec += 1000000; - timeval_diff.tv_sec--; - } /* end if */ - time_diff = (double)timeval_diff.tv_sec + ((double)timeval_diff.tv_usec / (double)1000000.0f); - HDfprintf(file->logfp, " (%fs @ %.6lu.%.6llu)\n", time_diff, (unsigned long long)timeval_start.tv_sec, (unsigned long long)timeval_start.tv_usec); - - /* Add to total read time */ - file->total_read_time += time_diff; - } /* end if */ + /* Add the read time, if we're tracking that. + * Note that the read time is NOT emitted for when just H5FD_LOG_TIME_READ + * is set. + */ + if(file->fa.flags & H5FD_LOG_TIME_READ) + HDfprintf(file->logfp, " (%fs @ %f)\n", read_times.elapsed, read_timer.initial.elapsed); else HDfprintf(file->logfp, "\n"); -#else /* H5_HAVE_GETTIMEOFDAY */ - HDfprintf(file->logfp, "\n"); -#endif /* H5_HAVE_GETTIMEOFDAY */ } /* end if */ /* Update current position */ @@ -1356,11 +1349,11 @@ done: } /* end if */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_read() */ +} /* end H5FD__log_read() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_write + * Function: H5FD__log_write * * Purpose: Writes SIZE bytes of data to FILE beginning at address ADDR * from buffer BUF according to data transfer properties in @@ -1374,19 +1367,22 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_log_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, +H5FD__log_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, size_t size, const void *buf) { H5FD_log_t *file = (H5FD_log_t *)_file; size_t orig_size = size; /* Save the original size for later */ haddr_t orig_addr = addr; -#ifdef H5_HAVE_GETTIMEOFDAY - struct timeval timeval_start, timeval_stop; -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_t write_timer; /* Timer for write operation */ + H5_timevals_t write_times; /* Elapsed time for write operation */ +#ifndef H5_HAVE_PREADWRITE + H5_timer_t seek_timer; /* Timer for seek operation */ + H5_timevals_t seek_times; /* Elapsed time for seek operation */ +#endif /* H5_HAVE_PREADWRITE */ HDoff_t offset = (HDoff_t)addr; - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC HDassert(file && file->pub.cls); HDassert(size > 0); @@ -1418,59 +1414,56 @@ H5FD_log_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, had #ifndef H5_HAVE_PREADWRITE /* Seek to the correct location (if we don't have pwrite) */ if(addr != file->pos || OP_WRITE != file->op) { -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_SEEK) - HDgettimeofday(&timeval_start, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Start timer for seek() call */ + if(file->fa.flags & H5FD_LOG_TIME_SEEK) { + H5_timer_init(&seek_timer); + H5_timer_start(&seek_timer); + } /* end if */ + if(HDlseek(file->fd, (HDoff_t)addr, SEEK_SET) < 0) HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to seek to proper position") -#ifdef H5_HAVE_GETTIMEOFDAY + + /* Stop timer for seek() call */ if(file->fa.flags & H5FD_LOG_TIME_SEEK) - HDgettimeofday(&timeval_stop, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_stop(&seek_timer); - /* Log information about the seek */ + /* Add to the number of seeks, when tracking that */ if(file->fa.flags & H5FD_LOG_NUM_SEEK) file->total_seek_ops++; + + /* Add to the total seek time, when tracking that */ + if(file->fa.flags & H5FD_LOG_TIME_SEEK) { + H5_timer_get_times(seek_timer, &seek_times); + file->total_seek_time += seek_times.elapsed; + } /* end if */ + + /* Emit log string if we're tracking individual seek events. */ if(file->fa.flags & H5FD_LOG_LOC_SEEK) { HDfprintf(file->logfp, "Seek: From %10a To %10a", file->pos, addr); -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_SEEK) { - struct timeval timeval_diff; - double time_diff; - - /* Calculate the elapsed gettimeofday time */ - timeval_diff.tv_usec = timeval_stop.tv_usec - timeval_start.tv_usec; - timeval_diff.tv_sec = timeval_stop.tv_sec - timeval_start.tv_sec; - if(timeval_diff.tv_usec < 0) { - timeval_diff.tv_usec += 1000000; - timeval_diff.tv_sec--; - } /* end if */ - time_diff = (double)timeval_diff.tv_sec + ((double)timeval_diff.tv_usec / (double)1000000.0f); - HDfprintf(file->logfp, " (%fs @ %.6lu.%.6llu)\n", time_diff, (unsigned long long)timeval_start.tv_sec, (unsigned long long)timeval_start.tv_usec); - /* Add to total seek time */ - file->total_seek_time += time_diff; - } /* end if */ - else - HDfprintf(file->logfp, "\n"); -#else /* H5_HAVE_GETTIMEOFDAY */ - HDfprintf(file->logfp, "\n"); -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Add the seek time, if we're tracking that. + * Note that the seek time is NOT emitted for when just H5FD_LOG_TIME_SEEK + * is set. + */ + if(file->fa.flags & H5FD_LOG_TIME_SEEK) + HDfprintf(file->logfp, " (%fs @ %f)\n", seek_times.elapsed, seek_timer.initial.elapsed); + else + HDfprintf(file->logfp, "\n"); } /* end if */ } /* end if */ #endif /* H5_HAVE_PREADWRITE */ + /* Start timer for write operation */ + if(file->fa.flags&H5FD_LOG_TIME_WRITE) { + H5_timer_init(&write_timer); + H5_timer_start(&write_timer); + } /* end if */ + /* * Write the data, being careful of interrupted system calls and partial * results */ -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags&H5FD_LOG_TIME_WRITE) - HDgettimeofday(&timeval_start, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ while(size > 0) { - h5_posix_io_t bytes_in = 0; /* # of bytes to write */ h5_posix_io_ret_t bytes_wrote = -1; /* # of bytes written */ @@ -1511,14 +1504,22 @@ H5FD_log_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, had addr += (haddr_t)bytes_wrote; buf = (const char *)buf + bytes_wrote; } /* end while */ -#ifdef H5_HAVE_GETTIMEOFDAY + + /* Stop timer for write operation */ if(file->fa.flags & H5FD_LOG_TIME_WRITE) - HDgettimeofday(&timeval_stop, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_stop(&write_timer); - /* Log information about the write */ + /* Add to the number of writes, when tracking that */ if(file->fa.flags & H5FD_LOG_NUM_WRITE) file->total_write_ops++; + + /* Add to the total write time, when tracking that */ + if(file->fa.flags & H5FD_LOG_TIME_WRITE) { + H5_timer_get_times(write_timer, &write_times); + file->total_write_time += write_times.elapsed; + } /* end if */ + + /* Log information about the write */ if(file->fa.flags & H5FD_LOG_LOC_WRITE) { HDfprintf(file->logfp, "%10a-%10a (%10Zu bytes) (%s) Written", orig_addr, (orig_addr + orig_size) - 1, orig_size, flavors[type]); @@ -1530,29 +1531,14 @@ H5FD_log_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, had } /* end if */ } /* end if */ -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_WRITE) { - struct timeval timeval_diff; - double time_diff; - - /* Calculate the elapsed gettimeofday time */ - timeval_diff.tv_usec = timeval_stop.tv_usec - timeval_start.tv_usec; - timeval_diff.tv_sec = timeval_stop.tv_sec - timeval_start.tv_sec; - if(timeval_diff.tv_usec < 0) { - timeval_diff.tv_usec += 1000000; - timeval_diff.tv_sec--; - } /* end if */ - time_diff = (double)timeval_diff.tv_sec + ((double)timeval_diff.tv_usec / (double)1000000.0f); - HDfprintf(file->logfp, " (%fs @ %.6lu.%.6llu)\n", time_diff, (unsigned long long)timeval_start.tv_sec, (unsigned long long)timeval_start.tv_usec); - - /* Add to total write time */ - file->total_write_time += time_diff; - } /* end if */ - else - HDfprintf(file->logfp, "\n"); -#else /* H5_HAVE_GETTIMEOFDAY */ - HDfprintf(file->logfp, "\n"); -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Add the write time, if we're tracking that. + * Note that the write time is NOT emitted for when just H5FD_LOG_TIME_WRITE + * is set. + */ + if(file->fa.flags & H5FD_LOG_TIME_WRITE) + HDfprintf(file->logfp, " (%fs @ %f)\n", write_times.elapsed, write_timer.initial.elapsed); + else + HDfprintf(file->logfp, "\n"); } /* end if */ /* Update current position and eof */ @@ -1569,11 +1555,11 @@ done: } /* end if */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_write() */ +} /* end H5FD__log_write() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_truncate + * Function: H5FD__log_truncate * * Purpose: Makes sure that the true file size is the same (or larger) * than the end-of-address. @@ -1586,34 +1572,33 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_log_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_UNUSED closing) +H5FD__log_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_UNUSED closing) { H5FD_log_t *file = (H5FD_log_t *)_file; herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC HDassert(file); /* Extend the file to make sure it's large enough */ if(!H5F_addr_eq(file->eoa, file->eof)) { -#ifdef H5_HAVE_GETTIMEOFDAY - struct timeval timeval_start, timeval_stop; -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_t trunc_timer; /* Timer for truncate operation */ + H5_timevals_t trunc_times; /* Elapsed time for truncate operation */ + + /* Start timer for truncate operation */ + if(file->fa.flags & H5FD_LOG_TIME_TRUNCATE) { + H5_timer_init(&trunc_timer); + H5_timer_start(&trunc_timer); + } /* end if */ + #ifdef H5_HAVE_WIN32_API +{ LARGE_INTEGER li; /* 64-bit (union) integer for SetFilePointer() call */ DWORD dwPtrLow; /* Low-order pointer bits from SetFilePointer() * Only used as an error code here. */ - DWORD dwError; /* DWORD error code from GetLastError() */ - BOOL bError; /* Boolean error flag */ -#endif /* H5_HAVE_WIN32_API */ -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_TRUNCATE) - HDgettimeofday(&timeval_start, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ -#ifdef H5_HAVE_WIN32_API /* Windows uses this odd QuadPart union for 32/64-bit portability */ li.QuadPart = (__int64)file->eoa; @@ -1624,51 +1609,48 @@ H5FD_log_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_U */ dwPtrLow = SetFilePointer(file->hFile, li.LowPart, &li.HighPart, FILE_BEGIN); if(INVALID_SET_FILE_POINTER == dwPtrLow) { + DWORD dwError; /* DWORD error code from GetLastError() */ + dwError = GetLastError(); if(dwError != NO_ERROR ) HGOTO_ERROR(H5E_FILE, H5E_FILEOPEN, FAIL, "unable to set file pointer") - } + } /* end if */ - bError = SetEndOfFile(file->hFile); - if(0 == bError) + if(0 == SetEndOfFile(file->hFile)) HGOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to extend file properly") +} #else /* H5_HAVE_WIN32_API */ + /* Truncate/extend the file */ if(-1 == HDftruncate(file->fd, (HDoff_t)file->eoa)) HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to extend file properly") #endif /* H5_HAVE_WIN32_API */ -#ifdef H5_HAVE_GETTIMEOFDAY + + /* Stop timer for truncate operation */ if(file->fa.flags & H5FD_LOG_TIME_TRUNCATE) - HDgettimeofday(&timeval_stop, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_stop(&trunc_timer); - /* Log information about the truncate */ + /* Add to the number of truncates, when tracking that */ if(file->fa.flags & H5FD_LOG_NUM_TRUNCATE) file->total_truncate_ops++; + + /* Add to the total truncate time, when tracking that */ + if(file->fa.flags & H5FD_LOG_TIME_TRUNCATE) { + H5_timer_get_times(trunc_timer, &trunc_times); + file->total_truncate_time += trunc_times.elapsed; + } /* end if */ + + /* Emit log string if we're tracking individual truncate events. */ if(file->fa.flags & H5FD_LOG_TRUNCATE) { HDfprintf(file->logfp, "Truncate: To %10a", file->eoa); -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_TRUNCATE) { - struct timeval timeval_diff; - double time_diff; - - /* Calculate the elapsed gettimeofday time */ - timeval_diff.tv_usec = timeval_stop.tv_usec - timeval_start.tv_usec; - timeval_diff.tv_sec = timeval_stop.tv_sec - timeval_start.tv_sec; - if(timeval_diff.tv_usec < 0) { - timeval_diff.tv_usec += 1000000; - timeval_diff.tv_sec--; - } /* end if */ - time_diff = (double)timeval_diff.tv_sec + ((double)timeval_diff.tv_usec / (double)1000000.0f); - HDfprintf(file->logfp, " (%fs @ %.6lu.%.6llu)\n", time_diff, (unsigned long long)timeval_start.tv_sec, (unsigned long long)timeval_start.tv_usec); - /* Add to total truncate time */ - file->total_truncate_time += time_diff; - } /* end if */ - else - HDfprintf(file->logfp, "\n"); -#else /* H5_HAVE_GETTIMEOFDAY */ - HDfprintf(file->logfp, "\n"); -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Add the truncate time, if we're tracking that. + * Note that the truncate time is NOT emitted for when just H5FD_LOG_TIME_TRUNCATE + * is set. + */ + if(file->fa.flags & H5FD_LOG_TIME_TRUNCATE) + HDfprintf(file->logfp, " (%fs @ %f)\n", trunc_times.elapsed, trunc_timer.initial.elapsed); + else + HDfprintf(file->logfp, "\n"); } /* end if */ /* Update the eof value */ @@ -1681,11 +1663,11 @@ H5FD_log_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_U done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_truncate() */ +} /* end H5FD__log_truncate() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_lock + * Function: H5FD__log_lock * * Purpose: Place a lock on the file * @@ -1697,13 +1679,13 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_log_lock(H5FD_t *_file, hbool_t rw) +H5FD__log_lock(H5FD_t *_file, hbool_t rw) { H5FD_log_t *file = (H5FD_log_t *)_file; /* VFD file struct */ int lock_flags; /* file locking flags */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC /* Sanity check */ HDassert(file); @@ -1713,19 +1695,23 @@ H5FD_log_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if(HDflock(file->fd, lock_flags | LOCK_NB) < 0) { - if(ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file") + } done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_lock() */ +} /* end H5FD__log_lock() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_unlock + * Function: H5FD__log_unlock * * Purpose: Remove the existing lock on the file * @@ -1736,23 +1722,27 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_log_unlock(H5FD_t *_file) +H5FD__log_unlock(H5FD_t *_file) { H5FD_log_t *file = (H5FD_log_t *)_file; /* VFD file struct */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC HDassert(file); if(HDflock(file->fd, LOCK_UN) < 0) { - if(ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file") + } done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_unlock() */ +} /* end H5FD__log_unlock() */ diff --git a/src/H5FDlog.h b/src/H5FDlog.h index a69bb18..db51f3d 100644 --- a/src/H5FDlog.h +++ b/src/H5FDlog.h @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu> + * Programmer: Quincey Koziol * Monday, April 17, 2000 * * Purpose: The public header file for the log driver. diff --git a/src/H5FDmirror.c b/src/H5FDmirror.c new file mode 100644 index 0000000..6bee4a7 --- /dev/null +++ b/src/H5FDmirror.c @@ -0,0 +1,2017 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Transmit write-only operations to a receiver/writer process on + * a remote host. + */ + +#include "H5FDdrvr_module.h" /* This source code file is part of the H5FD driver module */ + +#include "H5private.h" /* Generic Functions */ + +#ifdef H5_HAVE_MIRROR_VFD + +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File access */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5FDmirror.h" /* "Mirror" definitions */ +#include "H5FDmirror_priv.h" /* Private header for the mirror VFD */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5Pprivate.h" /* Property lists */ + +/* The driver identification number, initialized at runtime */ +static hid_t H5FD_MIRROR_g = 0; + +/* Virtual file structure for a Mirror Driver */ +typedef struct H5FD_mirror_t { + H5FD_t pub; /* Public stuff, must be first */ + H5FD_mirror_fapl_t fa; /* Configuration structure */ + haddr_t eoa; /* End of allocated region */ + haddr_t eof; /* End of file; current file size */ + int sock_fd; /* Handle of socket to remote operator */ + H5FD_mirror_xmit_t xmit; /* Primary communication header */ + uint32_t xmit_i; /* Counter of transmission sent and rec'd */ +} H5FD_mirror_t; + +/* + * These macros check for overflow of various quantities. These macros + * assume that HDoff_t is signed and haddr_t and size_t are unsigned. + * + * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t' + * is too large to be represented by the second argument + * of the file seek function. + * + * SIZE_OVERFLOW: Checks whether a buffer size of type `hsize_t' is too + * large to be represented by the `size_t' type. + * + * REGION_OVERFLOW: Checks whether an address and size pair describe data + * which can be addressed entirely by the second + * argument of the file seek function. + */ +#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) +#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR)) + +#ifndef BSWAP_64 +#define BSWAP_64(X) \ + (uint64_t)( (((X) & 0x00000000000000FF) << 56) \ + | (((X) & 0x000000000000FF00) << 40) \ + | (((X) & 0x0000000000FF0000) << 24) \ + | (((X) & 0x00000000FF000000) << 8) \ + | (((X) & 0x000000FF00000000) >> 8) \ + | (((X) & 0x0000FF0000000000) >> 24) \ + | (((X) & 0x00FF000000000000) >> 40) \ + | (((X) & 0xFF00000000000000) >> 56)) +#endif /* BSWAP_64 */ + +/* Debugging flabs for verbose tracing -- nonzero to enable */ +#define MIRROR_DEBUG_OP_CALLS 0 +#define MIRROR_DEBUG_XMIT_BYTES 0 + +#if MIRROR_DEBUG_XMIT_BYTES +#define LOG_XMIT_BYTES(label, buf, len) do { \ + ssize_t bytes_written = 0; \ + const unsigned char *b = NULL; \ + \ + HDfprintf(stdout, "%s bytes:\n```\n", (label)); \ + \ + /* print whole lines */ \ + while ((len - bytes_written) >= 32) { \ + b = (const unsigned char *)(buf) + bytes_written; \ + HDfprintf(stdout, \ + "%04zX %02X%02X%02X%02X %02X%02X%02X%02X" \ + " %02X%02X%02X%02X %02X%02X%02X%02X" \ + " %02X%02X%02X%02X %02X%02X%02X%02X" \ + " %02X%02X%02X%02X %02X%02X%02X%02X\n", \ + bytes_written, \ + b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], \ + b[8], b[9], b[10],b[11], b[12],b[13],b[14],b[15], \ + b[16],b[17],b[18],b[19], b[20],b[21],b[22],b[23], \ + b[24],b[25],b[26],b[27], b[28],b[29],b[30],b[31]); \ + bytes_written += 32; \ + } \ + \ + /* start partial line */ \ + if (len > bytes_written) { \ + HDfprintf(stdout, "%04zX ", bytes_written); \ + } \ + \ + /* partial line blocks */ \ + while ((len - bytes_written) >= 4) { \ + HDfprintf(stdout, " %02X%02X%02X%02X", \ + (buf)[bytes_written], (buf)[bytes_written+1], \ + (buf)[bytes_written+2], (buf)[bytes_written+3]); \ + bytes_written += 4; \ + } \ + \ + /* block separator before partial block */ \ + if (len > bytes_written) { \ + HDfprintf(stdout, " "); \ + } \ + \ + /* partial block individual bytes */ \ + while (len > bytes_written) { \ + HDfprintf(stdout, "%02X", (buf)[bytes_written++]); \ + } \ + \ + /* end partial line */ \ + HDfprintf(stdout, "\n"); \ + HDfprintf(stdout, "```\n"); \ + HDfflush(stdout); \ +} while (0) +#else +#define LOG_XMIT_BYTES(label, buf, len) /* no-op */ +#endif /* MIRROR_DEBUG_XMIT_BYTE */ + +#if MIRROR_DEBUG_OP_CALLS +#define LOG_OP_CALL(name) do { \ + HDprintf("called %s()\n", (name)); \ + HDfflush(stdout); \ +} while (0) +#else +#define LOG_OP_CALL(name) /* no-op */ +#endif /* MIRROR_DEBUG_OP_CALLS */ + +/* Prototypes */ +static herr_t H5FD__mirror_term(void); +static void *H5FD__mirror_fapl_get(H5FD_t *_file); +static void *H5FD__mirror_fapl_copy(const void *_old_fa); +static herr_t H5FD__mirror_fapl_free(void *_fa); +static haddr_t H5FD__mirror_get_eoa(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD__mirror_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); +static haddr_t H5FD__mirror_get_eof(const H5FD_t *_file, H5FD_mem_t type); +static H5FD_t *H5FD__mirror_open(const char *name, unsigned flags, + hid_t fapl_id, haddr_t maxaddr); +static herr_t H5FD__mirror_close(H5FD_t *_file); +static herr_t H5FD__mirror_query(const H5FD_t *_file, unsigned long *flags); +static herr_t H5FD__mirror_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, + haddr_t addr, size_t size, const void *buf); +static herr_t H5FD__mirror_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, + haddr_t addr, size_t size, void *buf); +static herr_t H5FD__mirror_truncate(H5FD_t *_file, hid_t dxpl_id, + hbool_t closing); +static herr_t H5FD__mirror_lock(H5FD_t *_file, hbool_t rw); +static herr_t H5FD__mirror_unlock(H5FD_t *_file); + +static herr_t H5FD__mirror_verify_reply(H5FD_mirror_t *file); + +static const H5FD_class_t H5FD_mirror_g = { + "mirror", /* name */ + MAXADDR, /* maxaddr */ + H5F_CLOSE_WEAK, /* fc_degree */ + H5FD__mirror_term, /* terminate */ + NULL, /* sb_size */ + NULL, /* sb_encode */ + NULL, /* sb_decode */ + 0, /* fapl_size */ + H5FD__mirror_fapl_get, /* fapl_get */ + H5FD__mirror_fapl_copy, /* fapl_copy */ + H5FD__mirror_fapl_free, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD__mirror_open, /* open */ + H5FD__mirror_close, /* close */ + NULL, /* cmp */ + H5FD__mirror_query, /* query */ + NULL, /* get_type_map */ + NULL, /* alloc */ + NULL, /* free */ + H5FD__mirror_get_eoa, /* get_eoa */ + H5FD__mirror_set_eoa, /* set_eoa */ + H5FD__mirror_get_eof, /* get_eof */ + NULL, /* get_handle */ + H5FD__mirror_read, /* read */ + H5FD__mirror_write, /* write */ + NULL, /* flush */ + H5FD__mirror_truncate, /* truncate */ + H5FD__mirror_lock, /* lock */ + H5FD__mirror_unlock, /* unlock */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ +}; + +/* Declare a free list to manage the transmission buffers */ +H5FL_BLK_DEFINE_STATIC(xmit); + +/* Declare a free list to manage the H5FD_mirror_t struct */ +H5FL_DEFINE_STATIC(H5FD_mirror_t); + +/* Declare a free list to manage the H5FD_mirror_xmit_open_t struct */ +H5FL_DEFINE_STATIC(H5FD_mirror_xmit_open_t); + + +/*------------------------------------------------------------------------- + * Function: H5FD__init_package + * + * Purpose: Initializes any interface-specific data or routines. + * + * Return: Non-negative on success/Negative on failure + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__init_package(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + if(H5FD_mirror_init() < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize mirror VFD"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__init_package() */ + + +/* ------------------------------------------------------------------------- + * Function: H5FD_mirror_init + * + * Purpose: Initialize this driver by registering the driver with the + * library. + * + * Return: Success: The driver ID for the mirror driver. + * Failure: Negative + * ------------------------------------------------------------------------- + */ +hid_t +H5FD_mirror_init(void) +{ + hid_t ret_value = H5I_INVALID_HID; + + FUNC_ENTER_NOAPI(FAIL) + + LOG_OP_CALL(FUNC); + + if(H5I_VFL != H5I_get_type(H5FD_MIRROR_g)) + H5FD_MIRROR_g = H5FD_register(&H5FD_mirror_g, sizeof(H5FD_class_t), FALSE); + + ret_value = H5FD_MIRROR_g; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_mirror_init() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_term + * + * Purpose: Shut down the VFD + * + * Returns: SUCCEED (Can't fail) + * --------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_term(void) +{ + FUNC_ENTER_STATIC_NOERR + + /* Reset VFL ID */ + H5FD_MIRROR_g = 0; + + LOG_OP_CALL(FUNC); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD__mirror_term() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_decode_uint16 + * + * Purpose: Extract a 16-bit integer in "network" (Big-Endian) word order + * from the byte-buffer and return it with the local word order at + * the destination pointer. + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * Return: The number of bytes read from the buffer (2). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_decode_uint16(uint16_t *out, const unsigned char *_buf) +{ + uint16_t n = 0; + + LOG_OP_CALL(__func__); + + HDassert(_buf && out); + + HDmemcpy(&n, _buf, sizeof(n)); + *out = (uint16_t)HDntohs(n); + + return 2; /* number of bytes eaten */ +} /* end H5FD__mirror_xmit_decode_uint16() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_decode_uint32 + * + * Purpose: Extract a 32-bit integer in "network" (Big-Endian) word order + * from the byte-buffer and return it with the local word order at + * the destination pointer. + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * Return: The number of bytes read from the buffer (4). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_decode_uint32(uint32_t *out, const unsigned char *_buf) +{ + uint32_t n = 0; + + LOG_OP_CALL(__func__); + + HDassert(_buf && out); + + HDmemcpy(&n, _buf, sizeof(n)); + *out = (uint32_t)HDntohl(n); + + return 4; /* number of bytes eaten */ +} /* end H5FD__mirror_xmit_decode_uint32() */ + + +/* --------------------------------------------------------------------------- + * Function: is_host_little_endian + * + * Purpose: Determine whether the host machine is is little-endian. + * + * Store an intger with a known value, re-map the memory to a + * character array, and inspect the array's contents. + * + * Return: The number of bytes written to the buffer (8). + * + * Programmer: Jacob Smith + * 2020-03-05 + * --------------------------------------------------------------------------- + */ +static hbool_t +is_host_little_endian(void) +{ + union { + uint32_t u32; + uint8_t u8[4]; + } echeck; + echeck.u32 = 0xA1B2C3D4; + + if(echeck.u8[0] == 0xD4) + return TRUE; + else + return FALSE; +} /* end is_host_little_endian() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_decode_uint64 + * + * Purpose: Extract a 64-bit integer in "network" (Big-Endian) word order + * from the byte-buffer and return it with the local word order. + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * WARNING: Does not accommodate other forms of endianness, + * e.g. "middle-endian". + * + * Return: The number of bytes written to the buffer (8). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_decode_uint64(uint64_t *out, const unsigned char *_buf) +{ + uint64_t n = 0; + + LOG_OP_CALL(__func__); + + HDassert(_buf && out); + + HDmemcpy(&n, _buf, sizeof(n)); + if(TRUE == is_host_little_endian()) + *out = BSWAP_64(n); + else + *out = n; + + return 8; +} /* end H5FD__mirror_xmit_decode_uint64() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_decode_uint8 + * + * Purpose: Extract a 8-bit integer in "network" (Big-Endian) word order + * from the byte-buffer and return it with the local word order at + * the destination pointer. + * (yes, it's one byte). + * + * Return: The number of bytes read from the buffer (1). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_decode_uint8(uint8_t *out, const unsigned char *_buf) +{ + LOG_OP_CALL(__func__); + + HDassert(_buf && out); + + HDmemcpy(out, _buf, sizeof(uint8_t)); + + return 1; /* number of bytes eaten */ +} /* end H5FD__mirror_xmit_decode_uint8() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_encode_uint16 + * + * Purpose: Encode a 16-bit integer in "network" (Big-Endian) word order + * in place in the destination bytes-buffer. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer (2). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_encode_uint16(unsigned char *_dest, uint16_t v) +{ + uint16_t n = 0; + + LOG_OP_CALL(__func__); + + HDassert(_dest); + + n = (uint16_t)HDhtons(v); + HDmemcpy(_dest, &n, sizeof(n)); + + return 2; +} /* end H5FD__mirror_xmit_encode_uint16() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_encode_uint32 + * + * Purpose: Encode a 32-bit integer in "network" (Big-Endian) word order + * in place in the destination bytes-buffer. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer (4). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_encode_uint32(unsigned char *_dest, uint32_t v) +{ + uint32_t n = 0; + + LOG_OP_CALL(__func__); + + HDassert(_dest); + + n = (uint32_t)HDhtonl(v); + HDmemcpy(_dest, &n, sizeof(n)); + + return 4; +} /* end H5FD__mirror_xmit_encode_uint32() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_encode_uint64 + * + * Purpose: Encode a 64-bit integer in "network" (Big-Endian) word order + * in place in the destination bytes-buffer. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer (8). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_encode_uint64(unsigned char *_dest, uint64_t v) +{ + uint64_t n = v; + + LOG_OP_CALL(__func__); + + HDassert(_dest); + + if(TRUE == is_host_little_endian()) + n = BSWAP_64(v); + HDmemcpy(_dest, &n, sizeof(n)); + + return 8; +} /* H5FD__mirror_xmit_encode_uint64() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_encode_uint8 + * + * Purpose: Encode a 8-bit integer in "network" (Big-Endian) word order + * in place in the destination bytes-buffer. + * (yes, it's one byte). + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes read from the buffer (1). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_encode_uint8(unsigned char *dest, uint8_t v) +{ + LOG_OP_CALL(__func__); + + HDassert(dest); + + HDmemcpy(dest, &v, sizeof(v)); + + return 1; +} /* end H5FD__mirror_xmit_encode_uint8() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_decode_header + * + * Purpose: Extract a mirror_xmit_t "header" from the bytes-buffer. + * + * Fields will be lifted from the buffer and stored in the + * target structure, using in the correct location (different + * systems may insert different padding between components) and + * word order (Big- vs Little-Endian). + * + * The resulting structure should be sanity-checked with + * H5FD_mirror_xmit_is_xmit() before use. + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * Return: The number of bytes consumed from the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_decode_header(H5FD_mirror_xmit_t *out, + const unsigned char *buf) +{ + size_t n_eaten = 0; + + LOG_OP_CALL(__func__); + + HDassert(out && buf); + + n_eaten += H5FD__mirror_xmit_decode_uint32(&(out->magic), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint8(&(out->version), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint32(&(out->session_token), + &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint32(&(out->xmit_count), + &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint8(&(out->op), &buf[n_eaten]); + HDassert(n_eaten == H5FD_MIRROR_XMIT_HEADER_SIZE); + + return n_eaten; +} /* end H5FD_mirror_xmit_decode_header() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_decode_lock + * + * Purpose: Extract a mirror_xmit_lock_t from the bytes-buffer. + * + * Fields will be lifted from the buffer and stored in the + * target structure, using in the correct location (different + * systems may insert different padding between components) and + * word order (Big- vs Little-Endian). + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * The resulting structure should be sanity-checked with + * H5FD_mirror_xmit_is_lock() before use. + * + * Return: The number of bytes consumed from the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_decode_lock(H5FD_mirror_xmit_lock_t *out, + const unsigned char *buf) +{ + size_t n_eaten = 0; + + LOG_OP_CALL(__func__); + + HDassert(out && buf); + + n_eaten += H5FD_mirror_xmit_decode_header(&(out->pub), buf); + n_eaten += H5FD__mirror_xmit_decode_uint64(&(out->rw), &buf[n_eaten]); + HDassert(n_eaten == H5FD_MIRROR_XMIT_LOCK_SIZE); + + return n_eaten; +} /* end H5FD_mirror_xmit_decode_lock() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_decode_open + * + * Purpose: Extract a mirror_xmit_open_t from the bytes-buffer. + * + * Fields will be lifted from the buffer and stored in the + * target structure, using in the correct location (different + * systems may insert different padding between components) and + * word order (Big- vs Little-Endian). + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * The resulting structure should be sanity-checked with + * H5FD_mirror_xmit_is_open() before use. + * + * Return: The maximum number of bytes that this decoding operation might + * have consumed from the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_decode_open(H5FD_mirror_xmit_open_t *out, + const unsigned char *buf) +{ + size_t n_eaten = 0; + + LOG_OP_CALL(__func__); + + HDassert(out && buf); + + n_eaten += H5FD_mirror_xmit_decode_header(&(out->pub), buf); + n_eaten += H5FD__mirror_xmit_decode_uint32(&(out->flags), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint64(&(out->maxaddr), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint64(&(out->size_t_blob), + &buf[n_eaten]); + HDassert((H5FD_MIRROR_XMIT_OPEN_SIZE - H5FD_MIRROR_XMIT_FILEPATH_MAX) + == n_eaten); + HDstrncpy(out->filename, (const char *)&buf[n_eaten], + H5FD_MIRROR_XMIT_FILEPATH_MAX - 1); + out->filename[H5FD_MIRROR_XMIT_FILEPATH_MAX - 1] = 0; /* force final NULL */ + + return H5FD_MIRROR_XMIT_OPEN_SIZE; +} /* end H5FD_mirror_xmit_decode_open() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_decode_reply + * + * Purpose: Extract a mirror_xmit_reply_t from the bytes-buffer. + * + * Fields will be lifted from the buffer and stored in the + * target structure, using in the correct location (different + * systems may insert different padding between components) and + * word order (Big- vs Little-Endian). + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * The resulting structure should be sanity-checked with + * H5FD_mirror_xmit_is_reply() before use. + * + * Return: The maximum number of bytes that this decoding operation might + * have consumed from the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_decode_reply(H5FD_mirror_xmit_reply_t *out, + const unsigned char *buf) +{ + size_t n_eaten = 0; + + LOG_OP_CALL(__func__); + + HDassert(out && buf); + + n_eaten += H5FD_mirror_xmit_decode_header(&(out->pub), buf); + n_eaten += H5FD__mirror_xmit_decode_uint32(&(out->status), &buf[n_eaten]); + HDassert((H5FD_MIRROR_XMIT_REPLY_SIZE - H5FD_MIRROR_STATUS_MESSAGE_MAX) + == n_eaten); + HDstrncpy(out->message, (const char *)&buf[n_eaten], + H5FD_MIRROR_STATUS_MESSAGE_MAX - 1); + out->message[H5FD_MIRROR_STATUS_MESSAGE_MAX - 1] = 0; /* force NULL term */ + + return H5FD_MIRROR_XMIT_REPLY_SIZE; +} /* end H5FD_mirror_xmit_decode_reply() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_decode_set_eoa + * + * Purpose: Extract a mirror_xmit_eoa_t from the bytes-buffer. + * + * Fields will be lifted from the buffer and stored in the + * target structure, using in the correct location (different + * systems may insert different padding between components) and + * word order (Big- vs Little-Endian). + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * The resulting structure should be sanity-checked with + * H5FD_mirror_xmit_is_set_eoa() before use. + * + * Return: The number of bytes consumed from the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_decode_set_eoa(H5FD_mirror_xmit_eoa_t *out, + const unsigned char *buf) +{ + size_t n_eaten = 0; + + LOG_OP_CALL(__func__); + + HDassert(out && buf); + + n_eaten += H5FD_mirror_xmit_decode_header(&(out->pub), buf); + n_eaten += H5FD__mirror_xmit_decode_uint8(&(out->type), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint64(&(out->eoa_addr), &buf[n_eaten]); + HDassert(n_eaten == H5FD_MIRROR_XMIT_EOA_SIZE); + + return n_eaten; +} /* end H5FD_mirror_xmit_decode_set_eoa() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_decode_write + * + * Purpose: Extract a mirror_xmit_write_t from the bytes-buffer. + * + * Fields will be lifted from the buffer and stored in the + * target structure, using in the correct location (different + * systems may insert different padding between components) and + * word order (Big- vs Little-Endian). + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * The resulting structure should be sanity-checked with + * H5FD_mirror_xmit_is_write() before use. + * + * Return: The number of bytes consumed from the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_decode_write(H5FD_mirror_xmit_write_t *out, + const unsigned char *buf) +{ + size_t n_eaten = 0; + + LOG_OP_CALL(__func__); + + HDassert(out && buf); + + n_eaten += H5FD_mirror_xmit_decode_header(&(out->pub), buf); + n_eaten += H5FD__mirror_xmit_decode_uint8(&(out->type), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint64(&(out->offset), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint64(&(out->size), &buf[n_eaten]); + HDassert(n_eaten == H5FD_MIRROR_XMIT_WRITE_SIZE); + + return n_eaten; +} /* end H5FD_mirror_xmit_decode_write() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_encode_header + * + * Purpose: Encode a mirror_xmit_t "header" to the bytes-buffer. + * + * Fields will be packed into the buffer in a predictable manner, + * any numbers stored in "network" (Big-Endian) word order. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_encode_header(unsigned char *dest, + const H5FD_mirror_xmit_t *x) +{ + size_t n_writ = 0; + + LOG_OP_CALL(__func__); + + HDassert(dest && x); + + n_writ += H5FD__mirror_xmit_encode_uint32((dest+n_writ), x->magic); + n_writ += H5FD__mirror_xmit_encode_uint8((dest+n_writ), x->version); + n_writ += H5FD__mirror_xmit_encode_uint32((dest+n_writ), x->session_token); + n_writ += H5FD__mirror_xmit_encode_uint32((dest+n_writ), x->xmit_count); + n_writ += H5FD__mirror_xmit_encode_uint8((dest+n_writ), x->op); + HDassert(n_writ == H5FD_MIRROR_XMIT_HEADER_SIZE); + + return n_writ; +} /* end H5FD_mirror_xmit_encode_header() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_encode_lock + * + * Purpose: Encode a mirror_xmit_lock_t to the bytes-buffer. + * Fields will be packed into the buffer in a predictable manner, + * any numbers stored in "network" (Big-Endian) word order. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_encode_lock(unsigned char *dest, + const H5FD_mirror_xmit_lock_t *x) +{ + size_t n_writ = 0; + + LOG_OP_CALL(__func__); + + HDassert(dest && x); + + n_writ += H5FD_mirror_xmit_encode_header(dest, + (const H5FD_mirror_xmit_t *)&(x->pub)); + n_writ += H5FD__mirror_xmit_encode_uint64(&dest[n_writ], x->rw); + HDassert(n_writ == H5FD_MIRROR_XMIT_LOCK_SIZE); + + return n_writ; +} /* end H5FD_mirror_xmit_encode_lock() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_encode_open + * + * Purpose: Encode a mirror_xmit_open_t to the bytes-buffer. + * Fields will be packed into the buffer in a predictable manner, + * any numbers stored in "network" (Big-Endian) word order. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The maximum number of bytes that this decoding operation might + * have written into the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_encode_open(unsigned char *dest, + const H5FD_mirror_xmit_open_t *x) +{ + size_t n_writ = 0; + + LOG_OP_CALL(__func__); + + HDassert(dest && x); + + /* clear entire structure, but especially its filepath string area */ + HDmemset(dest, 0, H5FD_MIRROR_XMIT_OPEN_SIZE); + + n_writ += H5FD_mirror_xmit_encode_header(dest, + (const H5FD_mirror_xmit_t *)&(x->pub)); + n_writ += H5FD__mirror_xmit_encode_uint32(&dest[n_writ], x->flags); + n_writ += H5FD__mirror_xmit_encode_uint64(&dest[n_writ], x->maxaddr); + n_writ += H5FD__mirror_xmit_encode_uint64(&dest[n_writ], x->size_t_blob); + HDassert((H5FD_MIRROR_XMIT_OPEN_SIZE - H5FD_MIRROR_XMIT_FILEPATH_MAX) + == n_writ); + HDstrncpy((char *)&dest[n_writ], x->filename, H5FD_MIRROR_XMIT_FILEPATH_MAX); + + return H5FD_MIRROR_XMIT_OPEN_SIZE; +} /* end H5FD_mirror_xmit_encode_open() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_encode_reply + * + * Purpose: Encode a mirror_xmit_reply_t to the bytes-buffer. + * + * Fields will be packed into the buffer in a predictable manner, + * any numbers stored in "network" (Big-Endian) word order. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The maximum number of bytes that this decoding operation might + * have written into the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_encode_reply(unsigned char *dest, + const H5FD_mirror_xmit_reply_t *x) +{ + size_t n_writ = 0; + + LOG_OP_CALL(__func__); + + HDassert(dest && x); + + /* clear entire structure, but especially its message string area */ + HDmemset(dest, 0, H5FD_MIRROR_XMIT_REPLY_SIZE); + + n_writ += H5FD_mirror_xmit_encode_header(dest, + (const H5FD_mirror_xmit_t *)&(x->pub)); + n_writ += H5FD__mirror_xmit_encode_uint32(&dest[n_writ], x->status); + HDassert((H5FD_MIRROR_XMIT_REPLY_SIZE - H5FD_MIRROR_STATUS_MESSAGE_MAX) + == n_writ); + HDstrncpy((char *)&dest[n_writ], x->message, H5FD_MIRROR_STATUS_MESSAGE_MAX); + + return H5FD_MIRROR_XMIT_REPLY_SIZE; +} /* end H5FD_mirror_xmit_encode_reply() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_encode_set_eoa + * + * Purpose: Encode a mirror_xmit_eoa_t to the bytes-buffer. + * + * Fields will be packed into the buffer in a predictable manner, + * any numbers stored in "network" (Big-Endian) word order. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_encode_set_eoa(unsigned char *dest, + const H5FD_mirror_xmit_eoa_t *x) +{ + size_t n_writ = 0; + + LOG_OP_CALL(__func__); + + HDassert(dest && x); + + n_writ += H5FD_mirror_xmit_encode_header(dest, + (const H5FD_mirror_xmit_t *)&(x->pub)); + n_writ += H5FD__mirror_xmit_encode_uint8(&dest[n_writ], x->type); + n_writ += H5FD__mirror_xmit_encode_uint64(&dest[n_writ], x->eoa_addr); + HDassert(n_writ == H5FD_MIRROR_XMIT_EOA_SIZE); + + return n_writ; +} /* end H5FD_mirror_xmit_encode_set_eoa() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_encode_write + * + * Purpose: Encode a mirror_xmit_write_t to the bytes-buffer. + * + * Fields will be packed into the buffer in a predictable manner, + * any numbers stored in "network" (Big-Endian) word order. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_encode_write(unsigned char *dest, + const H5FD_mirror_xmit_write_t *x) +{ + size_t n_writ = 0; + + LOG_OP_CALL(__func__); + + HDassert(dest && x); + + n_writ += H5FD_mirror_xmit_encode_header(dest, + (const H5FD_mirror_xmit_t *)&(x->pub)); + n_writ += H5FD__mirror_xmit_encode_uint8(&dest[n_writ], x->type); + n_writ += H5FD__mirror_xmit_encode_uint64(&dest[n_writ], x->offset); + n_writ += H5FD__mirror_xmit_encode_uint64(&dest[n_writ], x->size); + HDassert(n_writ == H5FD_MIRROR_XMIT_WRITE_SIZE); + + return n_writ; +} /* end H5FD_mirror_xmit_encode_write() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_close + * + * Purpose: Verify that a mirror_xmit_t is a valid CLOSE xmit. + * + * Checks header validity and op code. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +H5_ATTR_PURE hbool_t +H5FD_mirror_xmit_is_close(const H5FD_mirror_xmit_t *xmit) +{ + LOG_OP_CALL(__func__); + + HDassert(xmit); + + if((TRUE == H5FD_mirror_xmit_is_xmit(xmit)) && (H5FD_MIRROR_OP_CLOSE == xmit->op)) + return TRUE; + + return FALSE; +} /* end H5FD_mirror_xmit_is_close() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_lock + * + * Purpose: Verify that a mirror_xmit_lock_t is a valid LOCK xmit. + * + * Checks header validity and op code. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +H5_ATTR_PURE hbool_t +H5FD_mirror_xmit_is_lock(const H5FD_mirror_xmit_lock_t *xmit) +{ + LOG_OP_CALL(__func__); + + HDassert(xmit); + + if((TRUE == H5FD_mirror_xmit_is_xmit(&(xmit->pub))) && (H5FD_MIRROR_OP_LOCK == xmit->pub.op)) + return TRUE; + + return FALSE; +} /* end H5FD_mirror_xmit_is_lock() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_open + * + * Purpose: Verify that a mirror_xmit_open_t is a valid OPEN xmit. + * + * Checks header validity and op code. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +H5_ATTR_PURE hbool_t +H5FD_mirror_xmit_is_open(const H5FD_mirror_xmit_open_t *xmit) +{ + LOG_OP_CALL(__func__); + + HDassert(xmit); + + if((TRUE == H5FD_mirror_xmit_is_xmit(&(xmit->pub))) && (H5FD_MIRROR_OP_OPEN == xmit->pub.op)) + + return TRUE; + + return FALSE; +} /* end H5FD_mirror_xmit_is_open() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_eoa + * + * Purpose: Verify that a mirror_xmit_eoa_t is a valid SET-EOA xmit. + * + * Checks header validity and op code. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +H5_ATTR_PURE hbool_t +H5FD_mirror_xmit_is_set_eoa(const H5FD_mirror_xmit_eoa_t *xmit) +{ + LOG_OP_CALL(__func__); + + HDassert(xmit); + + if((TRUE == H5FD_mirror_xmit_is_xmit(&(xmit->pub))) && (H5FD_MIRROR_OP_SET_EOA == xmit->pub.op)) + return TRUE; + + return FALSE; +} /* end H5FD_mirror_xmit_is_eoa() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_reply + * + * Purpose: Verify that a mirror_xmit_reply_t is a valid REPLY xmit. + * + * Checks header validity and op code. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +H5_ATTR_PURE hbool_t +H5FD_mirror_xmit_is_reply(const H5FD_mirror_xmit_reply_t *xmit) +{ + LOG_OP_CALL(__func__); + + HDassert(xmit); + + if((TRUE == H5FD_mirror_xmit_is_xmit(&(xmit->pub))) && (H5FD_MIRROR_OP_REPLY == xmit->pub.op)) + return TRUE; + + return FALSE; +} /* end H5FD_mirror_xmit_is_reply() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_write + * + * Purpose: Verify that a mirror_xmit_write_t is a valid WRITE xmit. + * + * Checks header validity and op code. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +H5_ATTR_PURE hbool_t +H5FD_mirror_xmit_is_write(const H5FD_mirror_xmit_write_t *xmit) +{ + LOG_OP_CALL(__func__); + + HDassert(xmit); + + if((TRUE == H5FD_mirror_xmit_is_xmit(&(xmit->pub))) && (H5FD_MIRROR_OP_WRITE == xmit->pub.op)) + return TRUE; + + return FALSE; +} /* end H5FD_mirror_xmit_is_write() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_xmit + * + * Purpose: Verify that a mirror_xmit_t is well-formed. + * + * Checks magic number and structure version. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +H5_ATTR_PURE hbool_t +H5FD_mirror_xmit_is_xmit(const H5FD_mirror_xmit_t *xmit) +{ + LOG_OP_CALL(__func__); + + HDassert(xmit); + + if((H5FD_MIRROR_XMIT_MAGIC != xmit->magic) || (H5FD_MIRROR_XMIT_CURR_VERSION != xmit->version)) + return FALSE; + + return TRUE; +} /* end H5FD_mirror_xmit_is_xmit() */ + + +/* ---------------------------------------------------------------------------- + * Function: H5FD__mirror_verify_reply + * + * Purpose: Wait for and read reply data from remote processes. + * Sanity-check that a reply is well-formed and valid. + * If all checks pass, inspect the reply contents and handle + * reported error, if not an OK reply. + * + * Return: SUCCEED if ok, else FAIL. + * ---------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_verify_reply(H5FD_mirror_t *file) +{ + unsigned char *xmit_buf = NULL; + struct H5FD_mirror_xmit_reply_t reply; + ssize_t read_ret = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + HDassert(file && file->sock_fd); + + xmit_buf = H5FL_BLK_MALLOC(xmit, H5FD_MIRROR_XMIT_BUFFER_MAX); + if(NULL == xmit_buf) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate xmit buffer"); + + read_ret = HDread(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_REPLY_SIZE); + if(read_ret < 0) + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to read reply"); + if(read_ret != H5FD_MIRROR_XMIT_REPLY_SIZE) + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unexpected read size"); + + LOG_XMIT_BYTES("reply", xmit_buf, read_ret); + + if(H5FD_mirror_xmit_decode_reply(&reply, xmit_buf) != H5FD_MIRROR_XMIT_REPLY_SIZE) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to decode reply xmit"); + + if(H5FD_mirror_xmit_is_reply(&reply) != TRUE) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "xmit op code was not REPLY"); + + if(reply.pub.session_token != file->xmit.session_token) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "wrong session"); + if(reply.pub.xmit_count != (file->xmit_i)++) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "xmit out of sync"); + if(reply.status != H5FD_MIRROR_STATUS_OK) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "%s", (const char *)(reply.message)); + +done: + if(xmit_buf) + xmit_buf = H5FL_BLK_FREE(xmit, xmit_buf); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__mirror_verify_reply() */ + + +/* ------------------------------------------------------------------------- + * Function: H5FD__mirror_fapl_get + * + * Purpose: Get the file access propety list which could be used to create + * an identical file. + * + * Return: Success: pointer to the new file access property list value. + * Failure: NULL + * ------------------------------------------------------------------------- + */ +static void * +H5FD__mirror_fapl_get(H5FD_t *_file) +{ + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + H5FD_mirror_fapl_t *fa = NULL; + void *ret_value = NULL; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + fa = (H5FD_mirror_fapl_t *)H5MM_calloc(sizeof(H5FD_mirror_fapl_t)); + if(NULL == fa) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "calloc failed"); + + HDmemcpy(fa, &(file->fa), sizeof(H5FD_mirror_fapl_t)); + + ret_value = fa; + +done: + if(ret_value == NULL) + if(fa != NULL) + H5MM_xfree(fa); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__mirror_fapl_get() */ + + +/* ------------------------------------------------------------------------- + * Function: H5FD__mirror_fapl_copy + * + * Purpose: Copies the mirror vfd-specific file access properties. + * + * Return: Success: Pointer to a new property list + * Failure: NULL + * ------------------------------------------------------------------------- + */ +static void * +H5FD__mirror_fapl_copy(const void *_old_fa) +{ + const H5FD_mirror_fapl_t *old_fa = (const H5FD_mirror_fapl_t *)_old_fa; + H5FD_mirror_fapl_t *new_fa = NULL; + void *ret_value = NULL; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + new_fa = (H5FD_mirror_fapl_t *)H5MM_malloc(sizeof(H5FD_mirror_fapl_t)); + if(new_fa == NULL) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "memory allocation failed"); + + HDmemcpy(new_fa, old_fa, sizeof(H5FD_mirror_fapl_t)); + ret_value = new_fa; + +done: + if(ret_value == NULL) + if(new_fa != NULL) + H5MM_xfree(new_fa); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__mirror_fapl_copy() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_fapl_free + * + * Purpose: Frees the mirror VFD-specific file access properties. + * + * Return: SUCCEED (cannot fail) + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_fapl_free(void *_fa) +{ + H5FD_mirror_fapl_t *fa = (H5FD_mirror_fapl_t*)_fa; + + FUNC_ENTER_STATIC_NOERR + + LOG_OP_CALL(FUNC); + + /* sanity check */ + HDassert(fa != NULL); + HDassert(fa->magic == H5FD_MIRROR_FAPL_MAGIC); + + fa->magic += 1; /* invalidate */ + H5MM_xfree(fa); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD__mirror_fapl_free() */ + + +/* ------------------------------------------------------------------------- + * Function: H5Pget_fapl_mirror + * + * Purpose: Get the configuration information for this fapl. + * Data is memcopied into the fa_out pointer. + * + * Return: SUCCEED/FAIL + * ------------------------------------------------------------------------- + */ +herr_t +H5Pget_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa_out) +{ + const H5FD_mirror_fapl_t *fa = NULL; + H5P_genplist_t *plist = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*x", fapl_id, fa_out); + + LOG_OP_CALL(FUNC); + + if(NULL == fa_out) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fa_out is NULL"); + + plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); + if(NULL == plist) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list"); + if(H5P_peek_driver(plist) != H5FD_MIRROR) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver"); + + fa = (const H5FD_mirror_fapl_t *)H5P_peek_driver_info(plist); + if(NULL == fa) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info"); + + HDassert(fa->magic == H5FD_MIRROR_FAPL_MAGIC); /* sanity check */ + + HDmemcpy(fa_out, fa, sizeof(H5FD_mirror_fapl_t)); + +done: + FUNC_LEAVE_API(ret_value); +} /* end H5Pget_fapl_mirror() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_fapl_mirror + * + * Purpose: Modify the file access property list to use the mirror + * driver (H5FD_MIRROR) defined in this source file. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa) +{ + H5P_genplist_t *plist = NULL; + herr_t ret_value = FAIL; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*x", fapl_id, fa); + + LOG_OP_CALL(FUNC); + + plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); + if(NULL == plist) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list"); + if(NULL == fa) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null fapl_t pointer"); + if(H5FD_MIRROR_FAPL_MAGIC != fa->magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid fapl_t magic"); + if(H5FD_MIRROR_CURR_FAPL_T_VERSION != fa->version) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unknown fapl_t version"); + + ret_value = H5P_set_driver(plist, H5FD_MIRROR, (const void *)fa); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_fapl_mirror() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_open + * + * Purpose: Create and/or opens a file as an HDF5 file. + * + * Initiate connection with remote Server/Writer. + * If successful, the remote file is open. + * + * Return: Success: A pointer to a new file data structure. The + * public fields will be initialized by the + * caller, which is always H5FD_open(). + * Failure: NULL + *------------------------------------------------------------------------- + */ +static H5FD_t * +H5FD__mirror_open(const char *name, + unsigned flags, + hid_t fapl_id, + haddr_t maxaddr) +{ + int live_socket = -1; + struct sockaddr_in target_addr; + socklen_t addr_size; + unsigned char *xmit_buf = NULL; + H5FD_mirror_fapl_t fa; + H5FD_mirror_t *file = NULL; + H5FD_mirror_xmit_open_t *open_xmit = NULL; + H5FD_t *ret_value = NULL; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + /* --------------- */ + /* Check arguments */ + /* --------------- */ + + if(!name || !*name) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name"); + if(HDstrlen(name) >= H5FD_MIRROR_XMIT_FILEPATH_MAX) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "filename is too long"); + if(0 == maxaddr || HADDR_UNDEF == maxaddr) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr"); + if(ADDR_OVERFLOW(maxaddr)) + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr"); + + if(H5Pget_fapl_mirror(fapl_id, &fa) == FAIL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "can't get config info"); + if(H5FD_MIRROR_FAPL_MAGIC != fa.magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid fapl magic"); + if(H5FD_MIRROR_CURR_FAPL_T_VERSION != fa.version) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid fapl version"); + + /* --------------------- */ + /* Handshake with remote */ + /* --------------------- */ + + live_socket = HDsocket(AF_INET, SOCK_STREAM, 0); + if(live_socket < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "can't create socket"); + + target_addr.sin_family = AF_INET; + target_addr.sin_port = HDhtons((uint16_t)fa.handshake_port); + target_addr.sin_addr.s_addr = HDinet_addr(fa.remote_ip); + HDmemset(target_addr.sin_zero, '\0', sizeof target_addr.sin_zero); + + addr_size = sizeof(target_addr); + if(HDconnect(live_socket, (struct sockaddr *)&target_addr, addr_size) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "can't connect to remote server"); + + /* ------------- */ + /* Open the file */ + /* ------------- */ + + file = (H5FD_mirror_t *)H5FL_CALLOC(H5FD_mirror_t); + if(NULL == file) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate file struct"); + + file->sock_fd = live_socket; + file->xmit_i = 0; + + file->xmit.magic = H5FD_MIRROR_XMIT_MAGIC; + file->xmit.version = H5FD_MIRROR_XMIT_CURR_VERSION; + file->xmit.xmit_count = file->xmit_i++; + file->xmit.session_token = (uint32_t)(0x01020304 ^ file->sock_fd); /* TODO: hashing? */ + /* int --> uint32_t may truncate on some systems... shouldn't matter? */ + + open_xmit = (H5FD_mirror_xmit_open_t *)H5FL_CALLOC(H5FD_mirror_xmit_open_t); + if(NULL == open_xmit) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate open_xmit struct"); + + file->xmit.op = H5FD_MIRROR_OP_OPEN; + open_xmit->pub = file->xmit; + open_xmit->flags = (uint32_t)flags; + open_xmit->maxaddr = (uint64_t)maxaddr; + open_xmit->size_t_blob = (uint64_t)((size_t)(-1)); + HDsnprintf(open_xmit->filename, H5FD_MIRROR_XMIT_FILEPATH_MAX-1, "%s", name); + + xmit_buf = H5FL_BLK_MALLOC(xmit, H5FD_MIRROR_XMIT_BUFFER_MAX); + if(NULL == xmit_buf) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate xmit buffer"); + + if(H5FD_mirror_xmit_encode_open(xmit_buf, open_xmit) != H5FD_MIRROR_XMIT_OPEN_SIZE) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, NULL, "unable to encode open"); + + LOG_XMIT_BYTES("open", xmit_buf, H5FD_MIRROR_XMIT_OPEN_SIZE); + + if(HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_OPEN_SIZE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, NULL, "unable to transmit open"); + + if(H5FD__mirror_verify_reply(file) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "invalid reply"); + + ret_value = (H5FD_t *)file; + +done: + if(NULL == ret_value) { + if(file) + file = H5FL_FREE(H5FD_mirror_t, file); + if(live_socket >= 0 && HDclose(live_socket) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "can't close socket"); + } + + if(open_xmit) + open_xmit = H5FL_FREE(H5FD_mirror_xmit_open_t, open_xmit); + if(xmit_buf) + xmit_buf = H5FL_BLK_FREE(xmit, xmit_buf); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__mirror_open() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_close + * + * Purpose: Closes the HDF5 file. + * + * Tries to send a CLOSE op to the remote Writer and expects + * a valid reply, then closes the socket. + * In error, attempts to send a deliberately invalid xmit to the + * Writer to get it to close/abort, then attempts to close the + * socket. + * + * Return: Success: SUCCEED + * Failure: FAIL, file possibly not closed but resources freed. + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_close(H5FD_t *_file) +{ + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + unsigned char *xmit_buf = NULL; + int xmit_encoded = 0; /* monitor point of failure */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + /* Sanity check */ + HDassert(file); + HDassert(file->sock_fd >= 0); + + file->xmit.xmit_count = (file->xmit_i)++; + file->xmit.op = H5FD_MIRROR_OP_CLOSE; + + xmit_buf = H5FL_BLK_MALLOC(xmit, H5FD_MIRROR_XMIT_BUFFER_MAX); + if(NULL == xmit_buf) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate xmit buffer"); + + if(H5FD_mirror_xmit_encode_header(xmit_buf, &(file->xmit)) != H5FD_MIRROR_XMIT_HEADER_SIZE) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to encode close"); + xmit_encoded = 1; + + LOG_XMIT_BYTES("close", xmit_buf, H5FD_MIRROR_XMIT_HEADER_SIZE); + + if(HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_HEADER_SIZE) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to transmit close"); + + if(H5FD__mirror_verify_reply(file) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + + if(HDclose(file->sock_fd) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "can't close socket"); + +done: + if(ret_value == FAIL) { + if(xmit_encoded == 0) { + /* Encode failed; send GOODBYE to force writer halt. + * We can ignore any response from the writer, if we receive + * any reply at all. + */ + if(HDwrite(file->sock_fd, "GOODBYE", HDstrlen("GOODBYE")) < 0) { + HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to transmit close"); + if(HDclose(file->sock_fd) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "can't close socket"); + file->sock_fd = -1; /* invalidate for later */ + } /* end if problem writing goodbye; go down hard */ + else + if(HDshutdown(file->sock_fd, SHUT_WR) < 0) + HDONE_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't shutdown socket write: %s", HDstrerror(errno)); + } /* end if xmit encode failed */ + + if(file->sock_fd >= 0) + if(HDclose(file->sock_fd) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "can't close socket"); + } /* end if error */ + + file = H5FL_FREE(H5FD_mirror_t, file); /* always release resources */ + + if(xmit_buf) + xmit_buf = H5FL_BLK_FREE(xmit, xmit_buf); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__mirror_close() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_query + * + * Purpose: Get the driver feature flags implemented by the driver. + * + * Return: SUCCEED (non-negative) (can't fail) + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_query(const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags) +{ + FUNC_ENTER_STATIC_NOERR; + + LOG_OP_CALL(FUNC); + + /* Notice: the Mirror VFD Writer currently uses only the Sec2 driver as + * the underying driver -- as such, the Mirror VFD implementation copies + * the Sec2 feature flags as its own. + * + * File pointer is always NULL/unused -- the H5FD_FEAT_IGNORE_DRVRINFO flag + * is never included. + * -- JOS 2020-01-13 + */ + if(flags) + *flags = H5FD_FEAT_AGGREGATE_METADATA \ + | H5FD_FEAT_ACCUMULATE_METADATA \ + | H5FD_FEAT_DATA_SIEVE \ + | H5FD_FEAT_AGGREGATE_SMALLDATA \ + | H5FD_FEAT_POSIX_COMPAT_HANDLE \ + | H5FD_FEAT_SUPPORTS_SWMR_IO \ + | H5FD_FEAT_DEFAULT_VFD_COMPATIBLE; + + FUNC_LEAVE_NOAPI(SUCCEED); +} /* end H5FD__mirror_query() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_get_eoa + * + * Purpose: Gets the end-of-address marker for the file. The EOA marker + * is the first address past the last byte allocated in the + * format address space. + * + * Required to register the driver. + * + * Return: The end-of-address marker. + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD__mirror_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_mirror_t *file = (const H5FD_mirror_t *)_file; + + FUNC_ENTER_STATIC_NOERR + + LOG_OP_CALL(FUNC); + + HDassert(file); + + FUNC_LEAVE_NOAPI(file->eoa) +} /* end H5FD__mirror_get_eoa() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_set_eoa + * + * Purpose: Set the end-of-address marker for the file. This function is + * called shortly after an existing HDF5 file is opened in order + * to tell the driver where the end of the HDF5 data is located. + * + * Return: SUCCEED / FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr) +{ + H5FD_mirror_xmit_eoa_t xmit_eoa; + unsigned char *xmit_buf = NULL; + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + HDassert(file); + + file->eoa = addr; /* local copy */ + + file->xmit.xmit_count = (file->xmit_i)++; + file->xmit.op = H5FD_MIRROR_OP_SET_EOA; + + xmit_eoa.pub = file->xmit; + xmit_eoa.type = (uint8_t)type; + xmit_eoa.eoa_addr = (uint64_t)addr; + + xmit_buf = H5FL_BLK_MALLOC(xmit, H5FD_MIRROR_XMIT_BUFFER_MAX); + if(NULL == xmit_buf) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate xmit buffer"); + + if(H5FD_mirror_xmit_encode_set_eoa(xmit_buf, &xmit_eoa) != H5FD_MIRROR_XMIT_EOA_SIZE) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to encode set-eoa"); + + LOG_XMIT_BYTES("set-eoa", xmit_buf, H5FD_MIRROR_XMIT_EOA_SIZE); + + if(HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_EOA_SIZE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to transmit set-eoa"); + + if(H5FD__mirror_verify_reply(file) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + +done: + if(xmit_buf) + xmit_buf = H5FL_BLK_FREE(xmit, xmit_buf); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__mirror_set_eoa() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_get_eof + * + * Purpose: Returns the end-of-file marker, which is the greater of + * either the filesystem end-of-file or the HDF5 end-of-address + * markers. + * + * Required to register the driver. + * + * Return: End of file address, the first address past the end of the + * "file", either the filesystem file or the HDF5 file. + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD__mirror_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_mirror_t *file = (const H5FD_mirror_t *)_file; + + FUNC_ENTER_STATIC_NOERR + + LOG_OP_CALL(FUNC); + + HDassert(file); + + FUNC_LEAVE_NOAPI(file->eof) +} /* end H5FD__mirror_get_eof() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_read + * + * Purpose: Required to register the driver, but if called, MUST fail. + * + * Return: FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_read(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED fapl_id, haddr_t H5_ATTR_UNUSED addr, + size_t H5_ATTR_UNUSED size, void H5_ATTR_UNUSED *buf) +{ + FUNC_ENTER_STATIC_NOERR + + LOG_OP_CALL(FUNC); + + FUNC_LEAVE_NOAPI(FAIL) +} /* end H5FD__mirror_read() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_write + * + * Purpose: Writes SIZE bytes of data to FILE beginning at address ADDR + * from buffer BUF according to data transfer properties in + * DXPL_ID. + * + * Send metadata regarding the write (location, size) to the + * remote Writer, then separately transmits the data. + * Both transmission expect an OK reply from the Writer. + * This two-exchange approach incurs significant overhead, + * but is a simple and modular approach. + * Start optimizations here. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, + haddr_t addr, size_t size, const void *buf) +{ + H5FD_mirror_xmit_write_t xmit_write; + unsigned char *xmit_buf = NULL; + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + HDassert(file); + HDassert(buf); + + file->xmit.xmit_count = (file->xmit_i)++; + file->xmit.op = H5FD_MIRROR_OP_WRITE; + + xmit_write.pub = file->xmit; + xmit_write.size = (uint64_t)size; + xmit_write.offset = (uint64_t)addr; + xmit_write.type = (uint8_t)type; + + xmit_buf = H5FL_BLK_MALLOC(xmit, H5FD_MIRROR_XMIT_BUFFER_MAX); + if(NULL == xmit_buf) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate xmit buffer"); + + /* Notify Writer of incoming data to write. */ + if(H5FD_mirror_xmit_encode_write(xmit_buf, &xmit_write) != H5FD_MIRROR_XMIT_WRITE_SIZE) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to encode write"); + + LOG_XMIT_BYTES("write", xmit_buf, H5FD_MIRROR_XMIT_WRITE_SIZE); + + if(HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_WRITE_SIZE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to transmit write"); + + /* Check that our write xmission was received */ + if(H5FD__mirror_verify_reply(file) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + + /* Send the data to be written */ + if(HDwrite(file->sock_fd, buf, size) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to transmit data"); + + /* Writer should reply that it got the data and is still okay/ready */ + if(H5FD__mirror_verify_reply(file) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + +done: + if(xmit_buf) + xmit_buf = H5FL_BLK_FREE(xmit, xmit_buf); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__mirror_write() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_truncate + * + * Purpose: Makes sure that the true file size is the same (or larger) + * than the end-of-address. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, + hbool_t H5_ATTR_UNUSED closing) +{ + unsigned char *xmit_buf = NULL; + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + file->xmit.xmit_count = (file->xmit_i)++; + file->xmit.op = H5FD_MIRROR_OP_TRUNCATE; + + xmit_buf = H5FL_BLK_MALLOC(xmit, H5FD_MIRROR_XMIT_BUFFER_MAX); + if(NULL == xmit_buf) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate xmit buffer"); + + if(H5FD_mirror_xmit_encode_header(xmit_buf, &(file->xmit)) != H5FD_MIRROR_XMIT_HEADER_SIZE) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to encode truncate"); + + LOG_XMIT_BYTES("truncate", xmit_buf, H5FD_MIRROR_XMIT_HEADER_SIZE); + + if(HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_HEADER_SIZE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to transmit truncate"); + + if(H5FD__mirror_verify_reply(file) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + +done: + if(xmit_buf) + xmit_buf = H5FL_BLK_FREE(xmit, xmit_buf); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__mirror_truncate() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_lock + * + * Purpose: To place an advisory lock on a file. + * The lock type to apply depends on the parameter "rw": + * TRUE--opens for write: an exclusive lock + * FALSE--opens for read: a shared lock + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_lock(H5FD_t *_file, hbool_t rw) +{ + H5FD_mirror_xmit_lock_t xmit_lock; + unsigned char *xmit_buf = NULL; + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + file->xmit.xmit_count = (file->xmit_i)++; + file->xmit.op = H5FD_MIRROR_OP_LOCK; + + xmit_lock.pub = file->xmit; + xmit_lock.rw = (uint64_t)rw; + + xmit_buf = H5FL_BLK_MALLOC(xmit, H5FD_MIRROR_XMIT_BUFFER_MAX); + if(NULL == xmit_buf) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate xmit buffer"); + + if(H5FD_mirror_xmit_encode_lock(xmit_buf, &xmit_lock) != H5FD_MIRROR_XMIT_LOCK_SIZE) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to encode lock"); + + LOG_XMIT_BYTES("lock", xmit_buf, H5FD_MIRROR_XMIT_LOCK_SIZE); + + if(HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_LOCK_SIZE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to transmit lock"); + + if(H5FD__mirror_verify_reply(file) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + +done: + if(xmit_buf) + xmit_buf = H5FL_BLK_FREE(xmit, xmit_buf); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__mirror_lock */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_unlock + * + * Purpose: Remove the existing lock on the file. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_unlock(H5FD_t *_file) +{ + unsigned char *xmit_buf = NULL; + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + file->xmit.xmit_count = (file->xmit_i)++; + file->xmit.op = H5FD_MIRROR_OP_UNLOCK; + + xmit_buf = H5FL_BLK_MALLOC(xmit, H5FD_MIRROR_XMIT_BUFFER_MAX); + if(NULL == xmit_buf) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate xmit buffer"); + + if(H5FD_mirror_xmit_encode_header(xmit_buf, &(file->xmit)) != H5FD_MIRROR_XMIT_HEADER_SIZE) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to encode unlock"); + + LOG_XMIT_BYTES("unlock", xmit_buf, H5FD_MIRROR_XMIT_HEADER_SIZE); + + if(HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_HEADER_SIZE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to transmit unlock"); + + if(H5FD__mirror_verify_reply(file) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + +done: + if(xmit_buf) + xmit_buf = H5FL_BLK_FREE(xmit, xmit_buf); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__mirror_unlock */ + +#endif /* H5_HAVE_MIRROR_VFD */ + diff --git a/src/H5FDmirror.h b/src/H5FDmirror.h new file mode 100644 index 0000000..7d15c1b --- /dev/null +++ b/src/H5FDmirror.h @@ -0,0 +1,79 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Public, shared definitions for Mirror VFD & remote Writer. + */ + +#ifndef H5FDmirror_H +#define H5FDmirror_H + +#ifdef H5_HAVE_MIRROR_VFD + +#define H5FD_MIRROR (H5FD_mirror_init()) + +#ifdef __cplusplus +extern "C" { +#endif + +/* ============================================================================ + * Mirror VFD use and operation. + * ============================================================================ + */ + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_fapl_t + * + * Used to pass configuraiton information to the Mirror VFD. + * Populate components as appropriate and pass structure pointer to + * `H5Pset_fapl_mirror()`. + * + * `magic` (uint32_t) + * Semi-unique number to sanity-check pointers to this structure type. + * MUST equal H5FD_MIRROR_FAPL_MAGIC to be considered valid. + * + * `version` (uint32_t) + * Indicates expected components of the structure. + * + * `handshake_port (int) + * Port number to expect to reach the "Mirror Server" on the remote host. + * + * `remote_ip` (char[]) + * IP address string of "Mirror Server" remote host. + * --------------------------------------------------------------------------- + */ +#define H5FD_MIRROR_FAPL_MAGIC 0xF8DD514C +#define H5FD_MIRROR_CURR_FAPL_T_VERSION 1 +#define H5FD_MIRROR_MAX_IP_LEN 32 +typedef struct H5FD_mirror_fapl_t { + uint32_t magic; + uint32_t version; + int handshake_port; + char remote_ip[H5FD_MIRROR_MAX_IP_LEN + 1]; +} H5FD_mirror_fapl_t; + +H5_DLL hid_t H5FD_mirror_init(void); +H5_DLL herr_t H5Pget_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa_out); +H5_DLL herr_t H5Pset_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa); + +#ifdef __cplusplus +} +#endif + +#else /* H5_HAVE_MIRROR_VFD */ + +#define H5FD_MIRROR (H5I_INAVLID_HID) + +#endif /* H5_HAVE_MIRROR_VFD */ + +#endif /* H5FDmirror_H */ + diff --git a/src/H5FDmirror_priv.h b/src/H5FDmirror_priv.h new file mode 100644 index 0000000..dc15441 --- /dev/null +++ b/src/H5FDmirror_priv.h @@ -0,0 +1,323 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Public, shared definitions for Mirror VFD & remote Writer. + */ + +#ifndef H5FDmirror_priv_H +#define H5FDmirror_priv_H + +#ifdef H5_HAVE_MIRROR_VFD + +#ifdef __cplusplus +extern "C" { +#endif + +/* = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = + * IPC - Mirror VFD and Remote Worker application. + * = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = + */ + +/* The maximum allowed size for a receiving buffer when accepting bytes to + * write. Writes larger than this size are performed by multiple accept-write + * steps by the Writer. */ +#define H5FD_MIRROR_DATA_BUFFER_MAX H5_GB /* 1 Gigabyte */ + +#define H5FD_MIRROR_XMIT_CURR_VERSION 1 +#define H5FD_MIRROR_XMIT_MAGIC 0x87F8005B + +#define H5FD_MIRROR_OP_OPEN 1 +#define H5FD_MIRROR_OP_CLOSE 2 +#define H5FD_MIRROR_OP_WRITE 3 +#define H5FD_MIRROR_OP_TRUNCATE 4 +#define H5FD_MIRROR_OP_REPLY 5 +#define H5FD_MIRROR_OP_SET_EOA 6 +#define H5FD_MIRROR_OP_LOCK 7 +#define H5FD_MIRROR_OP_UNLOCK 8 + +#define H5FD_MIRROR_STATUS_OK 0 +#define H5FD_MIRROR_STATUS_ERROR 1 +#define H5FD_MIRROR_STATUS_MESSAGE_MAX 256 /* Dedicated error message size */ + +/* Maximum length of a path/filename string, including the NULL-terminator. + * Must not be smaller than H5FD_SPLITTER_PATH_MAX. */ +#define H5FD_MIRROR_XMIT_FILEPATH_MAX 4097 + +/* Define the exact sizes of the various xmit blobs as sent over the wire. + * This is used to minimize the number of bytes transmitted as well as to + * sanity-check received bytes. + * Any modifications to the xmit structures and/or the encode/decode functions + * must be reflected here. + * */ +#define H5FD_MIRROR_XMIT_HEADER_SIZE 14 +#define H5FD_MIRROR_XMIT_EOA_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 9) +#define H5FD_MIRROR_XMIT_LOCK_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 8) +#define H5FD_MIRROR_XMIT_OPEN_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 20 + H5FD_MIRROR_XMIT_FILEPATH_MAX) +#define H5FD_MIRROR_XMIT_REPLY_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 4 + H5FD_MIRROR_STATUS_MESSAGE_MAX) +#define H5FD_MIRROR_XMIT_WRITE_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 17) + +/* Maximum length of any xmit. */ +#define H5FD_MIRROR_XMIT_BUFFER_MAX MAX2( MAX3(H5FD_MIRROR_XMIT_HEADER_SIZE, \ + H5FD_MIRROR_XMIT_EOA_SIZE, \ + H5FD_MIRROR_XMIT_LOCK_SIZE), \ + MAX3(H5FD_MIRROR_XMIT_OPEN_SIZE, \ + H5FD_MIRROR_XMIT_REPLY_SIZE, \ + H5FD_MIRROR_XMIT_WRITE_SIZE) ) \ + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_xmit_t + * + * Common structure 'header' for all mirror VFD/worker IPC. + * Must be the first component of a derived operation xmit structure, + * such as file-open or write command. + * + * `magic` (uint32_t) + * A "unique" number identifying the structure and endianness of + * transmitting maching. + * Must be set to H5FD_MIRROR_XMIT_MAGIC native to the VFD "sender". + * + * `version` (uint8_t) + * Number used to identify the structure membership. + * Allows sane modifications to this structure in the future. + * Must be set to H5FD_MIRROR_XMIT_CURR_VERSION. + * + * `session_token` (uint32_t) + * A "unique" number identifying the session between VFD sender and + * remote receiver/worker/writer. Exists to help sanity-check. + * + * `xmit_count` (uint32_t) + * Which transmission this is since the session began. + * Used to sanity-check transmission errors. + * First xmit (file-open) must be 0. + * + * `op` (uint8_t) + * Number identifying which operation to perform. + * Corresponds with the extended structure outside of this xmit header. + * Possible values are all defined H5FD_MIRROR_OP_* constants. + * + * --------------------------------------------------------------------------- + */ +typedef struct H5FD_mirror_xmit_t { + uint32_t magic; + uint8_t version; + uint32_t session_token; + uint32_t xmit_count; + uint8_t op; +} H5FD_mirror_xmit_t; + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_xmit_eoa_t + * + * Structure containing eoa-set information from VFD sender. + * + * `pub` (H5FD_mirror_xmit_t) + * Common transmission header, containing session information. + * Must be first. + * + * `type` (uint8_t) + * System-independent alias for H5F[D]_mem_t. + * Specifies datatype to be written. + * + * `eoa_addr` (uint64_t) + * New address for eoa. + * (Natively 'haddr_t', always a 64-bit field) + * + * --------------------------------------------------------------------------- + */ +typedef struct H5FD_mirror_xmit_eoa_t { + H5FD_mirror_xmit_t pub; + uint8_t type; + uint64_t eoa_addr; +} H5FD_mirror_xmit_eoa_t; + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_xmit_lock_t + * + * Structure containing eoa-set information from VFD sender. + * + * `pub` (H5FD_mirror_xmit_t) + * Common transmission header, containing session information. + * Must be first. + * + * `rw` (uint64_t) + * The Read/Write mode flag passed into H5FDlock(). + * (Natively `hbool_t`, an 'int') TODO: native int may be 64-bit? + * + * --------------------------------------------------------------------------- + */ +typedef struct H5FD_mirror_xmit_lock_t { + H5FD_mirror_xmit_t pub; + uint64_t rw; +} H5FD_mirror_xmit_lock_t; + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_xmit_open_t + * + * Structure containing file-open information from the VFD sender. + * + * `pub` (H5FD_mirror_xmit_t) + * Common transmission header, containing session information. + * Must be first. + * + * `flags` (uint32_t) + * VFL-layer file-open flags passed directly to H5FDopen(). + * (Natively 'unsigned [int]') TODO: native int may be 64-bit? + * + * `maxaddr` (uint64_t) + * VFL-layer maximum allowed address space for the file to open passed + * directly to H5FDopen(). + * (Natively 'haddr_t', always a 64-bit field) + * + * `size_t_blob` (uint64_t) + * A number indicating how large a size_t is on the sending system. + * Must be set to (uint64_t)((size_t)(-1)) + * (maximum possible value of size_t, cast to uint64_t). + * The receiving system inspects this value -- if the local (remote) + * size_t is smaller than that of the Sender, issues a warning. + * Not an error, as: + * 1. It is assumed that underlying file systems/drivers have become + * smart enough to handle file sizes that otherwise might be + * constrained. + * 2. The Mirror Writer ingests bytes to write multiple 'slices' if the + * size is greater than H5FD_MIRROR_DATA_BUFFER_MAX, regardless of + * any size_t storage size disparity. + * + * `filename` (char[]) + * String giving the filename and path of file to open. + * + * --------------------------------------------------------------------------- + */ +typedef struct H5FD_mirror_xmit_open_t { + H5FD_mirror_xmit_t pub; + uint32_t flags; + uint64_t maxaddr; + uint64_t size_t_blob; + char filename[H5FD_MIRROR_XMIT_FILEPATH_MAX]; +} H5FD_mirror_xmit_open_t; + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_xmit_reply_t + * + * Structure used by the remote receiver/worker/writer to respond to + * a command from the VFD sender. + * + * `pub` (H5FD_mirror_xmit_t) + * Common transmission header, containing session information. + * Must be first. + * + * `status` (uint32_t) + * Number indicating whether the command was successful or if an + * occured. + * Allowed values are H5FD_MIRROR_STATUS_OK and + * H5FD_MIRROR_STATUS_ERROR. + * + * `message` (char[]) + * Error message. Populated if and only if there was a problem. + * It is possible that a message may reach the end of the alloted + * space without a NULL terminator -- the onus is on the programmer to + * handle this situation. + * + * --------------------------------------------------------------------------- + */ +typedef struct H5FD_mirror_xmit_reply_t { + H5FD_mirror_xmit_t pub; + uint32_t status; + char message[H5FD_MIRROR_STATUS_MESSAGE_MAX]; +} H5FD_mirror_xmit_reply_t; + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_xmit_write_t + * + * Structure containing data-write information from VFD sender. + * + * The data to be written is transmitted in subsequent, packets + * and may be broken up into more than one transmission buffer. + * The VFD sender and remote receiver/worker/writer must coordinate + * the receipt of data. + * + * `pub` (H5FD_mirror_xmit_t) + * Common transmission header, containing session information. + * Must be first. + * + * `type` (uint8_t) + * Specifies datatype to be written. + * (Natively 'H5FD_mem_t', an enumerated type in H5Fpublic.h) + * + * `offset` (uint64_t) + * Start location of write in file. + * (Natively 'haddr_t', always a 64-bit field) + * + * `size` (uint64_t) + * Size of the data to be written, in bytes. + * (Natively 'size_t', accommodate the largest possible as 64-bits) + * + * --------------------------------------------------------------------------- + */ +typedef struct H5FD_mirror_xmit_write_t { + H5FD_mirror_xmit_t pub; + uint8_t type; + uint64_t offset; + uint64_t size; +} H5FD_mirror_xmit_write_t; + + + +/* Encode/decode routines are required to "pack" the xmit data into a known + * byte format for transmission over the wire. + * + * All component numbers must be stored in "network" word order (Big-Endian). + * + * All components must be packed in the order given in the structure definition. + * + * All components must be packed with zero padding between. + */ + +H5_DLL size_t H5FD__mirror_xmit_decode_uint16(uint16_t *out, const unsigned char *buf); +H5_DLL size_t H5FD__mirror_xmit_decode_uint32(uint32_t *out, const unsigned char *buf); +H5_DLL size_t H5FD__mirror_xmit_decode_uint64(uint64_t *out, const unsigned char *buf); +H5_DLL size_t H5FD__mirror_xmit_decode_uint8(uint8_t *out, const unsigned char *buf); +H5_DLL size_t H5FD__mirror_xmit_encode_uint16(unsigned char *dest, uint16_t v); +H5_DLL size_t H5FD__mirror_xmit_encode_uint32(unsigned char *dest, uint32_t v); +H5_DLL size_t H5FD__mirror_xmit_encode_uint64(unsigned char *dest, uint64_t v); +H5_DLL size_t H5FD__mirror_xmit_encode_uint8(unsigned char *dest, uint8_t v); + +H5_DLL size_t H5FD_mirror_xmit_decode_header(H5FD_mirror_xmit_t *out, const unsigned char *buf); +H5_DLL size_t H5FD_mirror_xmit_decode_lock(H5FD_mirror_xmit_lock_t *out, const unsigned char *buf); +H5_DLL size_t H5FD_mirror_xmit_decode_open(H5FD_mirror_xmit_open_t *out, const unsigned char *buf); +H5_DLL size_t H5FD_mirror_xmit_decode_reply(H5FD_mirror_xmit_reply_t *out, const unsigned char *buf); +H5_DLL size_t H5FD_mirror_xmit_decode_set_eoa(H5FD_mirror_xmit_eoa_t *out, const unsigned char *buf); +H5_DLL size_t H5FD_mirror_xmit_decode_write(H5FD_mirror_xmit_write_t *out, const unsigned char *buf); + +H5_DLL size_t H5FD_mirror_xmit_encode_header(unsigned char *dest, const H5FD_mirror_xmit_t *x); +H5_DLL size_t H5FD_mirror_xmit_encode_lock(unsigned char *dest, const H5FD_mirror_xmit_lock_t *x); +H5_DLL size_t H5FD_mirror_xmit_encode_open(unsigned char *dest, const H5FD_mirror_xmit_open_t *x); +H5_DLL size_t H5FD_mirror_xmit_encode_reply(unsigned char *dest, const H5FD_mirror_xmit_reply_t *x); +H5_DLL size_t H5FD_mirror_xmit_encode_set_eoa(unsigned char *dest, const H5FD_mirror_xmit_eoa_t *x); +H5_DLL size_t H5FD_mirror_xmit_encode_write(unsigned char *dest, const H5FD_mirror_xmit_write_t *x); + +H5_DLL hbool_t H5FD_mirror_xmit_is_close(const H5FD_mirror_xmit_t *xmit); +H5_DLL hbool_t H5FD_mirror_xmit_is_lock(const H5FD_mirror_xmit_lock_t *xmit); +H5_DLL hbool_t H5FD_mirror_xmit_is_open(const H5FD_mirror_xmit_open_t *xmit); +H5_DLL hbool_t H5FD_mirror_xmit_is_reply(const H5FD_mirror_xmit_reply_t *xmit); +H5_DLL hbool_t H5FD_mirror_xmit_is_set_eoa(const H5FD_mirror_xmit_eoa_t *xmit); +H5_DLL hbool_t H5FD_mirror_xmit_is_write(const H5FD_mirror_xmit_write_t *xmit); +H5_DLL hbool_t H5FD_mirror_xmit_is_xmit(const H5FD_mirror_xmit_t *xmit); + +#ifdef __cplusplus +} +#endif + +#endif /* H5_HAVE_MIRROR_VFD */ + +#endif /* H5FDmirror_priv_H */ + diff --git a/src/H5FDmulti.c b/src/H5FDmulti.c index 72f4da5..f80cac5 100644 --- a/src/H5FDmulti.c +++ b/src/H5FDmulti.c @@ -1856,7 +1856,7 @@ H5FD_multi_lock(H5FD_t *_file, hbool_t rw) } /* end if */ if(nerrors) - H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "error locking member files", -1) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTLOCKFILE, "error locking member files", -1) return 0; } /* H5FD_multi_lock() */ @@ -1893,7 +1893,7 @@ H5FD_multi_unlock(H5FD_t *_file) } END_MEMBERS; if(nerrors) - H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "error unlocking member files", -1) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTUNLOCKFILE, "error unlocking member files", -1) return 0; } /* H5FD_multi_unlock() */ diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h index 54ca8f7..61bf212 100644 --- a/src/H5FDpublic.h +++ b/src/H5FDpublic.h @@ -255,6 +255,8 @@ typedef enum H5F_mem_t H5FD_mem_t; * that creates a file which is compatible with the default VFD. * Generally, this means that the VFD creates a single file that follows * the canonical HDF5 file format. + * Regarding the Splitter VFD specifically, only drivers with this flag + * enabled may be used as the Write-Only (W/O) channel driver. */ #define H5FD_FEAT_DEFAULT_VFD_COMPATIBLE 0x00008000 diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c index ba4750e..ce11c05 100644 --- a/src/H5FDsec2.c +++ b/src/H5FDsec2.c @@ -39,6 +39,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_SEC2_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* The description of a file belonging to this driver. The 'eoa' and 'eof' * determine the amount of hdf5 address space in use and the high-water mark * of the file (the current size of the underlying filesystem file). The @@ -57,6 +60,7 @@ typedef struct H5FD_sec2_t { haddr_t eof; /* end of file; current file size */ haddr_t pos; /* current file I/O position */ H5FD_file_op_t op; /* last operation */ + hbool_t ignore_disabled_file_locks; char filename[H5FD_MAX_FILENAME_LEN]; /* Copy of file name from open operation */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely @@ -190,10 +194,20 @@ H5FL_DEFINE_STATIC(H5FD_sec2_t); static herr_t H5FD__init_package(void) { + char *lock_env_var = NULL; /* Environment variable pointer */ herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if(lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if(H5FD_sec2_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize sec2 VFD") @@ -316,6 +330,7 @@ H5FD_sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) struct _BY_HANDLE_FILE_INFORMATION fileinfo; #endif h5_stat_t sb; + H5P_genplist_t *plist; /* Property list pointer */ H5FD_t *ret_value = NULL; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -373,17 +388,26 @@ H5FD_sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) file->inode = sb.st_ino; #endif /* H5_HAVE_WIN32_API */ + /* Get the FAPL */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, NULL, "not a file access property list") + + /* Check the file locking flags in the fapl */ + if(ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if(H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + /* Retain a copy of the name used to open the file, for possible error reporting */ HDstrncpy(file->filename, name, sizeof(file->filename)); file->filename[sizeof(file->filename) - 1] = '\0'; /* Check for non-default FAPL */ if(H5P_FILE_ACCESS_DEFAULT != fapl_id) { - H5P_genplist_t *plist; /* Property list pointer */ - - /* Get the FAPL */ - if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) - HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, NULL, "not a file access property list") /* This step is for h5repart tool only. If user wants to change file driver from * family to one that uses single files (sec2, etc.) while using h5repart, this @@ -521,6 +545,12 @@ H5FD_sec2_query(const H5FD_t *_file, unsigned long *flags /* out */) FUNC_ENTER_NOAPI_NOINIT_NOERR /* Set the VFL feature flags that this driver supports */ + /* Notice: the Mirror VFD Writer currently uses only the Sec2 driver as + * the underying driver -- as such, the Mirror VFD implementation copies + * these feature flags as its own. Any modifications made here must be + * reflected in H5FDmirror.c + * -- JOS 2020-01-13 + */ if(flags) { *flags = 0; *flags |= H5FD_FEAT_AGGREGATE_METADATA; /* OK to aggregate metadata allocations */ @@ -955,11 +985,15 @@ H5FD_sec2_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if(HDflock(file->fd, lock_flags | LOCK_NB) < 0) { - if(ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file") + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -988,11 +1022,15 @@ H5FD_sec2_unlock(H5FD_t *_file) HDassert(file); if(HDflock(file->fd, LOCK_UN) < 0) { - if(ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDsplitter.c b/src/H5FDsplitter.c new file mode 100644 index 0000000..d0ed250d --- /dev/null +++ b/src/H5FDsplitter.c @@ -0,0 +1,1346 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: The Splitter VFD implements a file driver which relays all the + * VFD calls to an underlying VFD, and send all the write calls to + * another underlying VFD. Maintains two files simultaneously. + */ + +/* This source code file is part of the H5FD driver module */ +#include "H5FDdrvr_module.h" + +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File access */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5FDsplitter.h" /* Splitter file driver */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5Pprivate.h" /* Property lists */ + +/* The driver identification number, initialized at runtime */ +static hid_t H5FD_SPLITTER_g = 0; + +/* Driver-specific file access properties */ +typedef struct H5FD_splitter_fapl_t { + hid_t rw_fapl_id; /* fapl for the R/W channel */ + hid_t wo_fapl_id; /* fapl for the W/O channel */ + char wo_path[H5FD_SPLITTER_PATH_MAX + 1]; /* file name for the W/O channel */ + char log_file_path[H5FD_SPLITTER_PATH_MAX + 1]; /* file to record errors reported by the W/O channel */ + hbool_t ignore_wo_errs; /* TRUE to ignore errors on the W/O channel */ +} H5FD_splitter_fapl_t; + +/* The information of this splitter */ +typedef struct H5FD_splitter_t { + H5FD_t pub; /* public stuff, must be first */ + unsigned version; /* version of the H5FD_splitter_vfd_config_t structure used */ + H5FD_splitter_fapl_t fa; /* driver-specific file access properties */ + H5FD_t *rw_file; /* pointer of R/W channel */ + H5FD_t *wo_file; /* pointer of W/O channel */ + FILE *logfp; /* Log file pointer */ +} H5FD_splitter_t; + +/* + * These macros check for overflow of various quantities. These macros + * assume that HDoff_t is signed and haddr_t and size_t are unsigned. + * + * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t' + * is too large to be represented by the second argument + * of the file seek function. + * + * SIZE_OVERFLOW: Checks whether a buffer size of type `hsize_t' is too + * large to be represented by the `size_t' type. + * + * REGION_OVERFLOW: Checks whether an address and size pair describe data + * which can be addressed entirely by the second + * argument of the file seek function. + */ +#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) +#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR)) +#define SIZE_OVERFLOW(Z) ((Z) & ~(hsize_t)MAXADDR) +#define REGION_OVERFLOW(A,Z) (ADDR_OVERFLOW(A) || SIZE_OVERFLOW(Z) || \ + HADDR_UNDEF==(A)+(Z) || \ + (HDoff_t)((A)+(Z))<(HDoff_t)(A)) + +/* This macro provides a wrapper for shared fail-log-ignore behavior + * for errors arising in the splitter's W/O channel. + * Logs an error entry in a log file, if the file exists. + * If not set to ignore errors, registers an error with the library. + */ +#define H5FD_SPLITTER_WO_ERROR(file, funcname, errmajor, errminor, ret, mesg) \ +{ \ + H5FD__splitter_log_error((file), (funcname), (mesg)); \ + if(FALSE == (file)->fa.ignore_wo_errs) \ + HGOTO_ERROR((errmajor), (errminor), (ret), (mesg)) \ +} + +#define H5FD_SPLITTER_DEBUG_OP_CALLS 0 /* debugging print toggle; 0 disables */ + +#if H5FD_SPLITTER_DEBUG_OP_CALLS +#define H5FD_SPLITTER_LOG_CALL(name) do { \ + HDprintf("called %s()\n", (name)); \ + HDfflush(stdout); \ +} while (0) +#else +#define H5FD_SPLITTER_LOG_CALL(name) /* no-op */ +#endif /* H5FD_SPLITTER_DEBUG_OP_CALLS */ + +/* Private functions */ + +/* Print error messages from W/O channel to log file */ +static herr_t H5FD__splitter_log_error(const H5FD_splitter_t *file, const char *atfunc, const char *msg); +static int H5FD__copy_plist(hid_t fapl_id, hid_t *id_out_ptr); + +/* Prototypes */ +static herr_t H5FD__splitter_term(void); +static hsize_t H5FD__splitter_sb_size(H5FD_t *_file); +static herr_t H5FD__splitter_sb_encode(H5FD_t *_file, char *name/*out*/, unsigned char *buf/*out*/); +static herr_t H5FD__splitter_sb_decode(H5FD_t *_file, const char *name, const unsigned char *buf); +static void *H5FD__splitter_fapl_get(H5FD_t *_file); +static void *H5FD__splitter_fapl_copy(const void *_old_fa); +static herr_t H5FD__splitter_fapl_free(void *_fapl); +static H5FD_t *H5FD__splitter_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); +static herr_t H5FD__splitter_close(H5FD_t *_file); +static int H5FD__splitter_cmp(const H5FD_t *_f1, const H5FD_t *_f2); +static herr_t H5FD__splitter_query(const H5FD_t *_file, unsigned long *flags /* out */); +static herr_t H5FD__splitter_get_type_map(const H5FD_t *_file, H5FD_mem_t *type_map); +static haddr_t H5FD__splitter_alloc(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size); +static herr_t H5FD__splitter_free(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t size); +static haddr_t H5FD__splitter_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type); +static herr_t H5FD__splitter_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr); +static haddr_t H5FD__splitter_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type); +static herr_t H5FD__splitter_get_handle(H5FD_t *_file, hid_t H5_ATTR_UNUSED fapl, void** file_handle); +static herr_t H5FD__splitter_read(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, void *buf); +static herr_t H5FD__splitter_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, const void *buf); +static herr_t H5FD__splitter_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); +static herr_t H5FD__splitter_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); +static herr_t H5FD__splitter_lock(H5FD_t *_file, hbool_t rw); +static herr_t H5FD__splitter_unlock(H5FD_t *_file); + +static const H5FD_class_t H5FD_splitter_g = { + "splitter", /* name */ + MAXADDR, /* maxaddr */ + H5F_CLOSE_WEAK, /* fc_degree */ + H5FD__splitter_term, /* terminate */ + H5FD__splitter_sb_size, /* sb_size */ + H5FD__splitter_sb_encode, /* sb_encode */ + H5FD__splitter_sb_decode, /* sb_decode */ + sizeof(H5FD_splitter_fapl_t), /* fapl_size */ + H5FD__splitter_fapl_get, /* fapl_get */ + H5FD__splitter_fapl_copy, /* fapl_copy */ + H5FD__splitter_fapl_free, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD__splitter_open, /* open */ + H5FD__splitter_close, /* close */ + H5FD__splitter_cmp, /* cmp */ + H5FD__splitter_query, /* query */ + H5FD__splitter_get_type_map, /* get_type_map */ + H5FD__splitter_alloc, /* alloc */ + H5FD__splitter_free, /* free */ + H5FD__splitter_get_eoa, /* get_eoa */ + H5FD__splitter_set_eoa, /* set_eoa */ + H5FD__splitter_get_eof, /* get_eof */ + H5FD__splitter_get_handle, /* get_handle */ + H5FD__splitter_read, /* read */ + H5FD__splitter_write, /* write */ + H5FD__splitter_flush, /* flush */ + H5FD__splitter_truncate, /* truncate */ + H5FD__splitter_lock, /* lock */ + H5FD__splitter_unlock, /* unlock */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ +}; + +/* Declare a free list to manage the H5FD_splitter_t struct */ +H5FL_DEFINE_STATIC(H5FD_splitter_t); + +/* Declare a free list to manage the H5FD_splitter_fapl_t struct */ +H5FL_DEFINE_STATIC(H5FD_splitter_fapl_t); + + +/*------------------------------------------------------------------------- + * Function: H5FD__init_package + * + * Purpose: Initializes any interface-specific data or routines. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__init_package(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + if(H5FD_splitter_init() < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize splitter VFD") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__init_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_init + * + * Purpose: Initialize the splitter driver by registering it with the + * library. + * + * Return: Success: The driver ID for the splitter driver. + * Failure: Negative + *------------------------------------------------------------------------- + */ +hid_t +H5FD_splitter_init(void) +{ + hid_t ret_value = H5I_INVALID_HID; + + FUNC_ENTER_NOAPI(FAIL) + + H5FD_SPLITTER_LOG_CALL(FUNC); + + if(H5I_VFL != H5I_get_type(H5FD_SPLITTER_g)) + H5FD_SPLITTER_g = H5FDregister(&H5FD_splitter_g); + + ret_value = H5FD_SPLITTER_g; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_init() */ + + +/*--------------------------------------------------------------------------- + * Function: H5FD__splitter_term + * + * Purpose: Shut down the splitter VFD. + * + * Returns: SUCCEED (Can't fail) + *--------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_term(void) +{ + FUNC_ENTER_STATIC_NOERR + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Reset VFL ID */ + H5FD_SPLITTER_g = 0; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD__splitter_term() */ + + + /*------------------------------------------------------------------------- + * Function: H5FD__copy_plist + * + * Purpose: Sanity-wrapped H5P_copy_plist() for each channel. + * Utility function for operation in multiple locations. + * + * Return: 0 on success, -1 on error. + *------------------------------------------------------------------------- + */ +static int +H5FD__copy_plist(hid_t fapl_id, + hid_t *id_out_ptr) +{ + int ret_value = 0; + H5P_genplist_t *plist_ptr = NULL; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + HDassert(id_out_ptr != NULL); + + if(FALSE == H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, -1, "not a file access property list"); + + plist_ptr = (H5P_genplist_t *)H5I_object(fapl_id); + if(NULL == plist_ptr) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, -1, "unable to get property list"); + + *id_out_ptr = H5P_copy_plist(plist_ptr, FALSE); + if(H5I_INVALID_HID == *id_out_ptr) + HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, -1, "unable to copy file access property list"); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__copy_plist() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_fapl_splitter + * + * Purpose: Sets the file access property list to use the + * splitter driver. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *vfd_config) +{ + H5FD_splitter_fapl_t *info = NULL; + H5P_genplist_t *plist_ptr = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*Dr", fapl_id, vfd_config); + + H5FD_SPLITTER_LOG_CALL(FUNC); + + if(H5FD_SPLITTER_MAGIC != vfd_config->magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid configuration (magic number mismatch)") + if(H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION != vfd_config->version) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid config (version number mismatch)") + if(NULL == (plist_ptr = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid property list") + + /* Make sure that the W/O channel supports write-only capability. + * Some drivers (e.g. family or multi) do revision of the superblock + * in-memory, causing problems in that channel. + * Uses the feature flag H5FD_FEAT_DEFAULT_VFD_COMPATIBLE as the + * determining attribute. + */ + if(H5P_DEFAULT != vfd_config->wo_fapl_id) { + H5FD_class_t *wo_driver = NULL; + H5FD_driver_prop_t wo_driver_prop; + H5P_genplist_t *wo_plist_ptr = NULL; + unsigned long wo_driver_flags = 0; + + wo_plist_ptr = (H5P_genplist_t *)H5I_object(vfd_config->wo_fapl_id); + if(NULL == wo_plist_ptr) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + if(H5P_peek(wo_plist_ptr, H5F_ACS_FILE_DRV_NAME, &wo_driver_prop) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get driver ID & info") + wo_driver = (H5FD_class_t *)H5I_object(wo_driver_prop.driver_id); + if(NULL == wo_driver) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid driver ID in file access property list") + if(H5FD_driver_query(wo_driver, &wo_driver_flags) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't query VFD flags") + if(0 == (H5FD_FEAT_DEFAULT_VFD_COMPATIBLE & wo_driver_flags)) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unsuitable W/O driver") + } /* end if W/O VFD is non-default */ + + info = H5FL_CALLOC(H5FD_splitter_fapl_t); + if(NULL == info) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate file access property list struct") + + info->ignore_wo_errs = vfd_config->ignore_wo_errs; + HDstrncpy(info->wo_path, vfd_config->wo_path, H5FD_SPLITTER_PATH_MAX); + HDstrncpy(info->log_file_path, vfd_config->log_file_path, H5FD_SPLITTER_PATH_MAX); + info->rw_fapl_id = H5P_FILE_ACCESS_DEFAULT; /* pre-set value */ + info->wo_fapl_id = H5P_FILE_ACCESS_DEFAULT; /* pre-set value */ + + /* Set non-default channel FAPL IDs in splitter configuration info */ + if(H5P_DEFAULT != vfd_config->rw_fapl_id) { + if(FALSE == H5P_isa_class(vfd_config->rw_fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list") + info->rw_fapl_id = vfd_config->rw_fapl_id; + } + if(H5P_DEFAULT != vfd_config->wo_fapl_id) { + if(FALSE == H5P_isa_class(vfd_config->wo_fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list") + info->wo_fapl_id = vfd_config->wo_fapl_id; + } + + ret_value = H5P_set_driver(plist_ptr, H5FD_SPLITTER, info); + +done: + if(info) + info = H5FL_FREE(H5FD_splitter_fapl_t, info); + + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_fapl_splitter() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_fapl_splitter + * + * Purpose: Returns information about the splitter file access property + * list through the structure config_out. + * + * Will fail if config_out is received without pre-set valid + * magic and version information. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *config_out) +{ + const H5FD_splitter_fapl_t *fapl_ptr = NULL; + H5P_genplist_t *plist_ptr = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*Dr", fapl_id, config_out); + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Check arguments */ + if(TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + if(config_out == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "config_out pointer is null") + if(H5FD_SPLITTER_MAGIC != config_out->magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "info-out pointer invalid (magic number mismatch)") + if(H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION != config_out->version) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "info-out pointer invalid (version unsafe)") + + /* Pre-set out FAPL IDs with intent to replace these values */ + config_out->rw_fapl_id = H5I_INVALID_HID; + config_out->wo_fapl_id = H5I_INVALID_HID; + + /* Check and get the splitter fapl */ + if(NULL == (plist_ptr = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + if(H5FD_SPLITTER != H5P_peek_driver(plist_ptr)) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") + if(NULL == (fapl_ptr = (const H5FD_splitter_fapl_t *)H5P_peek_driver_info(plist_ptr))) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "unable to get specific-driver info") + + HDstrncpy(config_out->wo_path, fapl_ptr->wo_path, H5FD_SPLITTER_PATH_MAX); + HDstrncpy(config_out->log_file_path, fapl_ptr->log_file_path, H5FD_SPLITTER_PATH_MAX); + config_out->ignore_wo_errs = fapl_ptr->ignore_wo_errs; + + /* Copy R/W and W/O FAPLs */ + if(H5FD__copy_plist(fapl_ptr->rw_fapl_id, &(config_out->rw_fapl_id)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't copy R/W FAPL"); + if(H5FD__copy_plist(fapl_ptr->wo_fapl_id, &(config_out->wo_fapl_id)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't copy W/O FAPL"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_fapl_splitter() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_flush + * + * Purpose: Flushes all data to disk for both channels. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_flush(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t closing) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Public API for dxpl "context" */ + if(H5FDflush(file->rw_file, dxpl_id, closing) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTFLUSH, FAIL, "unable to flush R/W file") + if(H5FDflush(file->wo_file, dxpl_id, closing) < 0) + H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_CANTFLUSH, FAIL, "unable to flush W/O file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_flush() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_read + * + * Purpose: Reads SIZE bytes of data from the R/W channel, beginning at + * address ADDR into buffer BUF according to data transfer + * properties in DXPL_ID. + * + * Return: Success: SUCCEED + * The read result is written into the BUF buffer + * which should be allocated by the caller. + * Failure: FAIL + * The contents of BUF are undefined. + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, size_t size, void *buf) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + HDassert(file && file->pub.cls); + HDassert(buf); + + /* Check for overflow conditions */ + if(!H5F_addr_defined(addr)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "addr undefined, addr = %llu", (unsigned long long)addr) + if(REGION_OVERFLOW(addr, size)) + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, addr = %llu", (unsigned long long)addr) + + /* Only read from R/W channel */ + /* Public API for dxpl "context" */ + if(H5FDread(file->rw_file, type, dxpl_id, addr, size, buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "Reading from R/W channel failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_read() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_write + * + * Purpose: Writes SIZE bytes of data to R/W and W/O channels, beginning + * at address ADDR from buffer BUF according to data transfer + * properties in DXPL_ID. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, + haddr_t addr, size_t size, const void *buf) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + H5P_genplist_t *plist_ptr = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + if(NULL == (plist_ptr = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") + + /* Write to each file */ + /* Public API for dxpl "context" */ + if(H5FDwrite(file->rw_file, type, dxpl_id, addr, size, buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "R/W file write failed") + if(H5FDwrite(file->wo_file, type, dxpl_id, addr, size, buf) < 0) + H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_WRITEERROR, FAIL, "unable to write W/O file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_write() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_fapl_get + * + * Purpose: Returns a file access property list which indicates how the + * specified file is being accessed. The return list could be + * used to access another file the same way. + * + * Return: Success: Ptr to new file access property list with all + * members copied from the file struct. + * Failure: NULL + *------------------------------------------------------------------------- + */ +static void * +H5FD__splitter_fapl_get(H5FD_t *_file) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + void *ret_value = NULL; + + FUNC_ENTER_STATIC_NOERR + + H5FD_SPLITTER_LOG_CALL(FUNC); + + ret_value = H5FD__splitter_fapl_copy(&(file->fa)); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_fapl_get() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_fapl_copy + * + * Purpose: Copies the file access properties. + * + * Return: Success: Pointer to a new property list info structure. + * Failure: NULL + *------------------------------------------------------------------------- + */ +static void * +H5FD__splitter_fapl_copy(const void *_old_fa) +{ + const H5FD_splitter_fapl_t *old_fa_ptr = (const H5FD_splitter_fapl_t *)_old_fa; + H5FD_splitter_fapl_t *new_fa_ptr = NULL; + void *ret_value = NULL; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + HDassert(old_fa_ptr); + + new_fa_ptr = H5FL_CALLOC(H5FD_splitter_fapl_t); + if(NULL == new_fa_ptr) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate log file FAPL") + + HDmemcpy(new_fa_ptr, old_fa_ptr, sizeof(H5FD_splitter_fapl_t)); + HDstrncpy(new_fa_ptr->wo_path, old_fa_ptr->wo_path, H5FD_SPLITTER_PATH_MAX); + HDstrncpy(new_fa_ptr->log_file_path, old_fa_ptr->log_file_path, H5FD_SPLITTER_PATH_MAX); + + /* Copy R/W and W/O FAPLs */ + if(H5FD__copy_plist(old_fa_ptr->rw_fapl_id, &(new_fa_ptr->rw_fapl_id)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't copy R/W FAPL"); + if(H5FD__copy_plist(old_fa_ptr->wo_fapl_id, &(new_fa_ptr->wo_fapl_id)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't copy W/O FAPL"); + + ret_value = (void *)new_fa_ptr; + +done: + if(NULL == ret_value) + if(new_fa_ptr) + new_fa_ptr = H5FL_FREE(H5FD_splitter_fapl_t, new_fa_ptr); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_fapl_copy() */ + + +/*-------------------------------------------------------------------------- + * Function: H5FD__splitter_fapl_free + * + * Purpose: Releases the file access lists + * + * Return: SUCCEED/FAIL + *-------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_fapl_free(void *_fapl) +{ + H5FD_splitter_fapl_t *fapl = (H5FD_splitter_fapl_t*)_fapl; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Check arguments */ + HDassert(fapl); + + if(H5I_dec_ref(fapl->rw_fapl_id) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTDEC, FAIL, "can't close R/W FAPL ID") + if(H5I_dec_ref(fapl->wo_fapl_id) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTDEC, FAIL, "can't close W/O FAPL ID") + + /* Free the property list */ + fapl = H5FL_FREE(H5FD_splitter_fapl_t, fapl); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_fapl_free() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_open + * + * Purpose: Create and/or opens a file as an HDF5 file. + * + * Return: Success: A pointer to a new file data structure. The + * public fields will be initialized by the + * caller, which is always H5FD_open(). + * Failure: NULL + *------------------------------------------------------------------------- + */ +static H5FD_t * +H5FD__splitter_open(const char *name, unsigned flags, hid_t splitter_fapl_id, haddr_t maxaddr) +{ + H5FD_splitter_t *file_ptr = NULL; /* Splitter VFD info */ + const H5FD_splitter_fapl_t *fapl_ptr = NULL; /* Driver-specific property list */ + H5P_genplist_t *plist_ptr = NULL; + H5FD_t *ret_value = NULL; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Check arguments */ + if(!name || !*name) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name") + if(0 == maxaddr || HADDR_UNDEF == maxaddr) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr") + if(ADDR_OVERFLOW(maxaddr)) + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr") + if((H5P_FILE_ACCESS_DEFAULT == splitter_fapl_id) || + (H5FD_SPLITTER != H5Pget_driver(splitter_fapl_id)) ) + /* presupposes that H5P_FILE_ACCESS_DEFAULT is not a splitter */ + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "driver is not splitter") + + file_ptr = (H5FD_splitter_t *)H5FL_CALLOC(H5FD_splitter_t); + if(NULL == file_ptr) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate file struct") + file_ptr->fa.rw_fapl_id = H5I_INVALID_HID; + file_ptr->fa.wo_fapl_id = H5I_INVALID_HID; + + /* Get the driver-specific file access properties */ + plist_ptr = (H5P_genplist_t *)H5I_object(splitter_fapl_id); + if(NULL == plist_ptr) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list") + fapl_ptr = (const H5FD_splitter_fapl_t *)H5P_peek_driver_info(plist_ptr); + if(NULL == fapl_ptr) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "unable to get VFL driver info") + + /* Copy simpler info */ + HDstrncpy(file_ptr->fa.wo_path, fapl_ptr->wo_path, H5FD_SPLITTER_PATH_MAX); + HDstrncpy(file_ptr->fa.log_file_path, fapl_ptr->log_file_path, H5FD_SPLITTER_PATH_MAX); + file_ptr->fa.ignore_wo_errs = fapl_ptr->ignore_wo_errs; + + /* Copy R/W and W/O channel FAPLs. */ + if(H5FD__copy_plist(fapl_ptr->rw_fapl_id, &(file_ptr->fa.rw_fapl_id)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't copy R/W FAPL"); + if(H5FD__copy_plist(fapl_ptr->wo_fapl_id, &(file_ptr->fa.wo_fapl_id)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't copy W/O FAPL"); + + /* Prepare log file if necessary. + * If application wants to ignore the errors from W/O channel and + * provided a name for the log file, then open it + */ + if(!file_ptr->logfp) { + if(file_ptr->fa.log_file_path[0] != '\0') { + file_ptr->logfp = HDfopen(file_ptr->fa.log_file_path, "w"); + if(file_ptr->logfp == NULL) + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "unable to open log file") + } /* end if logfile path given */ + } /* end if logfile pointer/handle does not exist */ + + file_ptr->rw_file = H5FD_open(name, flags, fapl_ptr->rw_fapl_id, HADDR_UNDEF); + if(!file_ptr->rw_file) + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "unable to open R/W file") + + file_ptr->wo_file = H5FD_open(fapl_ptr->wo_path, flags, fapl_ptr->wo_fapl_id, HADDR_UNDEF); + if(!file_ptr->wo_file) + H5FD_SPLITTER_WO_ERROR(file_ptr, FUNC, H5E_VFL, H5E_CANTOPENFILE, NULL, "unable to open W/O file") + + ret_value = (H5FD_t*)file_ptr; + +done: + if(NULL == ret_value) { + if(file_ptr) { + if(H5I_INVALID_HID != file_ptr->fa.rw_fapl_id) + H5I_dec_ref(file_ptr->fa.rw_fapl_id); + if(H5I_INVALID_HID != file_ptr->fa.wo_fapl_id) + H5I_dec_ref(file_ptr->fa.wo_fapl_id); + if(file_ptr->rw_file) + H5FD_close(file_ptr->rw_file); + if(file_ptr->wo_file) + H5FD_close(file_ptr->wo_file); + if(file_ptr->logfp) + HDfclose(file_ptr->logfp); + H5FL_FREE(H5FD_splitter_t, file_ptr); + } + } /* end if error */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_open() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_close + * + * Purpose: Closes files on both read-write and write-only channels. + * + * Return: Success: SUCCEED + * Failure: FAIL, file not closed. + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_close(H5FD_t *_file) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Sanity check */ + HDassert(file); + + if(H5I_dec_ref(file->fa.rw_fapl_id) < 0) + HGOTO_ERROR(H5E_VFL, H5E_ARGS, FAIL, "can't close R/W FAPL") + if(H5I_dec_ref(file->fa.wo_fapl_id) < 0) + HGOTO_ERROR(H5E_VFL, H5E_ARGS, FAIL, "can't close W/O FAPL") + + if(file->rw_file) + if(H5FD_close(file->rw_file) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close R/W file") + if(file->wo_file) + if(H5FD_close(file->wo_file) == FAIL) + H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close W/O file") + + if(file->logfp) { + HDfclose(file->logfp); + file->logfp = NULL; + } + + /* Release the file info */ + file = H5FL_FREE(H5FD_splitter_t, file); + file = NULL; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_close() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_get_eoa + * + * Purpose: Returns the end-of-address marker for the file. The EOA + * marker is the first address past the last byte allocated in + * the format address space. + * + * Return: Success: The end-of-address-marker + * + * Failure: HADDR_UNDEF + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD__splitter_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_splitter_t *file = (const H5FD_splitter_t *)_file; + haddr_t ret_value = HADDR_UNDEF; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Sanity check */ + HDassert(file); + HDassert(file->rw_file); + + if((ret_value = H5FD_get_eoa(file->rw_file, type)) == HADDR_UNDEF) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, HADDR_UNDEF, "unable to get eoa") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_get_eoa */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_set_eoa + * + * Purpose: Set the end-of-address marker for the file. This function is + * called shortly after an existing HDF5 file is opened in order + * to tell the driver where the end of the HDF5 data is located. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC) + + /* Sanity check */ + HDassert(file); + HDassert(file->rw_file); + HDassert(file->wo_file); + + if(H5FD_set_eoa(file->rw_file, type, addr) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "H5FDset_eoa failed for R/W file") + + if(H5FD_set_eoa(file->wo_file, type, addr) < 0) + H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_CANTSET, FAIL, "unable to set EOA for W/O file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_set_eoa() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_get_eof + * + * Purpose: Returns the end-of-address marker for the file. The EOA + * marker is the first address past the last byte allocated in + * the format address space. + * + * Return: Success: The end-of-address-marker + * + * Failure: HADDR_UNDEF + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD__splitter_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_splitter_t *file = (const H5FD_splitter_t *)_file; + haddr_t ret_value = HADDR_UNDEF; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Sanity check */ + HDassert(file); + HDassert(file->rw_file); + + if(HADDR_UNDEF == (ret_value = H5FD_get_eof(file->rw_file, type))) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, HADDR_UNDEF, "unable to get eof") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_get_eof */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_truncate + * + * Purpose: Notify driver to truncate the file back to the allocated size. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + HDassert(file); + HDassert(file->rw_file); + HDassert(file->wo_file); + + if(H5FDtruncate(file->rw_file, dxpl_id, closing) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "unable to truncate R/W file") + + if(H5FDtruncate(file->wo_file, dxpl_id, closing) < 0) + H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_CANTUPDATE, FAIL, "unable to truncate W/O file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_truncate */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_sb_size + * + * Purpose: Obtains the number of bytes required to store the driver file + * access data in the HDF5 superblock. + * + * Return: Success: Number of bytes required. + * + * Failure: 0 if an error occurs or if the driver has no + * data to store in the superblock. + * + * NOTE: no public API for H5FD_sb_size, it needs to be added + *------------------------------------------------------------------------- + */ +static hsize_t +H5FD__splitter_sb_size(H5FD_t *_file) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + hsize_t ret_value = 0; + + FUNC_ENTER_STATIC_NOERR + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Sanity check */ + HDassert(file); + HDassert(file->rw_file); + + if(file->rw_file) + ret_value = H5FD_sb_size(file->rw_file); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_sb_size */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_sb_encode + * + * Purpose: Encode driver-specific data into the output arguments. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_sb_encode(H5FD_t *_file, char *name/*out*/, unsigned char *buf/*out*/) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Sanity check */ + HDassert(file); + HDassert(file->rw_file); + + if(file->rw_file && H5FD_sb_encode(file->rw_file, name, buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTENCODE, FAIL, "unable to encode the superblock in R/W file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_sb_encode */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_sb_decode + * + * Purpose: Decodes the driver information block. + * + * Return: SUCCEED/FAIL + * + * NOTE: no public API for H5FD_sb_size, need to add + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_sb_decode(H5FD_t *_file, const char *name, const unsigned char *buf) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Sanity check */ + HDassert(file); + HDassert(file->rw_file); + + if(H5FD_sb_load(file->rw_file, name, buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "unable to decode the superblock in R/W file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_sb_decode */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_cmp + * + * Purpose: Compare the keys of two files. + * + * Return: Success: A value like strcmp() + * Failure: Must never fail + *------------------------------------------------------------------------- + */ +static int +H5FD__splitter_cmp(const H5FD_t *_f1, const H5FD_t *_f2) +{ + const H5FD_splitter_t *f1 = (const H5FD_splitter_t *)_f1; + const H5FD_splitter_t *f2 = (const H5FD_splitter_t *)_f2; + herr_t ret_value = 0; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + H5FD_SPLITTER_LOG_CALL(FUNC); + + HDassert(f1); + HDassert(f2); + + ret_value = H5FD_cmp(f1->rw_file, f2->rw_file); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_cmp */ + + +/*-------------------------------------------------------------------------- + * Function: H5FD__splitter_get_handle + * + * Purpose: Returns a pointer to the file handle of low-level virtual + * file driver. + * + * Return: SUCCEED/FAIL + *-------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_get_handle(H5FD_t *_file, hid_t H5_ATTR_UNUSED fapl, + void **file_handle) +{ + H5FD_splitter_t *file = (H5FD_splitter_t*)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Check arguments */ + HDassert(file); + HDassert(file->rw_file); + HDassert(file_handle); + + /* Only do for R/W channel */ + if(H5FD_get_vfd_handle(file->rw_file, file->fa.rw_fapl_id, file_handle) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "unable to get handle of R/W file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_get_handle */ + + +/*-------------------------------------------------------------------------- + * Function: H5FD__splitter_lock + * + * Purpose: Sets a file lock. + * + * Return: SUCCEED/FAIL + *-------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_lock(H5FD_t *_file, hbool_t rw) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; /* VFD file struct */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + HDassert(file); + HDassert(file->rw_file); + + /* Place the lock on each file */ + if(H5FD_lock(file->rw_file, rw) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock R/W file") + + if(file->wo_file != NULL) + if(H5FD_lock(file->wo_file, rw) < 0) + H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock W/O file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_lock */ + + +/*-------------------------------------------------------------------------- + * Function: H5FD__splitter_unlock + * + * Purpose: Removes a file lock. + * + * Return: SUCCEED/FAIL + *-------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_unlock(H5FD_t *_file) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; /* VFD file struct */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Check arguments */ + HDassert(file); + HDassert(file->rw_file); + + /* Remove the lock on each file */ + if(H5FD_unlock(file->rw_file) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock R/W file") + + if(file->wo_file != NULL) + if(H5FD_unlock(file->wo_file) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock W/O file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_unlock */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_query + * + * Purpose: Set the flags that this VFL driver is capable of supporting. + * (listed in H5FDpublic.h) + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_query(const H5FD_t *_file, unsigned long *flags /* out */) +{ + const H5FD_splitter_t *file = (const H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + if(file) { + HDassert(file); + HDassert(file->rw_file); + + if(H5FDquery(file->rw_file, flags) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCK, FAIL, "unable to query R/W file"); + } + else { + /* There is no file. Because this is a pure passthrough VFD, + * it has no features of its own. + */ + if(flags) + *flags = 0; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_query() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_alloc + * + * Purpose: Allocate file memory. + * + * Return: Address of allocated space (HADDR_UNDEF if error). + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD__splitter_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; /* VFD file struct */ + haddr_t ret_value = HADDR_UNDEF; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Check arguments */ + HDassert(file); + HDassert(file->rw_file); + + /* Allocate memory for each file, only return the return value for R/W file. */ + if((ret_value = H5FDalloc(file->rw_file, type, dxpl_id, size)) == HADDR_UNDEF) + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "unable to allocate for R/W file") + + if(H5FDalloc(file->wo_file, type, dxpl_id, size) == HADDR_UNDEF) + H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "unable to alloc for W/O file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_alloc() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_get_type_map + * + * Purpose: Retrieve the memory type mapping for this file + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_get_type_map(const H5FD_t *_file, H5FD_mem_t *type_map) +{ + const H5FD_splitter_t *file = (const H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Check arguments */ + HDassert(file); + HDassert(file->rw_file); + + /* Retrieve memory type mapping for R/W channel only */ + if(H5FD_get_fs_type_map(file->rw_file, type_map) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "unable to allocate for R/W file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_get_type_map() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_free + * + * Purpose: Free the resources for the splitter VFD. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_free(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t size) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; /* VFD file struct */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Check arguments */ + HDassert(file); + HDassert(file->rw_file); + + if(H5FDfree(file->rw_file, type, dxpl_id, addr, size) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to free for R/W file") + + if(H5FDfree(file->wo_file, type, dxpl_id, addr, size) < 0) + H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_CANTINIT, FAIL, "unable to free for W/O file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_free() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_log_error + * + * Purpose: Log an error from the W/O channel appropriately. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_log_error(const H5FD_splitter_t *file, const char *atfunc, const char *msg) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC_NOERR + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Check arguments */ + HDassert(file); + HDassert(atfunc && *atfunc); + HDassert(msg && *msg); + + if(file->logfp != NULL) { + size_t size; + char *s; + + size = HDstrlen(atfunc) + HDstrlen(msg) + 3; /* ':', ' ', '\n' */ + s = (char *)HDmalloc(sizeof(char) * (size + 1)); + if(NULL == s) + ret_value = FAIL; + else if(size < (size_t)HDsnprintf(s, size+1, "%s: %s\n", atfunc, msg)) + ret_value = FAIL; + else if(size != HDfwrite(s, 1, size, file->logfp)) + ret_value = FAIL; + HDfree(s); + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_log_error() */ + diff --git a/src/H5FDsplitter.h b/src/H5FDsplitter.h new file mode 100644 index 0000000..5a5ef29 --- /dev/null +++ b/src/H5FDsplitter.h @@ -0,0 +1,99 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: The public header file for the "splitter" driver. + */ + +#ifndef H5FDsplitter_H +#define H5FDsplitter_H + +#define H5FD_SPLITTER (H5FD_splitter_init()) + +/* The version of the H5FD_splitter_vfd_config_t structure used */ +#define H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION 1 + +/* Maximum length of a filename/path string in the Write-Only channel, + * including the NULL-terminator. + */ +#define H5FD_SPLITTER_PATH_MAX 4096 + +/* Semi-unique constant used to help identify structure pointers */ +#define H5FD_SPLITTER_MAGIC 0x2B916880 + +/* ---------------------------------------------------------------------------- + * Structure: H5FD_spliiter_vfd_config_t + * + * One-stop shopping for configuring a Splitter VFD (rather than many + * paramaters passed into H5Pset/get functions). + * + * magic (int32_t) + * Semi-unique number, used to sanity-check that a given pointer is + * likely (or not) to be this structure type. MUST be first. + * If magic is not H5FD_SPLITTER_MAGIC, the structure (and/or pointer to) + * must be considered invalid. + * + * version (unsigned int) + * Version number of this structure -- informs component membership. + * If not H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION, the structure (and/or + * pointer to) must be considered invalid. + * + * rw_fapl_id (hid_t) + * Library-given identification number of the Read/Write channel driver + * File Access Property List. + * The driver must support read/write access. + * Must be set to H5P_DEFAULT or a valid FAPL ID. + * + * wo_fapl_id (hid_t) + * Library-given identification number of the Read/Write channel driver + * File Access Property List. + * The driver feature flags must include H5FD_FEAT_DEFAULT_VFD_COMPAITBLE. + * Must be set to H5P_DEFAULT or a valid FAPL ID. + * + * wo_file_path (char[H5FD_SPLITTER_PATH_MAX + 1]) + * String buffer for the Write-Only channel target file. + * Must be null-terminated, cannot be empty. + * + * log_file_path (char[H5FD_SPLITTER_PATH_MAX + 1]) + * String buffer for the Splitter VFD logging output. + * Must be null-terminated. + * If null, no logfile is created. + * + * ignore_wo_errors (hbool_t) + * Toggle flag for how judiciously to respond to errors on the Write-Only + * channel. + * + * ---------------------------------------------------------------------------- + */ +typedef struct H5FD_splitter_vfd_config_t { + int32_t magic; + unsigned int version; + hid_t rw_fapl_id; + hid_t wo_fapl_id; + char wo_path[H5FD_SPLITTER_PATH_MAX + 1]; + char log_file_path[H5FD_SPLITTER_PATH_MAX + 1]; + hbool_t ignore_wo_errs; +} H5FD_splitter_vfd_config_t; + +#ifdef __cplusplus +extern "C" { +#endif +H5_DLL hid_t H5FD_splitter_init(void); +H5_DLL herr_t H5Pset_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *config_ptr); +H5_DLL herr_t H5Pget_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *config_ptr); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/src/H5FDstdio.c b/src/H5FDstdio.c index d29a1b4..a769e86 100644 --- a/src/H5FDstdio.c +++ b/src/H5FDstdio.c @@ -52,6 +52,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_STDIO_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = -1; + /* The maximum number of bytes which can be written in a single I/O operation */ static size_t H5_STDIO_MAX_IO_BYTES_g = (size_t)-1; @@ -82,6 +85,7 @@ typedef struct H5FD_stdio_t { haddr_t eof; /* end of file; current file size */ haddr_t pos; /* current file I/O position */ unsigned write_access; /* Flag to indicate the file was opened with write access */ + hbool_t ignore_disabled_file_locks; H5FD_stdio_file_op op; /* last operation */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely @@ -231,11 +235,23 @@ static const H5FD_class_t H5FD_stdio_g = { hid_t H5FD_stdio_init(void) { + char *lock_env_var = NULL; /* Environment variable pointer */ + /* Clear the error stack */ H5Eclear2(H5E_DEFAULT); - if (H5I_VFL!=H5Iget_type(H5FD_STDIO_g)) + /* Check the use disabled file locks environment variable */ + lock_env_var = getenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && !strcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = 1; /* Override: Ignore disabled locks */ + else if(lock_env_var && (!strcmp(lock_env_var, "TRUE") || !strcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = 0; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = -1; /* Environment variable not set, or not set correctly */ + + if (H5I_VFL != H5Iget_type(H5FD_STDIO_g)) H5FD_STDIO_g = H5FDregister(&H5FD_stdio_g); + return H5FD_STDIO_g; } /* end H5FD_stdio_init() */ @@ -318,7 +334,7 @@ H5Pset_fapl_stdio(hid_t fapl_id) *------------------------------------------------------------------------- */ static H5FD_t * -H5FD_stdio_open( const char *name, unsigned flags, hid_t /*UNUSED*/ fapl_id, +H5FD_stdio_open( const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) { FILE *f = NULL; @@ -396,6 +412,21 @@ H5FD_stdio_open( const char *name, unsigned flags, hid_t /*UNUSED*/ fapl_id, file->eof = (haddr_t)x; } + /* Check the file locking flags in the fapl */ + if(ignore_disabled_file_locks_s != -1) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + hbool_t unused; + + /* Use the value in the property list */ + if(H5Pget_file_locking(fapl_id, &unused, &file->ignore_disabled_file_locks) < 0) { + free(file); + fclose(f); + H5Epush_ret(func, H5E_ERR_CLS, H5E_FILE, H5E_CANTGET, "unable to get use disabled file locks property", NULL); + } + } + /* Get the file descriptor (needed for truncate and some Windows information) */ #ifdef H5_HAVE_WIN32_API file->fd = _fileno(file->fp); @@ -1104,10 +1135,13 @@ H5FD_stdio_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if(flock(file->fd, lock_flags | LOCK_NB) < 0) { - if(ENOSYS == errno) - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)", -1) + if(file->ignore_disabled_file_locks && ENOSYS == errno) + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; else - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file lock failed", -1) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTLOCKFILE, "file lock failed", -1) } /* end if */ /* Flush the stream */ @@ -1152,10 +1186,13 @@ H5FD_stdio_unlock(H5FD_t *_file) /* Place a non-blocking lock on the file */ if(flock(file->fd, LOCK_UN) < 0) { - if(ENOSYS == errno) - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)", -1) + if(file->ignore_disabled_file_locks && ENOSYS == errno) + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; else - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file unlock failed", -1) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTUNLOCKFILE, "file unlock failed", -1) } /* end if */ #endif /* H5_HAVE_FLOCK */ diff --git a/src/H5Fint.c b/src/H5Fint.c index eb023d6..29bbd45 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -77,6 +77,7 @@ static herr_t H5F__get_objects(const H5F_t *f, unsigned types, size_t max_index, static int H5F__get_objects_cb(void *obj_ptr, hid_t obj_id, void *key); static herr_t H5F__build_name(const char *prefix, const char *file_name, char **full_name/*out*/); static char *H5F__getenv_prefix_name(char **env_prefix/*in,out*/); +static herr_t H5F__check_if_using_file_locks(H5P_genplist_t *fapl, hbool_t *use_file_locking); static herr_t H5F__build_actual_name(const H5F_t *f, const H5P_genplist_t *fapl, const char *name, char ** /*out*/ actual_name); static herr_t H5F__flush_phase1(H5F_t *f); static herr_t H5F__flush_phase2(H5F_t *f, hbool_t closing); @@ -86,6 +87,15 @@ static herr_t H5F__flush_phase2(H5F_t *f, hbool_t closing); /* Package Variables */ /*********************/ +/* Package initialization variable */ +hbool_t H5_PKG_INIT_VAR = FALSE; + +/* Based on the value of the HDF5_USE_FILE_LOCKING environment variable. + * TRUE/FALSE have obvious meanings. FAIL means the environment variable was + * not set, so the code should ignore it and use the fapl value instead. + */ +htri_t use_locks_env_g = FAIL; + /*****************************/ /* Library Private Variables */ @@ -102,6 +112,123 @@ H5FL_DEFINE(H5F_t); /* Declare a free list to manage the H5F_shared_t struct */ H5FL_DEFINE(H5F_shared_t); +/* File ID class */ +static const H5I_class_t H5I_FILE_CLS[1] = {{ + H5I_FILE, /* ID class value */ + 0, /* Class flags */ + 0, /* # of reserved IDs for class */ + (H5I_free_t)H5F__close_cb /* Callback routine for closing objects of this class */ +}}; + + + +/*-------------------------------------------------------------------------- +NAME + H5F__init_package -- Initialize interface-specific information +USAGE + herr_t H5F__init_package() +RETURNS + Non-negative on success/Negative on failure +DESCRIPTION + Initializes any interface-specific data or routines. + +--------------------------------------------------------------------------*/ +herr_t +H5F__init_package(void) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* + * Initialize the atom group for the file IDs. + */ + if(H5I_register_type(H5I_FILE_CLS) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to initialize interface") + + /* Check the file locking environment variable */ + if(H5F__parse_file_lock_env_var(&use_locks_env_g) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to parse file locking environment variable") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5F__init_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_term_package + * + * Purpose: Terminate this interface: free all memory and reset global + * variables to their initial values. Release all ID groups + * associated with this interface. + * + * Return: Success: Positive if anything was done that might + * have affected other interfaces; + * zero otherwise. + * + * Failure: Never fails + * + *------------------------------------------------------------------------- + */ +int +H5F_term_package(void) +{ + int n = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + if(H5_PKG_INIT_VAR) { + if(H5I_nmembers(H5I_FILE) > 0) { + (void)H5I_clear_type(H5I_FILE, FALSE, FALSE); + n++; /*H5I*/ + } /* end if */ + else { + /* Make certain we've cleaned up all the shared file objects */ + H5F_sfile_assert_num(0); + + /* Destroy the file object id group */ + n += (H5I_dec_type_ref(H5I_FILE) > 0); + + /* Mark closed */ + if(0 == n) + H5_PKG_INIT_VAR = FALSE; + } /* end else */ + } /* end if */ + + FUNC_LEAVE_NOAPI(n) +} /* end H5F_term_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5F__parse_file_lock_env_var + * + * Purpose: Parses the HDF5_USE_FILE_LOCKING environment variable. + * + * NOTE: This is done in a separate function so we can call it from + * the test code. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5F__parse_file_lock_env_var(htri_t *use_locks) +{ + char *lock_env_var = NULL; /* Environment variable pointer */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Check the file locking environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && (!HDstrcmp(lock_env_var, "FALSE") || !HDstrcmp(lock_env_var, "0"))) + *use_locks = FALSE; /* Override: Never use locks */ + else if(lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "BEST_EFFORT") || !HDstrcmp(lock_env_var, "1"))) + *use_locks = TRUE; /* Override: Always use locks */ + else + *use_locks = FAIL; /* Environment variable not set, or not set correctly */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5F__parse_file_lock_env_var() */ /*------------------------------------------------------------------------- @@ -1334,6 +1461,52 @@ H5F__dest(H5F_t *f, hbool_t flush) /*------------------------------------------------------------------------- + * Function: H5F__check_if_using_file_locks + * + * Purpose: Determines if this file will use file locks. + * + * There are three ways that file locking can be controlled: + * + * 1) The configure/cmake option that sets the H5_USE_FILE_LOCKING + * symbol (which is used as the default fapl value). + * + * 2) The H5Pset_file_locking() API call, which will override + * the configuration default. + * + * 3) The HDF5_USE_FILE_LOCKING environment variable, which overrides + * everything above. + * + * The main reason to disable file locking is to prevent errors on file + * systems where locking is not supported or has been disabled (as is + * often the case in parallel file systems). + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5F__check_if_using_file_locks(H5P_genplist_t *fapl, hbool_t *use_file_locking) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Make sure the out parameter has a value */ + *use_file_locking = TRUE; + + /* Check the fapl property */ + if(H5P_get(fapl, H5F_ACS_USE_FILE_LOCKING_NAME, use_file_locking) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get use file locking flag") + + /* Check the environment variable */ + if(use_locks_env_g != FAIL) + *use_file_locking = (use_locks_env_g == TRUE) ? TRUE : FALSE; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__check_if_using_file_locks() */ + + +/*------------------------------------------------------------------------- * Function: H5F_open * * Purpose: Opens (or creates) a file. This function understands the @@ -1422,8 +1595,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) hbool_t set_flag = FALSE; /*set the status_flags in the superblock */ hbool_t clear = FALSE; /*clear the status_flags */ hbool_t evict_on_close; /* evict on close value from plist */ - char *lock_env_var = NULL;/*env var pointer */ - hbool_t use_file_locking; /*read from env var */ + hbool_t use_file_locking = TRUE; /* Using file locks? */ hbool_t ci_load = FALSE; /* whether MDC ci load requested */ hbool_t ci_write = FALSE; /* whether MDC CI write requested */ H5F_t *ret_value = NULL; /*actual return value */ @@ -1441,15 +1613,13 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) if(NULL == (drvr = H5FD_get_class(fapl_id))) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to retrieve VFL class") - /* Check the environment variable that determines if we care - * about file locking. File locking should be used unless explicitly - * disabled. - */ - lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); - if(lock_env_var && !HDstrcmp(lock_env_var, "FALSE")) - use_file_locking = FALSE; - else - use_file_locking = TRUE; + /* Get the file access property list, for future queries */ + if(NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list") + + /* Check if we are using file locking */ + if (H5F__check_if_using_file_locks(a_plist, &use_file_locking) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to get file locking flag") /* * Opening a file is a two step process. First we try to open the @@ -1540,8 +1710,8 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) if(H5FD_lock(lf, (hbool_t)((flags & H5F_ACC_RDWR) ? TRUE : FALSE)) < 0) { /* Locking failed - Closing will remove the lock */ if(H5FD_close(lf) < 0) - HDONE_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to close low-level file info") - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to lock the file") + HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, NULL, "unable to close low-level file info") + HGOTO_ERROR(H5E_FILE, H5E_CANTLOCKFILE, NULL, "unable to lock the file") } /* end if */ /* Create the 'top' file structure */ @@ -1573,10 +1743,6 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) shared = file->shared; lf = shared->lf; - /* Get the file access property list, for future queries */ - if(NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list") - /* Check if page buffering is enabled */ if(H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &page_buf_size) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get page buffer size") @@ -1715,7 +1881,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) /* Remove the file lock for SWMR_WRITE */ if(use_file_locking && (H5F_INTENT(file) & H5F_ACC_SWMR_WRITE)) { if(H5FD_unlock(file->shared->lf) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to unlock the file") + HGOTO_ERROR(H5E_FILE, H5E_CANTUNLOCKFILE, NULL, "unable to unlock the file") } /* end if */ } /* end if */ else { /* H5F_ACC_RDONLY: check consistency of status_flags */ diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index e4892ed..6e3f3d4 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -389,6 +389,10 @@ H5FL_EXTERN(H5F_t); /* Declare a free list to manage the H5F_shared_t struct */ H5FL_EXTERN(H5F_shared_t); +/* Whether or not to use file locking (based on the environment variable) + * FAIL means ignore the environment variable. + */ +H5_DLLVAR htri_t use_locks_env_g; /******************************/ /* Package Private Prototypes */ @@ -407,6 +411,7 @@ H5_DLL herr_t H5F__start_swmr_write(H5F_t *f); H5_DLL herr_t H5F__close(hid_t file_id); H5_DLL herr_t H5F__close_cb(H5F_t *f); H5_DLL herr_t H5F__set_libver_bounds(H5F_t *f, H5F_libver_t low, H5F_libver_t high); +H5_DLL herr_t H5F__parse_file_lock_env_var(htri_t *use_locks); /* File mount related routines */ H5_DLL herr_t H5F__mount(H5G_loc_t *loc, const char *name, H5F_t *child, hid_t plist_id); @@ -466,6 +471,7 @@ H5_DLL herr_t H5F__get_sohm_mesg_count_test(hid_t fid, unsigned type_id, size_t H5_DLL herr_t H5F__check_cached_stab_test(hid_t file_id); H5_DLL herr_t H5F__get_maxaddr_test(hid_t file_id, haddr_t *maxaddr); H5_DLL herr_t H5F__get_sbe_addr_test(hid_t file_id, haddr_t *sbe_addr); +H5_DLL herr_t H5F__reparse_file_lock_variable_test(void); #endif /* H5F_TESTING */ #endif /* _H5Fpkg_H */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 57318a7..fd161e2 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -516,6 +516,8 @@ typedef struct H5F_t H5F_t; #define H5F_ACS_PAGE_BUFFER_SIZE_NAME "page_buffer_size" /* the maximum size for the page buffer cache */ #define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME "page_buffer_min_meta_perc" /* the min metadata percentage for the page buffer cache */ #define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME "page_buffer_min_raw_perc" /* the min raw data percentage for the page buffer cache */ +#define H5F_ACS_USE_FILE_LOCKING_NAME "use_file_locking" /* whether or not we use file locks for SWMR control and to prevent multiple writers */ +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME "ignore_disabled_file_locks" /* whether or not we ignore "locks disabled" errors */ /* ======================== File Mount properties ====================*/ #define H5F_MNT_SYM_LOCAL_NAME "local" /* Whether absolute symlinks local to file. */ diff --git a/src/H5Ftest.c b/src/H5Ftest.c index df9c933..e9d697a 100644 --- a/src/H5Ftest.c +++ b/src/H5Ftest.c @@ -232,3 +232,36 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F__get_sbe_addr_test() */ + +/*------------------------------------------------------------------------- + * Function: H5F__reparse_file_lock_variable_test + * + * Purpose: Re-parse the file locking environment variable. + * + * Since getenv(3) is fairly expensive, we only parse it once, + * when the library opens. This test function is used to + * re-parse the environment variable after we've changed it + * with setnev(3). + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Summer 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F__reparse_file_lock_variable_test(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check the file locking environment variable */ + if(H5F__parse_file_lock_env_var(&use_locks_env_g) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to parse file locking environment variable") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__reparse_file_lock_variable_test() */ + diff --git a/src/H5Gpublic.h b/src/H5Gpublic.h index 170b74d..3f74d45 100644 --- a/src/H5Gpublic.h +++ b/src/H5Gpublic.h @@ -15,7 +15,7 @@ * * Created: H5Gpublic.h * Jul 11 1997 - * Robb Matzke <matzke@llnl.gov> + * Robb Matzke * * Purpose: Public declarations for the H5G package * diff --git a/src/H5Opublic.h b/src/H5Opublic.h index dcbf85a..2ef0c0d 100644 --- a/src/H5Opublic.h +++ b/src/H5Opublic.h @@ -15,7 +15,7 @@ * * Created: H5Opublic.h * Aug 5 1997 - * Robb Matzke <matzke@llnl.gov> + * Robb Matzke * * Purpose: Public declarations for the H5O (object header) * package. diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index b2a1a30..208c8dd 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -259,6 +259,29 @@ #define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF 0 #define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_ENC H5P__encode_unsigned #define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEC H5P__decode_unsigned +/* Definition for using file locking or not. The default is set + * via the configure step. + */ +#define H5F_ACS_USE_FILE_LOCKING_SIZE sizeof(hbool_t) +#if defined H5_USE_FILE_LOCKING && H5_USE_FILE_LOCKING +#define H5F_ACS_USE_FILE_LOCKING_DEF TRUE +#else +#define H5F_ACS_USE_FILE_LOCKING_DEF FALSE +#endif +#define H5F_ACS_USE_FILE_LOCKING_ENC H5P__encode_hbool_t +#define H5F_ACS_USE_FILE_LOCKING_DEC H5P__decode_hbool_t +/* Definition for whether we ignore file locking errors when we can + * tell that file locking has been disabled on the file system. + * The default is set via the configure step. + */ +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_SIZE sizeof(hbool_t) +#if defined H5_IGNORE_DISABLED_FILE_LOCKS && H5_IGNORE_DISABLED_FILE_LOCKS +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF TRUE +#else +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF FALSE +#endif +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_ENC H5P__encode_hbool_t +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEC H5P__decode_hbool_t /******************/ @@ -398,6 +421,8 @@ static const H5AC_cache_image_config_t H5F_def_mdc_initCacheImageCfg_g = H5F_ACS static const size_t H5F_def_page_buf_size_g = H5F_ACS_PAGE_BUFFER_SIZE_DEF; /* Default page buffer size */ static const unsigned H5F_def_page_buf_min_meta_perc_g = H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEF; /* Default page buffer minimum metadata size */ static const unsigned H5F_def_page_buf_min_raw_perc_g = H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF; /* Default page buffer mininum raw data size */ +static const hbool_t H5F_def_use_file_locking_g = H5F_ACS_USE_FILE_LOCKING_DEF; /* Default use file locking flag */ +static const hbool_t H5F_def_ignore_disabled_file_locks_g = H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF; /* Default ignore disabled file locks flag */ /*------------------------------------------------------------------------- @@ -647,6 +672,16 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the use file locking flag */ + if(H5P__register_real(pclass, H5F_ACS_USE_FILE_LOCKING_NAME, H5F_ACS_USE_FILE_LOCKING_SIZE, &H5F_def_use_file_locking_g, + NULL, NULL, NULL, H5F_ACS_USE_FILE_LOCKING_ENC, H5F_ACS_USE_FILE_LOCKING_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the ignore disabled file locks flag */ + if(H5P__register_real(pclass, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_SIZE, &H5F_def_ignore_disabled_file_locks_g, + NULL, NULL, NULL, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_ENC, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__facc_reg_prop() */ @@ -4586,6 +4621,98 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Pget_evict_on_close() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_file_locking + * + * Purpose: Sets the file locking property values. + * + * Overrides the default file locking flag setting that was + * set when the library was configured. + * + * Can be overridden by the HDF5_USE_FILE_LOCKING environment + * variable. + * + * File locking is used when creating/opening a file to prevent + * problematic file accesses. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Spring 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_file_locking(hid_t fapl_id, hbool_t use_file_locking, hbool_t ignore_when_disabled) +{ + H5P_genplist_t *plist; /* property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "ibb", fapl_id, use_file_locking, ignore_when_disabled); + + /* Make sure this is a fapl */ + if(TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not a file access plist") + + /* Get the plist structure */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Set values */ + if(H5P_set(plist, H5F_ACS_USE_FILE_LOCKING_NAME, &use_file_locking) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set use file locking property") + if(H5P_set(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &ignore_when_disabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set ignore disabled file locks property") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_file_locking() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_file_locking + * + * Purpose: Gets the file locking property values. + * + * File locking is used when creating/opening a file to prevent + * problematic file accesses. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Spring 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_file_locking(hid_t fapl_id, hbool_t *use_file_locking, hbool_t *ignore_when_disabled) +{ + H5P_genplist_t *plist; /* property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "i*b*b", fapl_id, use_file_locking, ignore_when_disabled); + + /* Make sure this is a fapl */ + if(TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not an access plist") + + /* Get the plist structure */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get values */ + if(H5P_get(plist, H5F_ACS_USE_FILE_LOCKING_NAME, use_file_locking) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get use file locking property") + if(H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, ignore_when_disabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get ignore disabled file locks property") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_file_locking() */ + #ifdef H5_HAVE_PARALLEL /*------------------------------------------------------------------------- diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 5d6dea8..dbd0724 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -359,6 +359,8 @@ H5_DLL herr_t H5Pset_mdc_log_options(hid_t plist_id, hbool_t is_enabled, const c H5_DLL herr_t H5Pget_mdc_log_options(hid_t plist_id, hbool_t *is_enabled, char *location, size_t *location_size, hbool_t *start_on_access); H5_DLL herr_t H5Pset_evict_on_close(hid_t fapl_id, hbool_t evict_on_close); H5_DLL herr_t H5Pget_evict_on_close(hid_t fapl_id, hbool_t *evict_on_close); +H5_DLL herr_t H5Pset_file_locking(hid_t fapl_id, hbool_t use_file_locking, hbool_t ignore_when_disabled); +H5_DLL herr_t H5Pget_file_locking(hid_t fapl_id, hbool_t *use_file_locking, hbool_t *ignore_when_disabled); #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5Pset_all_coll_metadata_ops(hid_t plist_id, hbool_t is_collective); H5_DLL herr_t H5Pget_all_coll_metadata_ops(hid_t plist_id, hbool_t *is_collective); @@ -1815,12 +1815,6 @@ done: * Programmer: Robb Matzke * Friday, January 9, 1998 * - * Modifications: - * - * Robb Matzke, 1 Jun 1998 - * It is illegal to lock a named datatype since we must allow named - * types to be closed (to release file resources) but locking a type - * prevents that. *------------------------------------------------------------------------- */ herr_t @@ -1893,9 +1887,6 @@ done: * Programmer: Robb Matzke * Monday, December 8, 1997 * - * Modifications: - * Broke out from H5Tget_class - QAK - 6/4/99 - * *------------------------------------------------------------------------- */ H5T_class_t @@ -2882,10 +2873,6 @@ done: * Programmer: Raymond Lu * July 14, 2004 * - * Modification:Raymond Lu - * 17 February 2011 - * I changed the value for the APP_REF parameter of H5I_register - * from FALSE to TRUE. *------------------------------------------------------------------------- */ hid_t @@ -3046,10 +3033,6 @@ done: * Programmer: Robb Matzke * Friday, December 5, 1997 * - * Modifications: - * Raymond Lu - * 19 May 2011 - * We support fixed size or variable-length string now. *------------------------------------------------------------------------- */ H5T_t * @@ -5055,11 +5038,6 @@ H5T_path_noop(const H5T_path_t *p) * Programmer: Raymond Lu * 8 June 2007 * - * Modifications: Neil Fortner - * 19 September 2008 - * Changed return value to H5T_subset_info_t - * (to allow it to return copy_size) - * *------------------------------------------------------------------------- */ H5T_subset_info_t * @@ -5154,15 +5132,18 @@ H5T_convert(H5T_path_t *tpath, hid_t src_id, hid_t dst_id, size_t nelmts, size_t buf_stride, size_t bkg_stride, void *buf, void *bkg) { #ifdef H5T_DEBUG - H5_timer_t timer; + H5_timer_t timer; /* Timer for conversion */ #endif herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) #ifdef H5T_DEBUG - if(H5DEBUG(T)) - H5_timer_begin(&timer); + if(H5DEBUG(T)) { + /* Initialize and start timer */ + H5_timer_init(&timer); + H5_timer_start(&timer); + } /* end if */ #endif /* Call the appropriate conversion callback */ @@ -5176,10 +5157,16 @@ H5T_convert(H5T_path_t *tpath, hid_t src_id, hid_t dst_id, size_t nelmts, HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, "datatype conversion failed") #ifdef H5T_DEBUG if(H5DEBUG(T)) { - H5_timer_end(&(tpath->stats.timer), &timer); + /* Stop timer */ + H5_timer_stop(&timer); + + /* Record elapsed timer info */ + H5_timer_get_times(timer, &tpath->stats.times); + + /* Increment # of calls and # of elements converted */ tpath->stats.ncalls++; tpath->stats.nelmts += nelmts; - } + } /* end if */ #endif done: diff --git a/src/H5Tdbg.c b/src/H5Tdbg.c index eb648f3..6188138 100644 --- a/src/H5Tdbg.c +++ b/src/H5Tdbg.c @@ -13,11 +13,11 @@ /*------------------------------------------------------------------------- * - * Created: H5Tdbg.c - * Jul 19 2007 - * Quincey Koziol <koziol@hdfgroup.org> + * Created: H5Tdbg.c + * Jul 19 2007 + * Quincey Koziol * - * Purpose: Dump debugging information about a datatype + * Purpose: Dump debugging information about a datatype * *------------------------------------------------------------------------- */ @@ -95,25 +95,24 @@ herr_t H5T__print_stats(H5T_path_t H5_ATTR_UNUSED * path, int H5_ATTR_UNUSED * nprint/*in,out*/) { -#ifdef H5T_DEBUG - hsize_t nbytes; - char bandwidth[32]; -#endif - FUNC_ENTER_PACKAGE_NOERR #ifdef H5T_DEBUG - if (H5DEBUG(T) && path->stats.ncalls > 0) { - if (nprint && 0 == (*nprint)++) { - HDfprintf(H5DEBUG(T), "H5T: type conversion statistics:\n"); - HDfprintf(H5DEBUG(T), " %-16s %10s %10s %8s %8s %8s %10s\n", - "Conversion", "Elmts", "Calls", "User", - "System", "Elapsed", "Bandwidth"); - HDfprintf(H5DEBUG(T), " %-16s %10s %10s %8s %8s %8s %10s\n", - "----------", "-----", "-----", "----", - "------", "-------", "---------"); - } - if (path->src && path->dst) + if(H5DEBUG(T) && path->stats.ncalls > 0) { + hsize_t nbytes; + char bandwidth[32]; + + if(nprint && 0 == (*nprint)++) { + HDfprintf(H5DEBUG(T), "H5T: type conversion statistics:\n"); + HDfprintf(H5DEBUG(T), " %-16s %10s %10s %8s %8s %8s %10s\n", + "Conversion", "Elmts", "Calls", "User", + "System", "Elapsed", "Bandwidth"); + HDfprintf(H5DEBUG(T), " %-16s %10s %10s %8s %8s %8s %10s\n", + "----------", "-----", "-----", "----", + "------", "-------", "---------"); + } /* end if */ + + if(path->src && path->dst) nbytes = MAX(H5T_get_size(path->src), H5T_get_size(path->dst)); else if (path->src) nbytes = H5T_get_size(path->src); @@ -121,18 +120,20 @@ H5T__print_stats(H5T_path_t H5_ATTR_UNUSED * path, int H5_ATTR_UNUSED * nprint/* nbytes = H5T_get_size(path->dst); else nbytes = 0; - nbytes *= path->stats.nelmts; - H5_bandwidth(bandwidth, (double)nbytes, path->stats.timer.etime); - HDfprintf(H5DEBUG(T), " %-16s %10Hd %10d %8.2f %8.2f %8.2f %10s\n", - path->name, - path->stats.nelmts, - path->stats.ncalls, - path->stats.timer.utime, - path->stats.timer.stime, - path->stats.timer.etime, - bandwidth); - } + + nbytes *= path->stats.nelmts; + H5_bandwidth(bandwidth, (double)nbytes, path->stats.times.elapsed); + HDfprintf(H5DEBUG(T), " %-16s %10Hd %10d %8T %8T %8T %10s\n", + path->name, + path->stats.nelmts, + path->stats.ncalls, + path->stats.times.user, + path->stats.times.system, + path->stats.times.elapsed, + bandwidth); + } /* end if */ #endif + FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5T__print_stats() */ @@ -405,18 +406,18 @@ H5T_debug(const H5T_t *dt, FILE *stream) } /* end else */ } else if (H5T_ENUM == dt->shared->type) { - size_t base_size; + size_t base_size; /* Enumeration data type */ HDfprintf(stream, " "); H5T_debug(dt->shared->parent, stream); base_size = dt->shared->parent->shared->size; for (i = 0; i < dt->shared->u.enumer.nmembs; i++) { - size_t k; + size_t k; HDfprintf(stream, "\n\"%s\" = 0x", dt->shared->u.enumer.name[i]); for (k = 0; k < base_size; k++) - HDfprintf(stream, "%02lx", (unsigned long)(dt->shared->u.enumer.value + (i * base_size) + k)); + HDfprintf(stream, "%02p", ((uint8_t *)dt->shared->u.enumer.value + (i * base_size) + k)); } /* end for */ HDfprintf(stream, "\n"); } diff --git a/src/H5Tpkg.h b/src/H5Tpkg.h index 05879ff..b8d2435 100644 --- a/src/H5Tpkg.h +++ b/src/H5Tpkg.h @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Monday, December 8, 1997 * * Purpose: This file contains declarations which are visible only within @@ -146,7 +146,7 @@ struct H5T_stats_t { unsigned ncalls; /*num calls to conversion function */ hsize_t nelmts; /*total data points converted */ - H5_timer_t timer; /*total time for conversion */ + H5_timevals_t times; /*total time for conversion */ }; /* Library internal datatype conversion functions are... */ @@ -12,13 +12,14 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Friday, October 10, 1997 */ #include "H5private.h" #include "H5Eprivate.h" +#include "H5MMprivate.h" /* Memory management */ #include "H5Oprivate.h" #include "H5VMprivate.h" @@ -33,21 +34,21 @@ typedef struct H5VM_memcpy_ud_t { /* Local prototypes */ static void -H5VM_stride_optimize1(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, +H5VM__stride_optimize1(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, const hsize_t *size, hsize_t *stride1); static void -H5VM_stride_optimize2(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, +H5VM__stride_optimize2(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, const hsize_t *size, hsize_t *stride1, hsize_t *stride2); #ifdef LATER static void -H5VM_stride_copy2(hsize_t nelmts, hsize_t elmt_size, +H5VM__stride_copy2(hsize_t nelmts, hsize_t elmt_size, unsigned dst_n, const hsize_t *dst_size, const ssize_t *dst_stride, void *_dst, unsigned src_n, const hsize_t *src_size, const ssize_t *src_stride, const void *_src); #endif /* LATER */ /*------------------------------------------------------------------------- - * Function: H5VM_stride_optimize1 + * Function: H5VM__stride_optimize1 * * Purpose: Given a stride vector which references elements of the * specified size, optimize the dimensionality, the stride @@ -62,15 +63,13 @@ H5VM_stride_copy2(hsize_t nelmts, hsize_t elmt_size, * Programmer: Robb Matzke * Saturday, October 11, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ static void -H5VM_stride_optimize1(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, +H5VM__stride_optimize1(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, const hsize_t *size, hsize_t *stride1) { - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR /* * This has to be true because if we optimize the dimensionality down to @@ -93,7 +92,7 @@ H5VM_stride_optimize1(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, /*------------------------------------------------------------------------- - * Function: H5VM_stride_optimize2 + * Function: H5VM__stride_optimize2 * * Purpose: Given two stride vectors which reference elements of the * specified size, optimize the dimensionality, the stride @@ -108,18 +107,13 @@ H5VM_stride_optimize1(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, * Programmer: Robb Matzke * Saturday, October 11, 1997 * - * Modifications: - * Unrolled loops for common cases - * Quincey Koziol - * ?, ? ?, 2001? - * *------------------------------------------------------------------------- */ static void -H5VM_stride_optimize2(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, +H5VM__stride_optimize2(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, const hsize_t *size, hsize_t *stride1, hsize_t *stride2) { - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR /* * This has to be true because if we optimize the dimensionality down to @@ -246,11 +240,6 @@ H5VM_stride_optimize2(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, * Programmer: Robb Matzke * Saturday, October 11, 1997 * - * Modifications: - * Unrolled loops for common cases - * Quincey Koziol - * ?, ? ?, 2001? - * *------------------------------------------------------------------------- */ hsize_t @@ -349,8 +338,6 @@ H5VM_hyper_stride(unsigned n, const hsize_t *size, * Programmer: Robb Matzke * Friday, October 17, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ htri_t @@ -400,8 +387,6 @@ done: * Programmer: Robb Matzke * Friday, October 10, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ herr_t @@ -438,7 +423,7 @@ H5VM_hyper_fill(unsigned n, const hsize_t *_size, /* Compute an optimal destination stride vector */ dst_start = H5VM_hyper_stride(n, size, total_size, offset, dst_stride); - H5VM_stride_optimize1(&n, &elmt_size, size, dst_stride); + H5VM__stride_optimize1(&n, &elmt_size, size, dst_stride); /* Copy */ ret_value = H5VM_stride_fill(n, elmt_size, size, dst_stride, dst+dst_start, @@ -475,11 +460,6 @@ H5VM_hyper_fill(unsigned n, const hsize_t *_size, * Programmer: Robb Matzke * Friday, October 10, 1997 * - * Modifications: - * Unrolled loops for common cases - * Quincey Koziol - * ?, ? ?, 2001? - * *------------------------------------------------------------------------- */ herr_t @@ -623,7 +603,7 @@ H5VM_hyper_copy(unsigned n, const hsize_t *_size, #endif /* NO_INLINED_CODE */ /* Optimize the strides as a pair */ - H5VM_stride_optimize2(&n, &elmt_size, size, dst_stride, src_stride); + H5VM__stride_optimize2(&n, &elmt_size, size, dst_stride, src_stride); /* Perform the copy in terms of stride */ ret_value = H5VM_stride_copy(n, elmt_size, size, @@ -644,8 +624,6 @@ H5VM_hyper_copy(unsigned n, const hsize_t *_size, * Programmer: Robb Matzke * Saturday, October 11, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ herr_t @@ -704,8 +682,6 @@ H5VM_stride_fill(unsigned n, hsize_t elmt_size, const hsize_t *size, * Programmer: Robb Matzke * Saturday, October 11, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ herr_t @@ -773,8 +749,6 @@ H5VM_stride_copy(unsigned n, hsize_t elmt_size, const hsize_t *size, * Programmer: Robb Matzke * Saturday, October 11, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ herr_t @@ -827,7 +801,7 @@ H5VM_stride_copy_s(unsigned n, hsize_t elmt_size, const hsize_t *size, #ifdef LATER /*------------------------------------------------------------------------- - * Function: H5VM_stride_copy2 + * Function: H5VM__stride_copy2 * * Purpose: Similar to H5VM_stride_copy() except the source and * destination each have their own dimensionality and size and @@ -839,12 +813,10 @@ H5VM_stride_copy_s(unsigned n, hsize_t elmt_size, const hsize_t *size, * Programmer: Robb Matzke * Saturday, October 11, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ static void -H5VM_stride_copy2(hsize_t nelmts, hsize_t elmt_size, +H5VM__stride_copy2(hsize_t nelmts, hsize_t elmt_size, /* destination */ unsigned dst_n, const hsize_t *dst_size, @@ -864,7 +836,7 @@ H5VM_stride_copy2(hsize_t nelmts, hsize_t elmt_size, int j; /* Local index variable */ hbool_t carry; - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR HDassert(elmt_size < SIZET_MAX); HDassert(dst_n>0); @@ -917,8 +889,6 @@ H5VM_stride_copy2(hsize_t nelmts, hsize_t elmt_size, * Programmer: Quincey Koziol * Thursday, June 18, 1998 * - * Modifications: - * *------------------------------------------------------------------------- */ herr_t @@ -974,8 +944,6 @@ H5VM_array_fill(void *_dst, const void *src, size_t size, size_t count) * Programmer: Quincey Koziol * Monday, April 28, 2003 * - * Modifications: - * *------------------------------------------------------------------------- */ herr_t @@ -1055,8 +1023,6 @@ H5VM_array_offset_pre(unsigned n, const hsize_t *acc, const hsize_t *offset) * Programmer: Quincey Koziol * Tuesday, June 22, 1999 * - * Modifications: - * *------------------------------------------------------------------------- */ hsize_t @@ -1139,8 +1105,6 @@ H5VM_array_calc_pre(hsize_t offset, unsigned n, const hsize_t *down, * Programmer: Quincey Koziol * Wednesday, April 16, 2003 * - * Modifications: - * *------------------------------------------------------------------------- */ herr_t diff --git a/src/H5VMprivate.h b/src/H5VMprivate.h index 26f59e2..5c975f6 100644 --- a/src/H5VMprivate.h +++ b/src/H5VMprivate.h @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Friday, October 10, 1997 */ #ifndef H5VMprivate_H @@ -28,16 +28,16 @@ typedef herr_t (*H5VM_opvv_func_t)(hsize_t dst_off, hsize_t src_off, size_t len, void *udata); /* Vector comparison functions like Fortran66 comparison operators */ -#define H5VM_vector_eq_s(N,V1,V2) (H5VM_vector_cmp_s (N, V1, V2)==0) -#define H5VM_vector_lt_s(N,V1,V2) (H5VM_vector_cmp_s (N, V1, V2)<0) -#define H5VM_vector_gt_s(N,V1,V2) (H5VM_vector_cmp_s (N, V1, V2)>0) -#define H5VM_vector_le_s(N,V1,V2) (H5VM_vector_cmp_s (N, V1, V2)<=0) -#define H5VM_vector_ge_s(N,V1,V2) (H5VM_vector_cmp_s (N, V1, V2)>=0) -#define H5VM_vector_eq_u(N,V1,V2) (H5VM_vector_cmp_u (N, V1, V2)==0) -#define H5VM_vector_lt_u(N,V1,V2) (H5VM_vector_cmp_u (N, V1, V2)<0) -#define H5VM_vector_gt_u(N,V1,V2) (H5VM_vector_cmp_u (N, V1, V2)>0) -#define H5VM_vector_le_u(N,V1,V2) (H5VM_vector_cmp_u (N, V1, V2)<=0) -#define H5VM_vector_ge_u(N,V1,V2) (H5VM_vector_cmp_u (N, V1, V2)>=0) +#define H5VM_vector_eq_s(N,V1,V2) (H5VM_vector_cmp_s(N, V1, V2) == 0) +#define H5VM_vector_lt_s(N,V1,V2) (H5VM_vector_cmp_s(N, V1, V2) < 0) +#define H5VM_vector_gt_s(N,V1,V2) (H5VM_vector_cmp_s(N, V1, V2) > 0) +#define H5VM_vector_le_s(N,V1,V2) (H5VM_vector_cmp_s(N, V1, V2) <= 0) +#define H5VM_vector_ge_s(N,V1,V2) (H5VM_vector_cmp_s(N, V1, V2) >= 0) +#define H5VM_vector_eq_u(N,V1,V2) (H5VM_vector_cmp_u(N, V1, V2) == 0) +#define H5VM_vector_lt_u(N,V1,V2) (H5VM_vector_cmp_u(N, V1, V2) < 0) +#define H5VM_vector_gt_u(N,V1,V2) (H5VM_vector_cmp_u(N, V1, V2) > 0) +#define H5VM_vector_le_u(N,V1,V2) (H5VM_vector_cmp_u(N, V1, V2) <= 0) +#define H5VM_vector_ge_u(N,V1,V2) (H5VM_vector_cmp_u(N, V1, V2) >= 0) /* Other functions */ #define H5VM_vector_cpy(N,DST,SRC) { \ @@ -146,6 +146,10 @@ H5_DLL ssize_t H5VM_memcpyvv(void *_dst, * elements in an array and array dimensions are always of type * size_t. * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: Success: Product of elements * * Failure: 1 if N is zero @@ -153,8 +157,6 @@ H5_DLL ssize_t H5VM_memcpyvv(void *_dst, * Programmer: Robb Matzke * Friday, October 10, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ static H5_INLINE hsize_t H5_ATTR_UNUSED @@ -177,6 +179,10 @@ done: * * Purpose: Determines if all elements of a vector are zero. * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: Success: TRUE if all elements are zero, * FALSE otherwise * @@ -185,8 +191,6 @@ done: * Programmer: Robb Matzke * Friday, October 10, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ static H5_INLINE htri_t H5_ATTR_UNUSED @@ -212,6 +216,10 @@ done: * * Purpose: Determines if all elements of a vector are zero. * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: Success: TRUE if all elements are zero, * FALSE otherwise * @@ -220,8 +228,6 @@ done: * Programmer: Robb Matzke * Friday, October 10, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ static H5_INLINE htri_t H5_ATTR_UNUSED @@ -248,6 +254,10 @@ done: * Purpose: Compares two vectors of the same size and determines if V1 is * lexicographically less than, equal, or greater than V2. * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: Success: -1 if V1 is less than V2 * 0 if they are equal * 1 if V1 is greater than V2 @@ -257,12 +267,10 @@ done: * Programmer: Robb Matzke * Friday, October 10, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ static H5_INLINE int H5_ATTR_UNUSED -H5VM_vector_cmp_u (unsigned n, const hsize_t *v1, const hsize_t *v2) +H5VM_vector_cmp_u(unsigned n, const hsize_t *v1, const hsize_t *v2) { int ret_value=0; /* Return value */ @@ -290,6 +298,10 @@ done: * Purpose: Compares two vectors of the same size and determines if V1 is * lexicographically less than, equal, or greater than V2. * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: Success: -1 if V1 is less than V2 * 0 if they are equal * 1 if V1 is greater than V2 @@ -299,12 +311,10 @@ done: * Programmer: Robb Matzke * Wednesday, April 8, 1998 * - * Modifications: - * *------------------------------------------------------------------------- */ static H5_INLINE int H5_ATTR_UNUSED -H5VM_vector_cmp_s (unsigned n, const hssize_t *v1, const hssize_t *v2) +H5VM_vector_cmp_s(unsigned n, const hssize_t *v1, const hssize_t *v2) { int ret_value=0; /* Return value */ @@ -331,13 +341,15 @@ done: * * Purpose: Increments V1 by V2 * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: void * * Programmer: Robb Matzke * Monday, October 13, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ static H5_INLINE void H5_ATTR_UNUSED @@ -380,6 +392,10 @@ static const unsigned char LogTable256[] = * The version on the web-site is for 32-bit quantities and this * version has been extended for 64-bit quantities. * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: log2(n) (always - no failure condition) * * Programmer: Quincey Koziol @@ -428,6 +444,10 @@ static const unsigned MultiplyDeBruijnBitPosition[32] = * This is from the "Bit Twiddling Hacks" at: * http://graphics.stanford.edu/~seander/bithacks.html#IntegerLogDeBruijn * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: log2(n) (always - no failure condition) * * Programmer: Quincey Koziol @@ -450,6 +470,10 @@ H5VM_log2_of2(uint32_t n) * * Purpose: Round up a number to the next power of 2 * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: Return the number which is a power of 2 * * Programmer: Vailin Choi; Nov 2014 @@ -478,6 +502,10 @@ H5VM_power2up(hsize_t n) * Purpose: Determine the # of bytes needed to encode values within a * range from 0 to a given limit * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: Number of bytes needed * * Programmer: Quincey Koziol @@ -507,6 +535,10 @@ static const unsigned char H5VM_bit_clear_g[8] = {0x7F, 0xBF, 0xDF, 0xEF, 0xF7, * to bit offset 7 in the first byte's low-bit position, then to * bit offset 8 in the second byte's high-bit position, etc. * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: TRUE/FALSE * * Programmer: Quincey Koziol @@ -534,6 +566,10 @@ H5VM_bit_get(const unsigned char *buf, size_t offset) * to bit offset 7 in the first byte's low-bit position, then to * bit offset 8 in the second byte's high-bit position, etc. * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: None * * Programmer: Quincey Koziol @@ -15,7 +15,7 @@ * * Created: H5WB.c * Jun 26 2007 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * * Purpose: Implements the "wrapped buffer" code for wrapping * an existing [staticly sized] buffer, in order to @@ -97,7 +97,6 @@ H5FL_BLK_DEFINE_STATIC(extra_buf); * NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Jun 26 2007 * *------------------------------------------------------------------------- @@ -151,7 +150,6 @@ done: * NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Jun 26 2007 * *------------------------------------------------------------------------- @@ -219,7 +217,6 @@ done: * NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Jun 26 2007 * *------------------------------------------------------------------------- @@ -257,7 +254,6 @@ done: * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Jun 26 2007 * *------------------------------------------------------------------------- diff --git a/src/H5WBprivate.h b/src/H5WBprivate.h index 4460808..3ec9261 100644 --- a/src/H5WBprivate.h +++ b/src/H5WBprivate.h @@ -15,7 +15,7 @@ * * Created: H5WBprivate.h * Jun 26 2007 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * * Purpose: Private header for library accessible wrapped buffer routines. * @@ -35,10 +35,10 @@ #ifdef H5Z_DEBUG typedef struct H5Z_stats_t { struct { - hsize_t total; /* total number of bytes processed */ - hsize_t errors; /* bytes of total attributable to errors */ - H5_timer_t timer; /* execution time including errors */ - } stats[2]; /* 0=output, 1=input */ + hsize_t total; /* total number of bytes processed */ + hsize_t errors; /* bytes of total attributable to errors */ + H5_timevals_t times; /* execution time including errors */ + } stats[2]; /* 0 = output, 1 = input */ } H5Z_stats_t; #endif /* H5Z_DEBUG */ @@ -68,7 +68,7 @@ static H5Z_stats_t *H5Z_stat_table_g = NULL; #endif /* H5Z_DEBUG */ /* Local functions */ -static int H5Z_find_idx(H5Z_filter_t id); +static int H5Z__find_idx(H5Z_filter_t id); static int H5Z__check_unregister_dset_cb(void *obj_ptr, hid_t obj_id, void *key); static int H5Z__check_unregister_group_cb(void *obj_ptr, hid_t obj_id, void *key); static int H5Z__flush_file_cb(void *obj_ptr, hid_t obj_id, void *key); @@ -132,9 +132,9 @@ H5Z_term_package(void) if(H5_PKG_INIT_VAR) { #ifdef H5Z_DEBUG - char comment[16], bandwidth[32]; - int dir, nprint = 0; - size_t i; + char comment[16], bandwidth[32]; + int dir, nprint = 0; + size_t i; if(H5DEBUG(Z)) { for(i = 0; i < H5Z_table_used_g; i++) { @@ -167,25 +167,26 @@ H5Z_term_package(void) */ H5_bandwidth(bandwidth, (double)(H5Z_stat_table_g[i].stats[dir].total), - H5Z_stat_table_g[i].stats[dir].timer.etime); + H5Z_stat_table_g[i].stats[dir].times.elapsed); /* Print the statistics */ - HDfprintf(H5DEBUG(Z), - " %s%-15s %10Hd %10Hd %8.2f %8.2f %8.2f " - "%10s\n", dir?"<":">", comment, + HDfprintf(H5DEBUG(Z), " %s%-15s %10Hd %10Hd %8T %8T %8T %10s\n", + (dir ? "<" : ">"), comment, H5Z_stat_table_g[i].stats[dir].total, H5Z_stat_table_g[i].stats[dir].errors, - H5Z_stat_table_g[i].stats[dir].timer.utime, - H5Z_stat_table_g[i].stats[dir].timer.stime, - H5Z_stat_table_g[i].stats[dir].timer.etime, + H5Z_stat_table_g[i].stats[dir].times.user, + H5Z_stat_table_g[i].stats[dir].times.system, + H5Z_stat_table_g[i].stats[dir].times.elapsed, bandwidth); } /* end for */ } /* end for */ } /* end if */ #endif /* H5Z_DEBUG */ + /* Free the table of filters */ - if (H5Z_table_g) { + if(H5Z_table_g) { H5Z_table_g = (H5Z_class2_t *)H5MM_xfree(H5Z_table_g); + #ifdef H5Z_DEBUG H5Z_stat_table_g = (H5Z_stats_t *)H5MM_xfree(H5Z_stat_table_g); #endif /* H5Z_DEBUG */ @@ -378,8 +379,8 @@ done: * Purpose: Same as the public version except this one allows filters * to be unset for predefined method numbers <H5Z_FILTER_RESERVED * - * Return: Non-negative on success - * Negative on failure + * Return: SUCCEED/FAIL + * *------------------------------------------------------------------------- */ herr_t @@ -700,8 +701,8 @@ H5Z_filter_avail(H5Z_filter_t id) HGOTO_DONE(TRUE) key.id = (int)id; - if (NULL != (filter_info = (const H5Z_class2_t *)H5PL_load(H5PL_TYPE_FILTER, key))) { - if (H5Z_register(filter_info) < 0) + if(NULL != (filter_info = (const H5Z_class2_t *)H5PL_load(H5PL_TYPE_FILTER, key))) { + if(H5Z_register(filter_info) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "unable to register loaded filter") HGOTO_DONE(TRUE) } /* end if */ @@ -712,7 +713,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5Z_prelude_callback + * Function: H5Z__prelude_callback * * Purpose: Makes a dataset creation "prelude" callback for the "can_apply" * or "set_local" routines. @@ -725,14 +726,14 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5Z_prelude_callback(const H5O_pline_t *pline, hid_t dcpl_id, hid_t type_id, +H5Z__prelude_callback(const H5O_pline_t *pline, hid_t dcpl_id, hid_t type_id, hid_t space_id, H5Z_prelude_type_t prelude_type) { H5Z_class2_t *fclass; /* Individual filter information */ size_t u; /* Local index variable */ htri_t ret_value = TRUE; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC HDassert(pline->nused > 0); @@ -790,11 +791,11 @@ H5Z_prelude_callback(const H5O_pline_t *pline, hid_t dcpl_id, hid_t type_id, done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5Z_prelude_callback() */ +} /* end H5Z__prelude_callback() */ /*------------------------------------------------------------------------- - * Function: H5Z_prepare_prelude_callback_dcpl + * Function: H5Z__prepare_prelude_callback_dcpl * * Purpose: Prepares to make a dataset creation "prelude" callback * for the "can_apply" or "set_local" routines. @@ -807,13 +808,13 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5Z_prepare_prelude_callback_dcpl(hid_t dcpl_id, hid_t type_id, H5Z_prelude_type_t prelude_type) +H5Z__prepare_prelude_callback_dcpl(hid_t dcpl_id, hid_t type_id, H5Z_prelude_type_t prelude_type) { hid_t space_id = -1; /* ID for dataspace describing chunk */ H5O_layout_t *dcpl_layout = NULL; /* Dataset's layout information */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC HDassert(H5I_GENPROP_LST == H5I_get_type(dcpl_id)); HDassert(H5I_DATATYPE == H5I_get_type(type_id)); @@ -861,7 +862,7 @@ H5Z_prepare_prelude_callback_dcpl(hid_t dcpl_id, hid_t type_id, H5Z_prelude_type } /* Make the callbacks */ - if (H5Z_prelude_callback(&dcpl_pline, dcpl_id, type_id, space_id, prelude_type) < 0) + if (H5Z__prelude_callback(&dcpl_pline, dcpl_id, type_id, space_id, prelude_type) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "unable to apply filter") } } @@ -875,7 +876,7 @@ done: dcpl_layout = (H5O_layout_t *)H5MM_xfree(dcpl_layout); FUNC_LEAVE_NOAPI(ret_value) -} /* end H5Z_prepare_prelude_callback_dcpl() */ +} /* end H5Z__prepare_prelude_callback_dcpl() */ /*------------------------------------------------------------------------- @@ -901,7 +902,7 @@ H5Z_can_apply(hid_t dcpl_id, hid_t type_id) FUNC_ENTER_NOAPI(FAIL) /* Make "can apply" callbacks for filters in pipeline */ - if (H5Z_prepare_prelude_callback_dcpl(dcpl_id, type_id, H5Z_PRELUDE_CAN_APPLY) < 0) + if (H5Z__prepare_prelude_callback_dcpl(dcpl_id, type_id, H5Z_PRELUDE_CAN_APPLY) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "unable to apply filter") done: @@ -932,7 +933,7 @@ H5Z_set_local(hid_t dcpl_id, hid_t type_id) FUNC_ENTER_NOAPI(FAIL) /* Make "set local" callbacks for filters in pipeline */ - if (H5Z_prepare_prelude_callback_dcpl(dcpl_id, type_id, H5Z_PRELUDE_SET_LOCAL) < 0) + if (H5Z__prepare_prelude_callback_dcpl(dcpl_id, type_id, H5Z_PRELUDE_SET_LOCAL) < 0) HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "local filter parameters not set") done: @@ -961,7 +962,7 @@ H5Z_can_apply_direct(const H5O_pline_t *pline) HDassert(pline->nused > 0); /* Make "can apply" callbacks for filters in pipeline */ - if (H5Z_prelude_callback(pline, (hid_t)-1, (hid_t)-1, (hid_t)-1, H5Z_PRELUDE_CAN_APPLY) < 0) + if (H5Z__prelude_callback(pline, (hid_t)-1, (hid_t)-1, (hid_t)-1, H5Z_PRELUDE_CAN_APPLY) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "unable to apply filter") done: @@ -994,7 +995,7 @@ H5Z_set_local_direct(const H5O_pline_t *pline) HDassert(pline->nused > 0); /* Make "set local" callbacks for filters in pipeline */ - if (H5Z_prelude_callback(pline, (hid_t)-1, (hid_t)-1, (hid_t)-1, H5Z_PRELUDE_SET_LOCAL) < 0) + if (H5Z__prelude_callback(pline, (hid_t)-1, (hid_t)-1, (hid_t)-1, H5Z_PRELUDE_SET_LOCAL) < 0) HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "local filter parameters not set") done: @@ -1167,7 +1168,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5Z_find_idx + * Function: H5Z__find_idx * * Purpose: Given a filter ID return the offset in the global array * that holds all the registered filters. @@ -1177,20 +1178,20 @@ done: *------------------------------------------------------------------------- */ static int -H5Z_find_idx(H5Z_filter_t id) +H5Z__find_idx(H5Z_filter_t id) { size_t i; /* Local index variable */ int ret_value = FAIL; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR - for (i = 0; i < H5Z_table_used_g; i++) - if (H5Z_table_g[i].id == id) + for(i = 0; i < H5Z_table_used_g; i++) + if(H5Z_table_g[i].id == id) HGOTO_DONE((int)i) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5Z_find_idx() */ +} /* end H5Z__find_idx() */ /*------------------------------------------------------------------------- @@ -1212,7 +1213,7 @@ H5Z_find(H5Z_filter_t id) FUNC_ENTER_NOAPI(NULL) /* Get the index in the global table */ - if ((idx = H5Z_find_idx(id)) < 0) + if ((idx = H5Z__find_idx(id)) < 0) HGOTO_ERROR(H5E_PLINE, H5E_NOTFOUND, NULL, "required filter %d is not registered", id) /* Set return value */ @@ -1248,20 +1249,22 @@ done: */ herr_t H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, - unsigned *filter_mask/*in,out*/, H5Z_EDC_t edc_read, - H5Z_cb_t cb_struct, size_t *nbytes/*in,out*/, - size_t *buf_size/*in,out*/, void **buf/*in,out*/) + unsigned *filter_mask/*in,out*/, H5Z_EDC_t edc_read, H5Z_cb_t cb_struct, + size_t *nbytes/*in,out*/, size_t *buf_size/*in,out*/, void **buf/*in,out*/) { - size_t i, idx, new_nbytes; - int fclass_idx; /* Index of filter class in global table */ - H5Z_class2_t *fclass=NULL; /* Filter class pointer */ + size_t idx; + size_t new_nbytes; + int fclass_idx; /* Index of filter class in global table */ + H5Z_class2_t *fclass = NULL; /* Filter class pointer */ #ifdef H5Z_DEBUG - H5Z_stats_t *fstats=NULL; /* Filter stats pointer */ - H5_timer_t timer; + H5Z_stats_t *fstats = NULL; /* Filter stats pointer */ + H5_timer_t timer; /* Timer for filter operations */ + H5_timevals_t times; /* Elapsed time for each operation */ #endif - unsigned failed = 0; - unsigned tmp_flags; - herr_t ret_value = SUCCEED; /* Return value */ + unsigned failed = 0; + unsigned tmp_flags; + size_t i; + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -1272,11 +1275,13 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, HDassert(buf && *buf); HDassert(!pline || pline->nused < H5Z_MAX_NFILTERS); - if (pline && (flags & H5Z_FLAG_REVERSE)) { /* Read */ - for (i = pline->nused; i > 0; --i) { - idx = i-1; - - if (*filter_mask & ((unsigned)1 << idx)) { +#ifdef H5Z_DEBUG + H5_timer_init(&timer); +#endif + if(pline && (flags & H5Z_FLAG_REVERSE)) { /* Read */ + for(i = pline->nused; i > 0; --i) { + idx = i - 1; + if(*filter_mask & ((unsigned)1 << idx)) { failed |= (unsigned)1 << idx; continue; /* filter excluded */ } @@ -1285,10 +1290,10 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, * indicate no plugin through HDF5_PRELOAD_PLUG (using the symbol "::"), * try to load it dynamically and register it. Otherwise, return failure */ - if ((fclass_idx = H5Z_find_idx(pline->filter[idx].id)) < 0) { - hbool_t issue_error = FALSE; + if ((fclass_idx = H5Z__find_idx(pline->filter[idx].id)) < 0) { H5PL_key_t key; const H5Z_class2_t *filter_info; + hbool_t issue_error = FALSE; /* Try loading the filter */ key.id = (int)(pline->filter[idx].id); @@ -1298,7 +1303,7 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "unable to register filter") /* Search in the table of registered filters again to find the dynamic filter just loaded and registered */ - if ((fclass_idx = H5Z_find_idx(pline->filter[idx].id)) < 0) + if((fclass_idx = H5Z__find_idx(pline->filter[idx].id)) < 0) issue_error = TRUE; } else @@ -1316,76 +1321,96 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, } /* end if */ fclass = &H5Z_table_g[fclass_idx]; + #ifdef H5Z_DEBUG fstats = &H5Z_stat_table_g[fclass_idx]; - H5_timer_begin (&timer); + H5_timer_start(&timer); #endif + tmp_flags = flags | (pline->filter[idx].flags); - tmp_flags |= (edc_read== H5Z_DISABLE_EDC) ? H5Z_FLAG_SKIP_EDC : 0; - new_nbytes = (fclass->filter)(tmp_flags, pline->filter[idx].cd_nelmts, - pline->filter[idx].cd_values, *nbytes, buf_size, buf); + tmp_flags |= (edc_read == H5Z_DISABLE_EDC) ? H5Z_FLAG_SKIP_EDC : 0; + new_nbytes = (fclass->filter)(tmp_flags, + pline->filter[idx].cd_nelmts, pline->filter[idx].cd_values, + *nbytes, buf_size, buf); #ifdef H5Z_DEBUG - H5_timer_end (&(fstats->stats[1].timer), &timer); + H5_timer_stop(&timer); + H5_timer_get_times(timer, ×); + fstats->stats[1].times.elapsed += times.elapsed; + fstats->stats[1].times.system += times.system; + fstats->stats[1].times.user += times.user; + fstats->stats[1].total += MAX(*nbytes, new_nbytes); - if (0 == new_nbytes) + if(0 == new_nbytes) fstats->stats[1].errors += *nbytes; #endif - if (0 == new_nbytes) { - if ((cb_struct.func && (H5Z_CB_FAIL == cb_struct.func(pline->filter[idx].id, *buf, *buf_size, cb_struct.op_data))) || !cb_struct.func) + if(0 == new_nbytes) { + if((cb_struct.func + && (H5Z_CB_FAIL == cb_struct.func(pline->filter[idx].id, *buf, *buf_size, cb_struct.op_data))) + || !cb_struct.func) HGOTO_ERROR(H5E_PLINE, H5E_READERROR, FAIL, "filter returned failure during read") *nbytes = *buf_size; failed |= (unsigned)1 << idx; - H5E_clear_stack (NULL); + H5E_clear_stack(NULL); } else *nbytes = new_nbytes; } } - else if (pline) { /* Write */ - for (idx = 0; idx < pline->nused; idx++) { - if (*filter_mask & ((unsigned)1 << idx)) { + else if(pline) { /* Write */ + for(idx = 0; idx < pline->nused; idx++) { + if(*filter_mask & ((unsigned)1 << idx)) { failed |= (unsigned)1 << idx; - continue; /*filter excluded*/ + continue; /* filter excluded */ } - if ((fclass_idx = H5Z_find_idx(pline->filter[idx].id)) < 0) { + if((fclass_idx = H5Z__find_idx(pline->filter[idx].id)) < 0) { /* Check if filter is optional -- If it isn't, then error */ - if ((pline->filter[idx].flags & H5Z_FLAG_OPTIONAL) == 0) + if((pline->filter[idx].flags & H5Z_FLAG_OPTIONAL) == 0) HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "required filter is not registered") - failed |= (unsigned)1 << idx; - H5E_clear_stack (NULL); - continue; /*filter excluded*/ - } + H5E_clear_stack(NULL); + continue; /* filter excluded */ + } /* end if */ + fclass = &H5Z_table_g[fclass_idx]; + #ifdef H5Z_DEBUG fstats = &H5Z_stat_table_g[fclass_idx]; - H5_timer_begin (&timer); + H5_timer_start(&timer); #endif - new_nbytes = (fclass->filter)(flags | (pline->filter[idx].flags), pline->filter[idx].cd_nelmts, - pline->filter[idx].cd_values, *nbytes, buf_size, buf); + + new_nbytes = (fclass->filter)(flags | (pline->filter[idx].flags), + pline->filter[idx].cd_nelmts, pline->filter[idx].cd_values, + *nbytes, buf_size, buf); + #ifdef H5Z_DEBUG - H5_timer_end (&(fstats->stats[0].timer), &timer); + H5_timer_stop(&timer); + H5_timer_get_times(timer, ×); + fstats->stats[0].times.elapsed += times.elapsed; + fstats->stats[0].times.system += times.system; + fstats->stats[0].times.user += times.user; + fstats->stats[0].total += MAX(*nbytes, new_nbytes); - if (0 == new_nbytes) + if(0 == new_nbytes) fstats->stats[0].errors += *nbytes; #endif - if (0 == new_nbytes) { - if (0 == (pline->filter[idx].flags & H5Z_FLAG_OPTIONAL)) { - if ((cb_struct.func && (H5Z_CB_FAIL == cb_struct.func (pline->filter[idx].id, *buf, *nbytes, cb_struct.op_data))) || !cb_struct.func) + + if(0 == new_nbytes) { + if(0 == (pline->filter[idx].flags & H5Z_FLAG_OPTIONAL)) { + if((cb_struct.func && (H5Z_CB_FAIL == cb_struct.func(pline->filter[idx].id, *buf, *nbytes, cb_struct.op_data))) + || !cb_struct.func) HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "filter returned failure") *nbytes = *buf_size; } - failed |= (unsigned)1 << idx; - H5E_clear_stack (NULL); + H5E_clear_stack(NULL); } else *nbytes = new_nbytes; - } + } /* end for */ } *filter_mask = failed; diff --git a/src/H5Zdeflate.c b/src/H5Zdeflate.c index 6d7b87e..d46108f 100644 --- a/src/H5Zdeflate.c +++ b/src/H5Zdeflate.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Friday, August 27, 1999 */ @@ -34,26 +34,26 @@ #endif /* Local function prototypes */ -static size_t H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, +static size_t H5Z__filter_deflate (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); /* This message derives from H5Z */ const H5Z_class2_t H5Z_DEFLATE[1] = {{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ H5Z_FILTER_DEFLATE, /* Filter id number */ - 1, /* encoder_present flag (set to true) */ - 1, /* decoder_present flag (set to true) */ - "deflate", /* Filter name for debugging */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ + "deflate", /* Filter name for debugging */ NULL, /* The "can apply" callback */ NULL, /* The "set local" callback */ - H5Z_filter_deflate, /* The actual filter function */ + H5Z__filter_deflate, /* The actual filter function */ }}; #define H5Z_DEFLATE_SIZE_ADJUST(s) (HDceil(((double)(s)) * (double)1.001f) + 12) /*------------------------------------------------------------------------- - * Function: H5Z_filter_deflate + * Function: H5Z__filter_deflate * * Purpose: Implement an I/O filter around the 'deflate' algorithm in * libz @@ -64,12 +64,10 @@ const H5Z_class2_t H5Z_DEFLATE[1] = {{ * Programmer: Robb Matzke * Thursday, April 16, 1998 * - * Modifications: - * *------------------------------------------------------------------------- */ static size_t -H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, +H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) { @@ -77,7 +75,7 @@ H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, int status; /* Status from zlib operation */ size_t ret_value = 0; /* Return value */ - FUNC_ENTER_NOAPI(0) + FUNC_ENTER_STATIC /* Sanity check */ HDassert(*buf_size > 0); diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h index a2a44fa..b37dcf3 100644 --- a/src/H5Zpublic.h +++ b/src/H5Zpublic.h @@ -11,7 +11,7 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Robb Matzke <matzke@llnl.gov> +/* Programmer: Robb Matzke * Thursday, April 16, 1998 */ diff --git a/src/H5err.txt b/src/H5err.txt index 30c646f..f3cca72 100644 --- a/src/H5err.txt +++ b/src/H5err.txt @@ -133,6 +133,8 @@ MINOR, FILEACC, H5E_NOTHDF5, Not an HDF5 file MINOR, FILEACC, H5E_BADFILE, Bad file ID accessed MINOR, FILEACC, H5E_TRUNCATED, File has been truncated MINOR, FILEACC, H5E_MOUNT, File mount error +MINOR, FILEACC, H5E_CANTLOCKFILE, Unable to lock file +MINOR, FILEACC, H5E_CANTUNLOCKFILE, Unable to unlock file # Generic low-level file I/O errors MINOR, FILE, H5E_SEEKERROR, Seek failed diff --git a/src/H5private.h b/src/H5private.h index c0d39fc..7793e0b 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -11,7 +11,7 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Robb Matzke <matzke@llnl.gov> +/* Programmer: Robb Matzke * Friday, October 30, 1998 * * Purpose: This file is included by all HDF5 library source files to @@ -362,6 +362,22 @@ #endif /* __cplusplus */ /* + * Networking headers used by the mirror VFD and related tests and utilities. + */ +#ifdef H5_HAVE_ARPA_INET_H +# include <arpa/inet.h> +#endif +#ifdef H5_HAVE_NETDB_H +# include <netdb.h> +#endif +#ifdef H5_HAVE_NETINET_IN_H +# include <netinet/in.h> +#endif +#ifdef H5_HAVE_SYS_SOCKET_H +# include <sys/socket.h> +#endif + +/* * Status return values for the `herr_t' type. * Since some unix/c routines use 0 and -1 (or more precisely, non-negative * vs. negative) as their return code, and some assumption had been made in @@ -556,28 +572,6 @@ #define H5_REQUEST_NULL NULL /* - * A macro to portably decrement enumerated types. - */ -#ifndef H5_DEC_ENUM -# define H5_DEC_ENUM(TYPE,VAR) (VAR)=((TYPE)((VAR)-1)) -#endif - -/* Double constant wrapper - * - * Quiets gcc warnings from -Wunsuffixed-float-constants. - * - * This is a really annoying warning since the standard specifies that - * constants of type double do NOT get a suffix so there's no way - * to specify a constant of type double. To quiet gcc, we specify floating - * point constants as type long double and cast to double. - * - * Note that this macro only needs to be used where using a double - * is important. For most code, suffixing constants with F will quiet the - * compiler and not produce erroneous code. - */ -#define H5_DOUBLE(S) ((double) S ## L) - -/* * Methods to compare the equality of floating-point values: * * 1. H5_XXX_ABS_EQUAL - check if the difference is smaller than the @@ -617,21 +611,37 @@ #define LOCK_UN 0x08 #endif /* H5_HAVE_FLOCK */ -/* - * Data types and functions for timing certain parts of the library. +/* Typedefs and functions for timing certain parts of the library. */ + +/* A set of elapsed/user/system times emitted as a time point by the + * platform-independent timers. */ typedef struct { - double utime; /*user time */ - double stime; /*system time */ - double etime; /*elapsed wall-clock time */ + double user; /* User time in seconds */ + double system; /* System time in seconds */ + double elapsed; /* Elapsed (wall clock) time in seconds */ +} H5_timevals_t; + +/* Timer structure for platform-independent timers */ +typedef struct { + H5_timevals_t initial; /* Current interval start time */ + H5_timevals_t final_interval; /* Last interval elapsed time */ + H5_timevals_t total; /* Total elapsed time for all intervals */ + hbool_t is_running; /* Whether timer is running */ } H5_timer_t; -H5_DLL void H5_timer_reset (H5_timer_t *timer); -H5_DLL void H5_timer_begin (H5_timer_t *timer); -H5_DLL void H5_timer_end (H5_timer_t *sum/*in,out*/, - H5_timer_t *timer/*in,out*/); +/* Returns library bandwidth as a pretty string */ H5_DLL void H5_bandwidth(char *buf/*out*/, double nbytes, double nseconds); + +/* Timer functionality */ H5_DLL time_t H5_now(void); +H5_DLL uint64_t H5_now_usec(void); +H5_DLL herr_t H5_timer_init(H5_timer_t *timer /*in,out*/); +H5_DLL herr_t H5_timer_start(H5_timer_t *timer /*in,out*/); +H5_DLL herr_t H5_timer_stop(H5_timer_t *timer /*in,out*/); +H5_DLL herr_t H5_timer_get_times(H5_timer_t timer, H5_timevals_t *times /*in,out*/); +H5_DLL herr_t H5_timer_get_total_times(H5_timer_t timer, H5_timevals_t *times /*in,out*/); +H5_DLL char *H5_timer_get_time_string(double seconds); /* Depth of object copy */ typedef enum { @@ -665,6 +675,9 @@ typedef struct { #ifndef HDabs #define HDabs(X) abs(X) #endif /* HDabs */ +#ifndef HDaccept + #define HDaccept(A,B,C) accept((A),(B),(C)) /* mirror VFD */ +#endif /* HDaccept */ #ifndef HDaccess #define HDaccess(F,M) access(F, M) #endif /* HDaccess */ @@ -711,6 +724,9 @@ typedef struct { #ifndef HDatoll #define HDatoll(S) atoll(S) #endif /* HDatol */ +#ifndef HDbind + #define HDbind(A,B,C) bind((A),(B),(C)) /* mirror VFD */ +#endif /* HDbind */ #ifndef HDbsearch #define HDbsearch(K,B,N,Z,F) bsearch(K,B,N,Z,F) #endif /* HDbsearch */ @@ -747,12 +763,18 @@ typedef struct { #ifndef HDclock #define HDclock() clock() #endif /* HDclock */ +#ifndef HDclock_gettime + #define HDclock_gettime(CID, TS) clock_gettime(CID, TS) +#endif /* HDclock_gettime */ #ifndef HDclose #define HDclose(F) close(F) #endif /* HDclose */ #ifndef HDclosedir #define HDclosedir(D) closedir(D) #endif /* HDclosedir */ +#ifndef HDconnect + #define HDconnect(A,B,C) connect((A),(B),(C)) /* mirror VFD */ +#endif /* HDconnect */ #ifndef HDcos #define HDcos(X) cos(X) #endif /* HDcos */ @@ -863,8 +885,8 @@ H5_DLL H5_ATTR_CONST int Nflock(int fd, int operation); #ifndef HDflock /* NOTE: flock(2) is not present on all POSIX systems. * If it is not present, we try a flock() equivalent based on - * fcntl(2), then fall back to a function that always fails if - * it is not present at all (Windows uses a separate Wflock() + * fcntl(2), then fall back to a function that always succeeds + * if it is not present at all (Windows uses a separate Wflock() * function). */ #if defined(H5_HAVE_FLOCK) @@ -953,7 +975,7 @@ typedef off_t h5_stat_size_t; #define H5_SIZEOF_H5_STAT_SIZE_T H5_SIZEOF_OFF_T #ifndef HDftell - #define HDftell(F) ftello(F) + #define HDftell(F) ftell(F) #endif /* HDftell */ #ifndef HDftruncate #define HDftruncate(F,L) ftruncate(F,L) @@ -997,6 +1019,9 @@ typedef off_t h5_stat_size_t; #ifndef HDgetgroups #define HDgetgroups(Z,G) getgroups(Z,G) #endif /* HDgetgroups */ +#ifndef HDgethostbyaddr + #define HDgethostbyaddr(A,B,C) gethostbyaddr((A),(B),(C)) /* mirror VFD */ +#endif /* HDgethostbyaddr */ #ifndef HDgethostname #define HDgethostname(N,L) gethostname(N,L) #endif /* HDgethostname */ @@ -1036,6 +1061,18 @@ typedef off_t h5_stat_size_t; #ifndef HDgmtime #define HDgmtime(T) gmtime(T) #endif /* HDgmtime */ +#ifndef HDhtonl + #define HDhtonl(X) htonl((X)) /* mirror VFD */ +#endif /* HDhtonl */ +#ifndef HDhtons + #define HDhtons(X) htons((X)) /* mirror VFD */ +#endif /* HDhtons */ +#ifndef HDinet_addr + #define HDinet_addr(C) inet_addr((C)) /* mirror VFD */ +#endif /* HDinet_addr */ +#ifndef HDinet_ntoa + #define HDinet_ntoa(C) inet_ntoa((C)) /* mirror VFD */ +#endif /* HDinet_ntoa */ #ifndef HDisalnum #define HDisalnum(C) isalnum((int)(C)) /*cast for solaris warning*/ #endif /* HDisalnum */ @@ -1090,6 +1127,9 @@ typedef off_t h5_stat_size_t; #ifndef HDlink #define HDlink(OLD,NEW) link(OLD,NEW) #endif /* HDlink */ +#ifndef HDlisten + #define HDlisten(A,B) listen((A),(B)) /* mirror VFD */ +#endif /* HDlisten */ #ifndef HDllround #define HDllround(V) llround(V) #endif /* HDround */ @@ -1171,6 +1211,12 @@ typedef off_t h5_stat_size_t; #ifndef HDnanosleep #define HDnanosleep(N, O) nanosleep(N, O) #endif /* HDnanosleep */ +#ifndef HDntohl + #define HDntohl(A) ntohl((A)) /* mirror VFD */ +#endif /* HDntohl */ +#ifndef HDntohs + #define HDntohs(A) ntohs((A)) /* mirror VFD */ +#endif /* HDntohs */ #ifndef HDopen #define HDopen(F,...) open(F,__VA_ARGS__) #endif /* HDopen */ @@ -1225,13 +1271,23 @@ typedef off_t h5_stat_size_t; #define HDrandom() HDrand() #endif /* HDrandom */ H5_DLL int HDrand(void); -#elif H5_HAVE_RANDOM + #ifndef HDsrandom + #define HDsrandom(S) HDsrand(S) + #endif /* HDsrandom */ + H5_DLL void HDsrand(unsigned int seed); +#elif defined(H5_HAVE_RANDOM) #ifndef HDrand #define HDrand() random() #endif /* HDrand */ #ifndef HDrandom #define HDrandom() random() #endif /* HDrandom */ + #ifndef HDsrand + #define HDsrand(S) srandom(S) + #endif /* HDsrand */ + #ifndef HDsrandom + #define HDsrandom(S) srandom(S) + #endif /* HDsrandom */ #else /* H5_HAVE_RANDOM */ #ifndef HDrand #define HDrand() rand() @@ -1239,6 +1295,12 @@ typedef off_t h5_stat_size_t; #ifndef HDrandom #define HDrandom() rand() #endif /* HDrandom */ + #ifndef HDsrand + #define HDsrand(S) srand(S) + #endif /* HDsrand */ + #ifndef HDsrandom + #define HDsrandom(S) srand(S) + #endif /* HDsrandom */ #endif /* H5_HAVE_RANDOM */ #ifndef HDread @@ -1302,12 +1364,21 @@ typedef off_t h5_stat_size_t; #ifndef HDsetsid #define HDsetsid() setsid() #endif /* HDsetsid */ +#ifndef HDsetsockopt + #define HDsetsockopt(A,B,C,D,E) setsockopt((A),(B),(C),(D),(E)) /* mirror VFD */ +#endif /* HDsetsockopt */ #ifndef HDsetuid #define HDsetuid(U) setuid(U) #endif /* HDsetuid */ #ifndef HDsetvbuf #define HDsetvbuf(F,S,M,Z) setvbuf(F,S,M,Z) #endif /* HDsetvbuf */ +#ifndef HDshutdown + #define HDshutdown(A, B) shutdown((A),(B)) /* mirror VFD */ +#endif /* HDshutdown */ +#ifndef HDsigaction + #define HDsigaction(S,A,O) sigaction((S),(A),(O)) +#endif /* HDsigaction */ #ifndef HDsigaddset #define HDsigaddset(S,N) sigaddset(S,N) #endif /* HDsigaddset */ @@ -1353,32 +1424,15 @@ typedef off_t h5_stat_size_t; #ifndef HDsnprintf #define HDsnprintf snprintf /*varargs*/ #endif /* HDsnprintf */ +#ifndef HDsocket + #define HDsocket(A,B,C) socket((A),(B),(C)) /* mirror VFD */ +#endif /* HDsocket */ #ifndef HDsprintf #define HDsprintf sprintf /*varargs*/ #endif /* HDsprintf */ #ifndef HDsqrt #define HDsqrt(X) sqrt(X) #endif /* HDsqrt */ -#ifdef H5_HAVE_RAND_R - H5_DLL void HDsrand(unsigned int seed); - #ifndef HDsrandom - #define HDsrandom(S) HDsrand(S) - #endif /* HDsrandom */ -#elif H5_HAVE_RANDOM - #ifndef HDsrand - #define HDsrand(S) srandom(S) - #endif /* HDsrand */ - #ifndef HDsrandom - #define HDsrandom(S) srandom(S) - #endif /* HDsrandom */ -#else /* H5_HAVE_RAND_R */ - #ifndef HDsrand - #define HDsrand(S) srand(S) - #endif /* HDsrand */ - #ifndef HDsrandom - #define HDsrandom(S) srand(S) - #endif /* HDsrandom */ -#endif /* H5_HAVE_RAND_R */ #ifndef HDsscanf #define HDsscanf(S,FMT,...) sscanf(S,FMT,__VA_ARGS__) #endif /* HDsscanf */ diff --git a/src/H5public.h b/src/H5public.h index 871c360..e451905 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -26,32 +26,32 @@ * it via H5public.h. The #ifndef _H5public_H guard above would * prevent repeated include. */ -#include "H5pubconf.h" /*from configure */ +#include "H5pubconf.h" /* From configure */ /* API Version macro wrapper definitions */ #include "H5version.h" #ifdef H5_HAVE_FEATURES_H -#include <features.h> /*for setting POSIX, BSD, etc. compatibility */ +#include <features.h> /* For setting POSIX, BSD, etc. compatibility */ #endif #ifdef H5_HAVE_SYS_TYPES_H #include <sys/types.h> #endif #ifdef H5_STDC_HEADERS -# include <limits.h> /*for H5T_NATIVE_CHAR defn in H5Tpublic.h */ -# include <stdarg.h> /*for variadic functions in H5VLpublic.h */ +# include <limits.h> /* For H5T_NATIVE_CHAR defn in H5Tpublic.h */ +# include <stdarg.h> /* For variadic functions in H5VLpublic.h */ #endif #ifndef __cplusplus # ifdef H5_HAVE_STDINT_H -# include <stdint.h> /*for C9x types */ +# include <stdint.h> /* For C9x types */ # endif #else # ifdef H5_HAVE_STDINT_H_CXX -# include <stdint.h> /*for C9x types when include from C++ */ +# include <stdint.h> /* For C9x types (when included from C++) */ # endif #endif #ifdef H5_HAVE_INTTYPES_H -# include <inttypes.h> /* For uint64_t on some platforms */ +# include <inttypes.h> /* C99/POSIX.1 header for uint64_t, PRIu64 */ #endif #ifdef H5_HAVE_STDDEF_H # include <stddef.h> @@ -61,7 +61,7 @@ # define MPICH_SKIP_MPICXX 1 # define OMPI_SKIP_MPICXX 1 # include <mpi.h> -#ifndef MPI_FILE_NULL /*MPIO may be defined in mpi.h already */ +#ifndef MPI_FILE_NULL /* MPIO may be defined in mpi.h already */ # include <mpio.h> #endif #endif @@ -94,15 +94,15 @@ extern "C" { #endif /* Version numbers */ -#define H5_VERS_MAJOR 1 /* For major interface/format changes */ -#define H5_VERS_MINOR 10 /* For minor interface/format changes */ -#define H5_VERS_RELEASE 7 /* For tweaks, bug-fixes, or development */ -#define H5_VERS_SUBRELEASE "1" /* For pre-releases like snap0 */ - /* Empty string for real releases. */ +#define H5_VERS_MAJOR 1 /* For major interface/format changes */ +#define H5_VERS_MINOR 10 /* For minor interface/format changes */ +#define H5_VERS_RELEASE 7 /* For tweaks, bug-fixes, or development */ +#define H5_VERS_SUBRELEASE "1" /* For pre-releases like snap0 */ + /* Empty string for real releases. */ #define H5_VERS_INFO "HDF5 library version: 1.10.7-1" /* Full version string */ -#define H5check() H5check_version(H5_VERS_MAJOR,H5_VERS_MINOR, \ - H5_VERS_RELEASE) +#define H5check() H5check_version(H5_VERS_MAJOR,H5_VERS_MINOR, \ + H5_VERS_RELEASE) /* macros for comparing the version */ #define H5_VERSION_GE(Maj,Min,Rel) \ @@ -122,8 +122,8 @@ extern "C" { * The negative failure value is most commonly -1, but don't bet on it. The * proper way to detect failure is something like: * - * if((dset = H5Dopen2(file, name)) < 0) - * fprintf(stderr, "unable to open the requested dataset\n"); + * if((dset = H5Dopen2(file, name)) < 0) + * fprintf(stderr, "unable to open the requested dataset\n"); */ typedef int herr_t; @@ -135,13 +135,13 @@ typedef int herr_t; * (false), positive (true), or negative (failure). The proper way to test * for truth from a htri_t function is: * - * if ((retval = H5Tcommitted(type))>0) { - * printf("data type is committed\n"); - * } else if (!retval) { - * printf("data type is not committed\n"); - * } else { - * printf("error determining whether data type is committed\n"); - * } + * if ((retval = H5Tcommitted(type)) > 0) { + * printf("data type is committed\n"); + * } else if (!retval) { + * printf("data type is not committed\n"); + * } else { + * printf("error determining whether data type is committed\n"); + * } */ #ifdef H5_HAVE_STDBOOL_H #include <stdbool.h> @@ -183,8 +183,8 @@ typedef long long ssize_t; */ #if H5_SIZEOF_LONG_LONG >= 8 H5_GCC_DIAG_OFF(long-long) -typedef unsigned long long hsize_t; -typedef signed long long hssize_t; +typedef unsigned long long hsize_t; +typedef signed long long hssize_t; H5_GCC_DIAG_ON(long-long) # define H5_SIZEOF_HSIZE_T H5_SIZEOF_LONG_LONG # define H5_SIZEOF_HSSIZE_T H5_SIZEOF_LONG_LONG @@ -229,7 +229,7 @@ H5_GCC_DIAG_ON(long-long) #else # error "nothing appropriate for H5_PRINTF_HADDR_FMT" #endif -#define HADDR_MAX (HADDR_UNDEF-1) +#define HADDR_MAX (HADDR_UNDEF-1) /* uint32_t type is used for creation order field for messages. It may be * defined in Posix.1g, otherwise it is defined here. @@ -297,7 +297,7 @@ typedef enum { H5_ITER_INC, /* Increasing order */ H5_ITER_DEC, /* Decreasing order */ H5_ITER_NATIVE, /* No particular order, whatever is fastest */ - H5_ITER_N /* Number of iteration orders */ + H5_ITER_N /* Number of iteration orders */ } H5_iter_order_t; /* Iteration callback values */ @@ -314,10 +314,10 @@ typedef enum { * links in groups/attributes on objects. */ typedef enum H5_index_t { - H5_INDEX_UNKNOWN = -1, /* Unknown index type */ - H5_INDEX_NAME, /* Index on names */ - H5_INDEX_CRT_ORDER, /* Index on creation order */ - H5_INDEX_N /* Number of indices defined */ + H5_INDEX_UNKNOWN = -1, /* Unknown index type */ + H5_INDEX_NAME, /* Index on names */ + H5_INDEX_CRT_ORDER, /* Index on creation order */ + H5_INDEX_N /* Number of indices defined */ } H5_index_t; /* @@ -353,9 +353,9 @@ H5_DLL herr_t H5get_free_list_sizes(size_t *reg_size, size_t *arr_size, size_t *blk_size, size_t *fac_size); H5_DLL herr_t H5get_alloc_stats(H5_alloc_stats_t *stats); H5_DLL herr_t H5get_libversion(unsigned *majnum, unsigned *minnum, - unsigned *relnum); + unsigned *relnum); H5_DLL herr_t H5check_version(unsigned majnum, unsigned minnum, - unsigned relnum); + unsigned relnum); H5_DLL herr_t H5is_library_threadsafe(hbool_t *is_ts); H5_DLL herr_t H5free_memory(void *mem); H5_DLL void *H5allocate_memory(size_t size, hbool_t clear); diff --git a/src/H5system.c b/src/H5system.c index a7ca3f8..e2747f4 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -13,11 +13,11 @@ /*------------------------------------------------------------------------- * - * Created: H5system.c - * Aug 21 2006 - * Quincey Koziol <koziol@hdfgroup.org> + * Created: H5system.c + * Aug 21 2006 + * Quincey Koziol * - * Purpose: System call wrapper implementations. + * Purpose: System call wrapper implementations. * *------------------------------------------------------------------------- */ @@ -30,10 +30,10 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5Fprivate.h" /* File access */ -#include "H5MMprivate.h" /* Memory management */ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File access */ +#include "H5MMprivate.h" /* Memory management */ /****************/ @@ -377,7 +377,7 @@ HDfprintf(FILE *stream, const char *fmt, ...) if(fwidth) len += HDsnprintf(format_templ + len, (sizeof(format_templ) - (size_t)(len + 1)), "%d", fwidth); HDstrncat(format_templ, "s", (sizeof(format_templ) - (size_t)(len + 1))); - fprintf(stream, format_templ, "UNDEF"); + n = fprintf(stream, format_templ, "UNDEF"); } } break; @@ -407,11 +407,25 @@ HDfprintf(FILE *stream, const char *fmt, ...) htri_t tri_var = HDva_arg(ap, htri_t); if(tri_var > 0) - fprintf(stream, "TRUE"); + n = fprintf(stream, "TRUE"); else if(!tri_var) - fprintf(stream, "FALSE"); + n = fprintf(stream, "FALSE"); else - fprintf(stream, "FAIL(%d)", (int)tri_var); + n = fprintf(stream, "FAIL(%d)", (int)tri_var); + } + break; + + case 'T': /* Elapsed time, in seconds */ + { + double seconds = HDva_arg(ap, double); + char *time_string = H5_timer_get_time_string(seconds); + + if(time_string) { + n = fprintf(stream, format_templ, time_string); + HDfree(time_string); + } /* end if */ + else + n = fprintf(stream, format_templ, "(error)"); } break; @@ -474,8 +488,6 @@ H5_GCC_DIAG_ON(format-nonliteral) * Programmer: Robb Matzke * Thursday, April 9, 1998 * - * Modifications: - * *------------------------------------------------------------------------- */ #ifndef HDstrtoll @@ -647,14 +659,14 @@ Pflock(int fd, int operation) { * Purpose: Wrapper function for systems where no file locking is * available. * - * Return: Failure: -1 (always fails) + * Return: 0 (success) * *------------------------------------------------------------------------- */ int H5_ATTR_CONST Nflock(int H5_ATTR_UNUSED fd, int H5_ATTR_UNUSED operation) { - /* just fail */ - return -1; + /* just succeed */ + return 0; } /* end Nflock() */ @@ -833,11 +845,88 @@ Wsetenv(const char *name, const char *value, int overwrite) #pragma comment(lib, "advapi32.lib") #endif + +/*------------------------------------------------------------------------- + * Function: H5_get_win32_times + * + * Purpose: Gets the elapsed, system and user times on Windows platforms. + * All time values are in seconds. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_WIN32_API +int +H5_get_win32_times(H5_timevals_t *tvs /*in,out*/) +{ + static HANDLE process_handle; + ULARGE_INTEGER kernel_start; + ULARGE_INTEGER user_start; + FILETIME KernelTime; + FILETIME UserTime; + FILETIME CreationTime; + FILETIME ExitTime; + LARGE_INTEGER counts_start; + static LARGE_INTEGER counts_freq; + static hbool_t is_initialized = FALSE; + BOOL err; + + HDassert(tvs); + + if(!is_initialized) { + /* NOTE: This is just a pseudo handle and does not need to be closed. */ + process_handle = GetCurrentProcess(); + err = QueryPerformanceFrequency(&counts_freq); + if(0 == err) + return -1; + is_initialized = TRUE; + } /* end if */ + + /************************* + * System and user times * + *************************/ + + err = GetProcessTimes(process_handle, &CreationTime, &ExitTime, &KernelTime, + &UserTime); + if(0 == err) + return -1; + + /* The 1.0E7 factor seems strange but it's due to the clock + * ticking in 100 ns increments. + */ + kernel_start.HighPart = KernelTime.dwHighDateTime; + kernel_start.LowPart = KernelTime.dwLowDateTime; + tvs->system = (double)(kernel_start.QuadPart / 1.0E7F); + + user_start.HighPart = UserTime.dwHighDateTime; + user_start.LowPart = UserTime.dwLowDateTime; + tvs->user = (double)(user_start.QuadPart / 1.0E7F); + + /**************** + * Elapsed time * + ****************/ + + err = QueryPerformanceCounter(&counts_start); + if(0 == err) + return -1; + + tvs->elapsed = (double)(counts_start.QuadPart) / (double)counts_freq.QuadPart; + + return 0; +} /* end H5_get_win32_times() */ +#endif + #define WloginBuffer_count 256 static char Wlogin_buffer[WloginBuffer_count]; + char* -Wgetlogin() +Wgetlogin(void) { #ifdef H5_HAVE_WINSOCK2_H @@ -1342,38 +1431,6 @@ H5_nanosleep(uint64_t nanosec) FUNC_LEAVE_NOAPI_VOID } /* end H5_nanosleep() */ - -/*-------------------------------------------------------------------------- - * Function: H5_get_time - * - * Purpose: Get the current time, as the time of seconds after the UNIX epoch - * - * Return: SUCCEED/FAIL - * - * Programmer: Quincey Koziol - * October 05, 2016 - *-------------------------------------------------------------------------- - */ -double -H5_get_time(void) -{ -#ifdef H5_HAVE_GETTIMEOFDAY - struct timeval curr_time; -#endif /* H5_HAVE_GETTIMEOFDAY */ - double ret_value = (double)0.0f; - - FUNC_ENTER_NOAPI_NOINIT_NOERR - -#ifdef H5_HAVE_GETTIMEOFDAY - HDgettimeofday(&curr_time, NULL); - - ret_value = (double)curr_time.tv_sec + ((double)curr_time.tv_usec / (double)1000000.0f); -#endif /* H5_HAVE_GETTIMEOFDAY */ - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5_get_time() */ - - #ifdef H5_HAVE_WIN32_API #define H5_WIN32_ENV_VAR_BUFFER_SIZE 32767 diff --git a/src/H5timer.c b/src/H5timer.c index 4b1ec06..61d6f0f 100644 --- a/src/H5timer.c +++ b/src/H5timer.c @@ -12,12 +12,11 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /*------------------------------------------------------------------------- - * * Created: H5timer.c * Aug 21 2006 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * - * Purpose: Internal 'timer' routines & support routines. + * Purpose: Internal, platform-independent 'timer' support routines. * *------------------------------------------------------------------------- */ @@ -32,20 +31,22 @@ /***********/ #include "H5private.h" /* Generic Functions */ -/* We need this for the struct rusage declaration */ -#if defined(H5_HAVE_GETRUSAGE) && defined(H5_HAVE_SYS_RESOURCE_H) -# include <sys/resource.h> -#endif - -#if defined(H5_HAVE_GETTIMEOFDAY) && defined(H5_HAVE_SYS_TIME_H) -#include <sys/time.h> -#endif - /****************/ /* Local Macros */ /****************/ +/* Size of a generated time string. + * Most time strings should be < 20 or so characters (max!) so this should be a + * safe size. Dynamically allocating the correct size would be painful. + */ +#define H5TIMER_TIME_STRING_LEN 1536 + +/* Conversion factors */ +#define H5_SEC_PER_DAY (double)(24.0F * 60.0F * 60.0F) +#define H5_SEC_PER_HOUR (double)(60.0F * 60.0F) +#define H5_SEC_PER_MIN (double)(60.0F) + /******************/ /* Local Typedefs */ @@ -76,104 +77,6 @@ /* Local Variables */ /*******************/ - -/*------------------------------------------------------------------------- - * Function: H5_timer_reset - * - * Purpose: Resets the timer struct to zero. Use this to reset a timer - * that's being used as an accumulator for summing times. - * - * Return: void - * - * Programmer: Robb Matzke - * Thursday, April 16, 1998 - * - *------------------------------------------------------------------------- - */ -void -H5_timer_reset (H5_timer_t *timer) -{ - HDassert(timer); - HDmemset(timer, 0, sizeof *timer); -} /* end H5_timer_reset() */ - - -/*------------------------------------------------------------------------- - * Function: H5_timer_begin - * - * Purpose: Initialize a timer to time something. - * - * Return: void - * - * Programmer: Robb Matzke - * Thursday, April 16, 1998 - * - *------------------------------------------------------------------------- - */ -void -H5_timer_begin (H5_timer_t *timer) -{ -#ifdef H5_HAVE_GETRUSAGE - struct rusage rusage; -#endif -#ifdef H5_HAVE_GETTIMEOFDAY - struct timeval etime; -#endif - - HDassert(timer); - -#ifdef H5_HAVE_GETRUSAGE - HDgetrusage (RUSAGE_SELF, &rusage); - timer->utime = (double)rusage.ru_utime.tv_sec + - ((double)rusage.ru_utime.tv_usec / (double)1e6F); - timer->stime = (double)rusage.ru_stime.tv_sec + - ((double)rusage.ru_stime.tv_usec / (double)1e6F); -#else - timer->utime = 0.0F; - timer->stime = 0.0F; -#endif -#ifdef H5_HAVE_GETTIMEOFDAY - HDgettimeofday (&etime, NULL); - timer->etime = (double)etime.tv_sec + ((double)etime.tv_usec / (double)1e6F); -#else - timer->etime = 0.0F; -#endif -} /* end H5_timer_begin() */ - - -/*------------------------------------------------------------------------- - * Function: H5_timer_end - * - * Purpose: This function should be called at the end of a timed region. - * The SUM is an optional pointer which will accumulate times. - * TMS is the same struct that was passed to H5_timer_start(). - * On return, TMS will contain total times for the timed region. - * - * Return: void - * - * Programmer: Robb Matzke - * Thursday, April 16, 1998 - * - *------------------------------------------------------------------------- - */ -void -H5_timer_end (H5_timer_t *sum/*in,out*/, H5_timer_t *timer/*in,out*/) -{ - H5_timer_t now; - - HDassert(timer); - H5_timer_begin(&now); - - timer->utime = MAX((double)0.0F, now.utime - timer->utime); - timer->stime = MAX((double)0.0F, now.stime - timer->stime); - timer->etime = MAX((double)0.0F, now.etime - timer->etime); - - if (sum) { - sum->utime += timer->utime; - sum->stime += timer->stime; - sum->etime += timer->etime; - } -} /* end H5_timer_end() */ /*------------------------------------------------------------------------- @@ -209,32 +112,35 @@ H5_bandwidth(char *buf/*out*/, double nbytes, double nseconds) if(nseconds <= (double)0.0F) HDstrcpy(buf, " NaN"); else { - bw = nbytes/nseconds; + bw = nbytes / nseconds; if(H5_DBL_ABS_EQUAL(bw, (double)0.0F)) HDstrcpy(buf, "0.000 B/s"); else if(bw < (double)1.0F) HDsprintf(buf, "%10.4e", bw); else if(bw < (double)H5_KB) { HDsprintf(buf, "%05.4f", bw); - HDstrcpy(buf+5, " B/s"); + HDstrcpy(buf + 5, " B/s"); } else if(bw < (double)H5_MB) { HDsprintf(buf, "%05.4f", bw / (double)H5_KB); - HDstrcpy(buf+5, " kB/s"); + HDstrcpy(buf + 5, " kB/s"); } else if(bw < (double)H5_GB) { HDsprintf(buf, "%05.4f", bw / (double)H5_MB); - HDstrcpy(buf+5, " MB/s"); + HDstrcpy(buf + 5, " MB/s"); } else if(bw < (double)H5_TB) { HDsprintf(buf, "%05.4f", bw / (double)H5_GB); - HDstrcpy(buf+5, " GB/s"); + HDstrcpy(buf + 5, " GB/s"); } else if(bw < (double)H5_PB) { HDsprintf(buf, "%05.4f", bw / (double)H5_TB); - HDstrcpy(buf+5, " TB/s"); + HDstrcpy(buf + 5, " TB/s"); + } else if(bw < (double)H5_EB) { + HDsprintf(buf, "%05.4f", bw / (double)H5_PB); + HDstrcpy(buf + 5, " PB/s"); } else { HDsprintf(buf, "%10.4e", bw); if(HDstrlen(buf) > 10) HDsprintf(buf, "%10.3e", bw); - } - } + } /* end else-if */ + } /* end else */ } /* end H5_bandwidth() */ @@ -269,3 +175,488 @@ H5_now(void) return(now); } /* end H5_now() */ + +/*------------------------------------------------------------------------- + * Function: H5_now_usec + * + * Purpose: Retrieves the current time, as microseconds after the UNIX epoch. + * + * Return: # of microseconds from the epoch (can't fail) + * + * Programmer: Quincey Koziol + * Tuesday, November 28, 2006 + * + *------------------------------------------------------------------------- + */ +uint64_t +H5_now_usec(void) +{ + uint64_t now; /* Current time, in microseconds */ + +#if defined(H5_HAVE_CLOCK_GETTIME) + { + struct timespec ts; + + HDclock_gettime(CLOCK_MONOTONIC, &ts); + now = (uint64_t)(ts.tv_sec * (1000 * 1000)) + (uint64_t)(ts.tv_nsec / 1000); + } +#elif defined(H5_HAVE_GETTIMEOFDAY) + { + struct timeval now_tv; + + HDgettimeofday(&now_tv, NULL); + now = (uint64_t)(now_tv.tv_sec * (1000 * 1000)) + (uint64_t)now_tv.tv_usec; + } +#else /* H5_HAVE_GETTIMEOFDAY */ + now = (uint64_t)(HDtime(NULL) * (1000 * 1000)); +#endif /* H5_HAVE_GETTIMEOFDAY */ + + return(now); +} /* end H5_now_usec() */ + + +/*-------------------------------------------------------------------------- + * Function: H5_get_time + * + * Purpose: Get the current time, as the time of seconds after the UNIX epoch + * + * Return: Success: A non-negative time value + * Failure: -1.0 (in theory, can't currently fail) + * + * Programmer: Quincey Koziol + * October 05, 2016 + *-------------------------------------------------------------------------- + */ +double +H5_get_time(void) +{ + double ret_value = (double)0.0f; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if defined(H5_HAVE_CLOCK_GETTIME) + { + struct timespec ts; + + HDclock_gettime(CLOCK_MONOTONIC, &ts); + ret_value = (double)ts.tv_sec + ((double)ts.tv_nsec / (double)1000000000.0f); + } +#elif defined(H5_HAVE_GETTIMEOFDAY) + { + struct timeval now_tv; + + HDgettimeofday(&now_tv, NULL); + ret_value = (double)now_tv.tv_sec + ((double)now_tv.tv_usec / (double)1000000.0f); + } +#else + ret_value = (double)HDtime(NULL); +#endif + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5_get_time() */ + + + +/*------------------------------------------------------------------------- + * Function: H5__timer_get_timevals + * + * Purpose: Internal platform-specific function to get time system, + * user and elapsed time values. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5__timer_get_timevals(H5_timevals_t *times /*in,out*/) +{ + /* Sanity check */ + HDassert(times); + + /* Windows call handles both system/user and elapsed times */ +#ifdef H5_HAVE_WIN32_API + if(H5_get_win32_times(times) < 0) { + times->elapsed = -1.0; + times->system = -1.0; + times->user = -1.0; + + return -1; + } /* end if */ +#else /* H5_HAVE_WIN32_API */ + + /************************* + * System and user times * + *************************/ +#if defined(H5_HAVE_GETRUSAGE) +{ + struct rusage res; + + if(HDgetrusage(RUSAGE_SELF, &res) < 0) + return -1; + times->system = (double)res.ru_stime.tv_sec + ((double)res.ru_stime.tv_usec / (double)1.0E6F); + times->user = (double)res.ru_utime.tv_sec + ((double)res.ru_utime.tv_usec / (double)1.0E6F); +} +#else + /* No suitable way to get system/user times */ + /* This is not an error condition, they just won't be available */ + times->system = -1.0; + times->user = -1.0; +#endif + + /**************** + * Elapsed time * + ****************/ + + times->elapsed = H5_get_time(); + +#endif /* H5_HAVE_WIN32_API */ + + return 0; +} /* end H5__timer_get_timevals() */ + + +/*------------------------------------------------------------------------- + * Function: H5_timer_init + * + * Purpose: Initialize a platform-independent timer. + * + * Timer usage is as follows: + * + * 1) Call H5_timer_init(), passing in a timer struct, to set + * up the timer. + * + * 2) Wrap any code you'd like to time with calls to + * H5_timer_start/stop(). For accurate timing, place these + * calls as close to the code of interest as possible. You + * can call start/stop multiple times on the same timer - + * when you do this, H5_timer_get_times() will return time + * values for the current/last session and + * H5_timer_get_total_times() will return the summed times + * of all sessions (see #3 and #4, below). + * + * 3) Use H5_timer_get_times() to get the current system, user + * and elapsed times from a running timer. If called on a + * stopped timer, this will return the time recorded at the + * stop point. + * + * 4) Call H5_timer_get_total_times() to get the total system, + * user and elapsed times recorded across multiple start/stop + * sessions. If called on a running timer, it will return the + * time recorded up to that point. On a stopped timer, it + * will return the time recorded at the stop point. + * + * NOTE: Obtaining a time point is not free! Keep in mind that + * the time functions make system calls and can have + * non-trivial overhead. If you call one of the get_time + * functions on a running timer, that overhead will be + * added to the reported times. + * + * 5) All times recorded will be in seconds. These can be + * converted into human-readable strings with the + * H5_timer_get_time_string() function. + * + * 6) A timer can be reset using by calling H5_timer_init() on + * it. This will set its state to 'stopped' and reset all + * accumulated times to zero. + * + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +herr_t +H5_timer_init(H5_timer_t *timer /*in,out*/) +{ + /* Sanity check */ + HDassert(timer); + + /* Initialize everything */ + HDmemset(timer, 0, sizeof(H5_timer_t)); + + return 0; +} /* end H5_timer_init() */ + + +/*------------------------------------------------------------------------- + * Function: H5_timer_start + * + * Purpose: Start tracking time in a platform-independent timer. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +herr_t +H5_timer_start(H5_timer_t *timer /*in,out*/) +{ + /* Sanity check */ + HDassert(timer); + + /* Start the timer + * This sets the "initial" times to the system-defined start times. + */ + if(H5__timer_get_timevals(&(timer->initial)) < 0) + return -1; + + timer->is_running = TRUE; + + return 0; +} /* end H5_timer_start() */ + + +/*------------------------------------------------------------------------- + * Function: H5_timer_stop + * + * Purpose: Stop tracking time in a platform-independent timer. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +herr_t +H5_timer_stop(H5_timer_t *timer /*in,out*/) +{ + /* Sanity check */ + HDassert(timer); + + /* Stop the timer */ + if(H5__timer_get_timevals(&(timer->final_interval)) < 0) + return -1; + + /* The "final" times are stored as intervals (final - initial) + * for more useful reporting to the user. + */ + timer->final_interval.elapsed = timer->final_interval.elapsed - timer->initial.elapsed; + timer->final_interval.system = timer->final_interval.system - timer->initial.system; + timer->final_interval.user = timer->final_interval.user - timer->initial.user; + + /* Add the intervals to the elapsed time */ + timer->total.elapsed += timer->final_interval.elapsed; + timer->total.system += timer->final_interval.system; + timer->total.user += timer->final_interval.user; + + timer->is_running = FALSE; + + return 0; +} /* end H5_timer_stop() */ + + +/*------------------------------------------------------------------------- + * Function: H5_timer_get_times + * + * Purpose: Get the system, user and elapsed times from a timer. These + * are the times since the timer was last started and will be + * 0.0 in a timer that has not been started since it was + * initialized. + * + * This function can be called either before or after + * H5_timer_stop() has been called. If it is called before the + * stop function, the timer will continue to run. + * + * The system and user times will be -1.0 if those times cannot + * be computed on a particular platform. The elapsed time will + * always be present. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +herr_t +H5_timer_get_times(H5_timer_t timer, H5_timevals_t *times /*in,out*/) +{ + /* Sanity check */ + HDassert(times); + + if(timer.is_running) { + H5_timevals_t now; + + /* Get the current times and report the current intervals without + * stopping the timer. + */ + if(H5__timer_get_timevals(&now) < 0) + return -1; + + times->elapsed = now.elapsed - timer.initial.elapsed; + times->system = now.system - timer.initial.system; + times->user = now.user - timer.initial.user; + } /* end if */ + else { + times->elapsed = timer.final_interval.elapsed; + times->system = timer.final_interval.system; + times->user = timer.final_interval.user; + } /* end else */ + + return 0; +} /* end H5_timer_get_times() */ + + +/*------------------------------------------------------------------------- + * Function: H5_timer_get_total_times + * + * Purpose: Get the TOTAL system, user and elapsed times recorded by + * the timer since its initialization. This is the sum of all + * times recorded while the timer was running. + * + * These will be 0.0 in a timer that has not been started + * since it was initialized. Calling H5_timer_init() on a + * timer will reset these values to 0.0. + * + * This function can be called either before or after + * H5_timer_stop() has been called. If it is called before the + * stop function, the timer will continue to run. + * + * The system and user times will be -1.0 if those times cannot + * be computed on a particular platform. The elapsed time will + * always be present. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +herr_t +H5_timer_get_total_times(H5_timer_t timer, H5_timevals_t *times /*in,out*/) +{ + /* Sanity check */ + HDassert(times); + + if(timer.is_running) { + H5_timevals_t now; + + /* Get the current times and report the current totals without + * stopping the timer. + */ + if(H5__timer_get_timevals(&now) < 0) + return -1; + + times->elapsed = timer.total.elapsed + (now.elapsed - timer.initial.elapsed); + times->system = timer.total.system + (now.system - timer.initial.system); + times->user = timer.total.user + (now.user - timer.initial.user); + } /* end if */ + else { + times->elapsed = timer.total.elapsed; + times->system = timer.total.system; + times->user = timer.total.user; + } /* end else */ + + return 0; +} /* end H5_timer_get_total_times() */ + + +/*------------------------------------------------------------------------- + * Function: H5_timer_get_time_string + * + * Purpose: Converts a time (in seconds) into a human-readable string + * suitable for log messages. + * + * Return: Success: The time string. + * + * The general format of the time string is: + * + * "N/A" time < 0 (invalid time) + * "%.f ns" time < 1 microsecond + * "%.1f us" time < 1 millisecond + * "%.1f ms" time < 1 second + * "%.2f s" time < 1 minute + * "%.f m %.f s" time < 1 hour + * "%.f h %.f m %.f s" longer times + * + * Failure: NULL + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +char * +H5_timer_get_time_string(double seconds) +{ + char *s; /* output string */ + + /* Used when the time is greater than 59 seconds */ + double days; + double hours; + double minutes; + double remainder_sec; + + /* Extract larger time units from count of seconds */ + if(seconds > (double)60.0F) { + /* Set initial # of seconds */ + remainder_sec = seconds; + + /* Extract days */ + days = HDfloor(remainder_sec / H5_SEC_PER_DAY); + remainder_sec -= (days * H5_SEC_PER_DAY); + + /* Extract hours */ + hours = HDfloor(remainder_sec / H5_SEC_PER_HOUR); + remainder_sec -= (hours * H5_SEC_PER_HOUR); + + /* Extract minutes */ + minutes = HDfloor(remainder_sec / H5_SEC_PER_MIN); + remainder_sec -= (minutes * H5_SEC_PER_MIN); + + /* The # of seconds left is in remainder_sec */ + } /* end if */ + + /* Allocate */ + if(NULL == (s = (char *)HDcalloc(H5TIMER_TIME_STRING_LEN, sizeof(char)))) + return NULL; + + /* Do we need a format string? Some people might like a certain + * number of milliseconds or s before spilling to the next highest + * time unit. Perhaps this could be passed as an integer. + * (name? round_up_size? ?) + */ + if(seconds < (double)0.0F) + HDsprintf(s, "N/A"); + else if(H5_DBL_ABS_EQUAL((double)0.0F, seconds)) + HDsprintf(s, "0.0 s"); + else if(seconds < (double)1.0E-6F) + /* t < 1 us, Print time in ns */ + HDsprintf(s, "%.f ns", seconds * (double)1.0E9F); + else if(seconds < (double)1.0E-3F) + /* t < 1 ms, Print time in us */ + HDsprintf(s, "%.1f us", seconds * (double)1.0E6F); + else if(seconds < (double)1.0F) + /* t < 1 s, Print time in ms */ + HDsprintf(s, "%.1f ms", seconds * (double)1.0E3F); + else if(seconds < H5_SEC_PER_MIN) + /* t < 1 m, Print time in s */ + HDsprintf(s, "%.2f s", seconds); + else if(seconds < H5_SEC_PER_HOUR) + /* t < 1 h, Print time in m and s */ + HDsprintf(s, "%.f m %.f s", minutes, remainder_sec); + else if(seconds < H5_SEC_PER_DAY) + /* t < 1 d, Print time in h, m and s */ + HDsprintf(s, "%.f h %.f m %.f s", hours, minutes, remainder_sec); + else + /* Print time in d, h, m and s */ + HDsprintf(s, "%.f d %.f h %.f m %.f s", days, hours, minutes, remainder_sec); + + return s; +} /* end H5_timer_get_time_string() */ + diff --git a/src/H5trace.c b/src/H5trace.c index 93ff681..cf14565 100644 --- a/src/H5trace.c +++ b/src/H5trace.c @@ -15,7 +15,7 @@ * * Created: H5trace.c * Aug 21 2006 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * * Purpose: Internal code for tracing API calls * @@ -125,8 +125,11 @@ H5_trace(const double *returning, const char *func, const char *type, ...) hssize_t i; void *vp = NULL; FILE *out = H5_debug_g.trace; - H5_timer_t event_time; - static H5_timer_t first_time = {0.0F, 0.0F, 0.0F}; + static hbool_t is_first_invocation = TRUE; + H5_timer_t function_timer; + H5_timevals_t function_times; + static H5_timer_t running_timer; + H5_timevals_t running_times; static int current_depth = 0; static int last_call_depth = 0; @@ -152,13 +155,18 @@ H5_trace(const double *returning, const char *func, const char *type, ...) } /* end else */ } /* end if */ - /* Get time for event */ - if(H5_DBL_ABS_EQUAL(first_time.etime, (double)0.0f)) - H5_timer_begin(&first_time); - if(H5_debug_g.ttimes) - H5_timer_begin(&event_time); - else - HDmemset(&event_time, 0, sizeof event_time); + /* Get time for event if the trace times flag is set */ + if(is_first_invocation && H5_debug_g.ttimes) { + /* start the library-wide timer */ + is_first_invocation = FALSE; + H5_timer_init(&running_timer); + H5_timer_start(&running_timer); + } /* end if */ + if(H5_debug_g.ttimes) { + /* start the timer for this function */ + H5_timer_init(&function_timer); + H5_timer_start(&function_timer); + } /* end if */ /* Print the first part of the line. This is the indication of the * nesting depth followed by the function name and either start of @@ -174,7 +182,9 @@ H5_trace(const double *returning, const char *func, const char *type, ...) if(H5_debug_g.ttimes) { char tmp[320]; - HDsprintf(tmp, "%.6f", event_time.etime-first_time.etime); + H5_timer_get_times(function_timer, &function_times); + H5_timer_get_times(running_timer, &running_times); + HDsprintf(tmp, "%.6f", (function_times.elapsed - running_times.elapsed)); HDfprintf(out, " %*s ", (int)HDstrlen(tmp), ""); } /* end if */ for(i = 0; i < current_depth; i++) @@ -189,8 +199,11 @@ H5_trace(const double *returning, const char *func, const char *type, ...) else { if(current_depth>last_call_depth) HDfputs(" = <delayed>\n", out); - if(H5_debug_g.ttimes) - HDfprintf(out, "@%.6f ", event_time.etime - first_time.etime); + if(H5_debug_g.ttimes) { + H5_timer_get_times(function_timer, &function_times); + H5_timer_get_times(running_timer, &running_times); + HDfprintf(out, "@%.6f ", (function_times.elapsed - running_times.elapsed)); + } /* end if */ for(i = 0; i < current_depth; i++) HDfputc('+', out); HDfprintf(out, "%*s%s(", 2*current_depth, "", func); @@ -296,7 +309,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) case 'a': if(ptr) { if(vp) - HDfprintf (out, "0x%p", vp); + HDfprintf(out, "0x%p", vp); else HDfprintf(out, "NULL"); } /* end if */ @@ -749,7 +762,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) break; default: - HDfprintf (out, "BADTYPE(D%c)", type[1]); + HDfprintf(out, "BADTYPE(D%c)", type[1]); goto error; } /* end switch */ break; @@ -1095,7 +1108,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) case 's': if(ptr) { if(vp) - HDfprintf (out, "0x%p", vp); + HDfprintf(out, "0x%p", vp); else HDfprintf(out, "NULL"); } /* end if */ @@ -1172,7 +1185,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) break; default: - HDfprintf (out, "BADTYPE(H%c)", type[1]); + HDfprintf(out, "BADTYPE(H%c)", type[1]); goto error; } /* end switch */ break; @@ -1356,7 +1369,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) break; case H5I_NTYPES: - HDfprintf (out, "%ld (ntypes - error)", (long)obj); + HDfprintf(out, "%ld (ntypes - error)", (long)obj); break; default: @@ -1460,7 +1473,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) else { int is = HDva_arg(ap, int); - HDfprintf (out, "%d", is); + HDfprintf(out, "%d", is); asize[argno] = is; } /* end else */ break; @@ -1576,7 +1589,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) break; default: - HDfprintf (out, "BADTYPE(I%c)", type[1]); + HDfprintf(out, "BADTYPE(I%c)", type[1]); goto error; } /* end switch */ break; @@ -1586,7 +1599,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) case 'l': if(ptr) { if(vp) - HDfprintf (out, "0x%p", vp); + HDfprintf(out, "0x%p", vp); else HDfprintf(out, "NULL"); } /* end if */ @@ -1730,7 +1743,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) else { off_t offset = HDva_arg(ap, off_t); - HDfprintf (out, "%ld", (long)offset); + HDfprintf(out, "%ld", (long)offset); } /* end else */ break; @@ -2394,7 +2407,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) break; default: - HDfprintf (out, "BADTYPE(T%c)", type[1]); + HDfprintf(out, "BADTYPE(T%c)", type[1]); goto error; } /* end switch */ break; @@ -2410,9 +2423,9 @@ H5_trace(const double *returning, const char *func, const char *type, ...) htri_t tri_var = HDva_arg (ap, htri_t); if(tri_var>0) - HDfprintf (out, "TRUE"); + HDfprintf(out, "TRUE"); else if(!tri_var) - HDfprintf (out, "FALSE"); + HDfprintf(out, "FALSE"); else HDfprintf(out, "FAIL(%d)", (int)tri_var); } /* end else */ @@ -2469,7 +2482,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) break; default: - HDfprintf (out, "BADTYPE(U%c)", type[1]); + HDfprintf(out, "BADTYPE(U%c)", type[1]); goto error; } /* end switch */ break; @@ -2650,9 +2663,12 @@ H5_trace(const double *returning, const char *func, const char *type, ...) } /* end for */ /* Display event time for return */ - if(returning && H5_debug_g.ttimes) - HDfprintf(out, " @%.6f [dt=%.6f]", (event_time.etime - first_time.etime), - (event_time.etime - *returning)); + if(returning && H5_debug_g.ttimes) { + H5_timer_get_times(function_timer, &function_times); + H5_timer_get_times(running_timer, &running_times); + HDfprintf(out, " @%.6f [dt=%.6f]", (function_times.elapsed - running_times.elapsed), + (function_times.elapsed - *returning)); + } /* end if */ error: HDva_end(ap); @@ -2660,10 +2676,13 @@ error: HDfprintf(out, ";\n"); else { last_call_depth = current_depth++; - HDfprintf (out, ")"); + HDfprintf(out, ")"); } /* end else */ HDfflush(out); - return event_time.etime; + if(H5_debug_g.ttimes) + return function_times.elapsed; + else + return 0.0F; } /* end H5_trace() */ diff --git a/src/Makefile.am b/src/Makefile.am index 0ec6c41..05a5856 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -63,7 +63,8 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5FA.c H5FAcache.c H5FAdbg.c H5FAdblock.c H5FAdblkpage.c H5FAhdr.c \ H5FAint.c H5FAstat.c H5FAtest.c \ H5FD.c H5FDcore.c H5FDfamily.c H5FDhdfs.c H5FDint.c H5FDlog.c \ - H5FDmulti.c H5FDsec2.c H5FDspace.c H5FDstdio.c H5FDtest.c \ + H5FDmulti.c H5FDsec2.c H5FDspace.c \ + H5FDsplitter.c H5FDstdio.c H5FDtest.c \ H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSint.c H5FSsection.c \ H5FSstat.c H5FStest.c \ H5G.c H5Gbtree2.c H5Gcache.c \ @@ -126,6 +127,11 @@ if DIRECT_VFD_CONDITIONAL libhdf5_la_SOURCES += H5FDdirect.c endif +# Only compile the mirror VFD if necessary +if MIRROR_VFD_CONDITIONAL + libhdf5_la_SOURCES += H5FDmirror.c +endif + # Only compile the read-only S3 VFD if necessary if ROS3_VFD_CONDITIONAL libhdf5_la_SOURCES += H5FDros3.c H5FDs3comms.c @@ -137,8 +143,8 @@ include_HEADERS = hdf5.h H5api_adpt.h H5overflow.h H5pubconf.h H5public.h H5vers H5Cpublic.h H5Dpublic.h \ H5Epubgen.h H5Epublic.h H5Fpublic.h \ H5FDpublic.h H5FDcore.h H5FDdirect.h H5FDfamily.h H5FDhdfs.h \ - H5FDlog.h H5FDmpi.h H5FDmpio.h H5FDmulti.h H5FDros3.h \ - H5FDsec2.h H5FDstdio.h H5FDwindows.h \ + H5FDlog.h H5FDmirror.h H5FDmpi.h H5FDmpio.h H5FDmulti.h H5FDros3.h \ + H5FDsec2.h H5FDsplitter.h H5FDstdio.h H5FDwindows.h \ H5Gpublic.h H5Ipublic.h H5Lpublic.h \ H5MMpublic.h H5Opublic.h H5Ppublic.h \ H5PLextern.h H5PLpublic.h \ @@ -44,10 +44,12 @@ #include "H5FDfamily.h" /* File families */ #include "H5FDhdfs.h" /* Hadoop HDFS */ #include "H5FDlog.h" /* sec2 driver with I/O logging (for debugging) */ +#include "H5FDmirror.h" /* Mirror VFD and IPC definitions */ #include "H5FDmpi.h" /* MPI-based file drivers */ #include "H5FDmulti.h" /* Usage-partitioned file family */ #include "H5FDros3.h" /* R/O S3 "file" I/O */ #include "H5FDsec2.h" /* POSIX unbuffered file I/O */ +#include "H5FDsplitter.h" /* Twin-channel (R/W & R/O) I/O passthrough */ #include "H5FDstdio.h" /* Standard C buffered I/O */ #ifdef H5_HAVE_WINDOWS #include "H5FDwindows.h" /* Win32 I/O */ diff --git a/src/libhdf5.settings.in b/src/libhdf5.settings.in index 422ca0b..128e107 100644 --- a/src/libhdf5.settings.in +++ b/src/libhdf5.settings.in @@ -79,6 +79,7 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ I/O filters (external): @EXTERNAL_FILTERS@ MPE: @MPE@ Direct VFD: @DIRECT_VFD@ + Mirror VFD: @MIRROR_VFD@ (Read-Only) S3 VFD: @ROS3_VFD@ (Read-Only) HDFS VFD: @HAVE_LIBHDFS@ dmalloc: @HAVE_DMALLOC@ @@ -87,5 +88,6 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ Using memory checker: @USINGMEMCHECKER@ Memory allocation sanity checks: @MEMORYALLOCSANITYCHECK@ Function stack tracing: @CODESTACK@ + Use file locking: @DESIRED_FILE_LOCKING@ Strict file format checks: @STRICT_FORMAT_CHECKS@ Optimization instrumentation: @INSTRUMENT_LIBRARY@ |