diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/CMakeLists.txt | 4 | ||||
| -rw-r--r-- | src/H5FDmirror.c | 1991 | ||||
| -rw-r--r-- | src/H5FDmirror.h | 371 | ||||
| -rw-r--r-- | src/H5FDpublic.h | 2 | ||||
| -rw-r--r-- | src/H5FDsec2.c | 6 | ||||
| -rw-r--r-- | src/H5FDsplitter.c | 1467 | ||||
| -rw-r--r-- | src/H5FDsplitter.h | 99 | ||||
| -rw-r--r-- | src/H5private.h | 66 | ||||
| -rw-r--r-- | src/Makefile.am | 9 | ||||
| -rw-r--r-- | src/hdf5.h | 2 | ||||
| -rw-r--r-- | src/libhdf5.settings.in | 1 |
11 files changed, 4013 insertions, 5 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f582056..6678aaf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -231,6 +231,7 @@ set (H5FD_SOURCES ${HDF5_SRC_DIR}/H5FDhdfs.c ${HDF5_SRC_DIR}/H5FDint.c ${HDF5_SRC_DIR}/H5FDlog.c + ${HDF5_SRC_DIR}/H5FDmirror.c ${HDF5_SRC_DIR}/H5FDmpi.c ${HDF5_SRC_DIR}/H5FDmpio.c ${HDF5_SRC_DIR}/H5FDmulti.c @@ -238,6 +239,7 @@ set (H5FD_SOURCES ${HDF5_SRC_DIR}/H5FDs3comms.c ${HDF5_SRC_DIR}/H5FDsec2.c ${HDF5_SRC_DIR}/H5FDspace.c + ${HDF5_SRC_DIR}/H5FDsplitter.c ${HDF5_SRC_DIR}/H5FDstdio.c ${HDF5_SRC_DIR}/H5FDtest.c ${HDF5_SRC_DIR}/H5FDwindows.c @@ -249,6 +251,7 @@ set (H5FD_HDRS ${HDF5_SRC_DIR}/H5FDfamily.h ${HDF5_SRC_DIR}/H5FDhdfs.h ${HDF5_SRC_DIR}/H5FDlog.h + ${HDF5_SRC_DIR}/H5FDmirror.h ${HDF5_SRC_DIR}/H5FDmpi.h ${HDF5_SRC_DIR}/H5FDmpio.h ${HDF5_SRC_DIR}/H5FDmulti.h @@ -256,6 +259,7 @@ set (H5FD_HDRS ${HDF5_SRC_DIR}/H5FDros3.h ${HDF5_SRC_DIR}/H5FDs3comms.h ${HDF5_SRC_DIR}/H5FDsec2.h + ${HDF5_SRC_DIR}/H5FDsplitter.h ${HDF5_SRC_DIR}/H5FDstdio.h ${HDF5_SRC_DIR}/H5FDwindows.h ) diff --git a/src/H5FDmirror.c b/src/H5FDmirror.c new file mode 100644 index 0000000..ea1f516 --- /dev/null +++ b/src/H5FDmirror.c @@ -0,0 +1,1991 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Transmit write-only operations to a receiver/writer process on + * a remote host. + */ + +#include "H5FDdrvr_module.h" /* This source code file is part of the H5FD driver module */ + +#include "H5private.h" /* Generic Functions */ + +#ifdef H5_HAVE_MIRROR_VFD + +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File access */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5FDmirror.h" /* "Mirror" definitions */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5Pprivate.h" /* Property lists */ + +/* The driver identification number, initialized at runtime */ +static hid_t H5FD_MIRROR_g = 0; + +/* Virtual file structure for a Mirror Driver */ +typedef struct H5FD_mirror_t { + H5FD_t pub; /* Public stuff, must be first */ + H5FD_mirror_fapl_t fa; /* Configuration structure */ + haddr_t eoa; /* End of allocated region */ + haddr_t eof; /* End of file; current file size */ + int sock_fd; /* Handle of socket to remote operator */ + H5FD_mirror_xmit_t xmit; /* Primary communication header */ + uint32_t xmit_i; /* Counter of transmission sent and rec'd */ +} H5FD_mirror_t; + +/* + * These macros check for overflow of various quantities. These macros + * assume that HDoff_t is signed and haddr_t and size_t are unsigned. + * + * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t' + * is too large to be represented by the second argument + * of the file seek function. + * + * SIZE_OVERFLOW: Checks whether a buffer size of type `hsize_t' is too + * large to be represented by the `size_t' type. + * + * REGION_OVERFLOW: Checks whether an address and size pair describe data + * which can be addressed entirely by the second + * argument of the file seek function. + */ +#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) +#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR)) + +#define _BSWAP_64(X) \ + (uint64_t)( (((X) & 0x00000000000000FF) << 56) \ + | (((X) & 0x000000000000FF00) << 40) \ + | (((X) & 0x0000000000FF0000) << 24) \ + | (((X) & 0x00000000FF000000) << 8) \ + | (((X) & 0x000000FF00000000) >> 8) \ + | (((X) & 0x0000FF0000000000) >> 24) \ + | (((X) & 0x00FF000000000000) >> 40) \ + | (((X) & 0xFF00000000000000) >> 56)) + +/* Debugging flabs for verbose tracing -- nonzero to enable */ +#define MIRROR_DEBUG_OP_CALLS 0 +#define MIRROR_DEBUG_XMIT_BYTES 0 + +#if MIRROR_DEBUG_XMIT_BYTES +#define LOG_XMIT_BYTES(label, buf, len) do { \ + ssize_t bytes_written = 0; \ + const unsigned char *b = NULL; \ + \ + HDfprintf(stdout, "%s bytes:\n```\n", (label)); \ + \ + /* print whole lines */ \ + while ((len - bytes_written) >= 32) { \ + b = (const unsigned char *)(buf) + bytes_written; \ + HDfprintf(stdout, \ + "%04zX %02X%02X%02X%02X %02X%02X%02X%02X" \ + " %02X%02X%02X%02X %02X%02X%02X%02X" \ + " %02X%02X%02X%02X %02X%02X%02X%02X" \ + " %02X%02X%02X%02X %02X%02X%02X%02X\n", \ + bytes_written, \ + b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], \ + b[8], b[9], b[10],b[11], b[12],b[13],b[14],b[15], \ + b[16],b[17],b[18],b[19], b[20],b[21],b[22],b[23], \ + b[24],b[25],b[26],b[27], b[28],b[29],b[30],b[31]); \ + bytes_written += 32; \ + } \ + \ + /* start partial line */ \ + if (len > bytes_written) { \ + HDfprintf(stdout, "%04zX ", bytes_written); \ + } \ + \ + /* partial line blocks */ \ + while ((len - bytes_written) >= 4) { \ + HDfprintf(stdout, " %02X%02X%02X%02X", \ + (buf)[bytes_written], (buf)[bytes_written+1], \ + (buf)[bytes_written+2], (buf)[bytes_written+3]); \ + bytes_written += 4; \ + } \ + \ + /* block separator before partial block */ \ + if (len > bytes_written) { \ + HDfprintf(stdout, " "); \ + } \ + \ + /* partial block individual bytes */ \ + while (len > bytes_written) { \ + HDfprintf(stdout, "%02X", (buf)[bytes_written++]); \ + } \ + \ + /* end partial line */ \ + HDfprintf(stdout, "\n"); \ + HDfprintf(stdout, "```\n"); \ + HDfflush(stdout); \ +} while (0) +#else +#define LOG_XMIT_BYTES(label, buf, len) /* no-op */ +#endif /* MIRROR_DEBUG_XMIT_BYTE */ + +#if MIRROR_DEBUG_OP_CALLS +#define LOG_OP_CALL(name) do { \ + HDprintf("called %s()\n", (name)); \ + fflush(stdout); \ +} while (0) +#else +#define LOG_OP_CALL(name) /* no-op */ +#endif /* MIRROR_DEBUG_OP_CALLS */ + +/* Prototypes */ +static herr_t H5FD_mirror_term(void); +static void *H5FD_mirror_fapl_get(H5FD_t *_file); +static void *H5FD_mirror_fapl_copy(const void *_old_fa); +static herr_t H5FD_mirror_fapl_free(void *_fa); +static haddr_t H5FD_mirror_get_eoa(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD_mirror_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); +static haddr_t H5FD_mirror_get_eof(const H5FD_t *_file, H5FD_mem_t type); +static H5FD_t *H5FD_mirror_open(const char *name, unsigned flags, \ + hid_t fapl_id, haddr_t maxaddr); +static herr_t H5FD_mirror_close(H5FD_t *_file); +static herr_t H5FD_mirror_query(const H5FD_t *_file, unsigned long *flags); +static herr_t H5FD_mirror_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, \ + haddr_t addr, size_t size, const void *buf); +static herr_t H5FD_mirror_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, + haddr_t addr, size_t size, void *buf); +static herr_t H5FD_mirror_truncate(H5FD_t *_file, hid_t dxpl_id, \ + hbool_t closing); +static herr_t H5FD_mirror_lock(H5FD_t *_file, hbool_t rw); +static herr_t H5FD_mirror_unlock(H5FD_t *_file); + +static herr_t H5FD__mirror_verify_reply(H5FD_mirror_t *file); + +static const H5FD_class_t H5FD_mirror_g = { + "mirror", /* name */ + MAXADDR, /* maxaddr */ + H5F_CLOSE_WEAK, /* fc_degree */ + H5FD_mirror_term, /* terminate */ + NULL, /* sb_size */ + NULL, /* sb_encode */ + NULL, /* sb_decode */ + 0, /* fapl_size */ + H5FD_mirror_fapl_get, /* fapl_get */ + H5FD_mirror_fapl_copy, /* fapl_copy */ + H5FD_mirror_fapl_free, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD_mirror_open, /* open */ + H5FD_mirror_close, /* close */ + NULL, /* cmp */ + H5FD_mirror_query, /* query */ + NULL, /* get_type_map */ + NULL, /* alloc */ + NULL, /* free */ + H5FD_mirror_get_eoa, /* get_eoa */ + H5FD_mirror_set_eoa, /* set_eoa */ + H5FD_mirror_get_eof, /* get_eof */ + NULL, /* get_handle */ + H5FD_mirror_read, /* read */ + H5FD_mirror_write, /* write */ + NULL, /* flush */ + H5FD_mirror_truncate, /* truncate */ + H5FD_mirror_lock, /* lock */ + H5FD_mirror_unlock, /* unlock */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ +}; + +/* Declare a free list to manage the H5FD_mirror_t struct */ +H5FL_DEFINE_STATIC(H5FD_mirror_t); + + +/*------------------------------------------------------------------------- + * Function: H5FD__init_package + * + * Purpose: Initializes any interface-specific data or routines. + * + * Return: Non-negative on success/Negative on failure + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__init_package(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + LOG_OP_CALL("H5FD__init_package"); + + if (H5FD_mirror_init() < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, + "unable to initialize mirror VFD"); + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__init_package() */ + + +/* ------------------------------------------------------------------------- + * Function: H5FD_mirror_init + * + * Purpose: Initialize this driver by registering the driver with the + * library. + * + * Return: Success: The driver ID for the mirror driver. + * Failure: Negative + * ------------------------------------------------------------------------- + */ +hid_t +H5FD_mirror_init(void) +{ + hid_t ret_value = H5I_INVALID_HID; + + FUNC_ENTER_NOAPI(FAIL) + + LOG_OP_CALL("H5FD_mirror_init"); + + if (H5I_VFL != H5I_get_type(H5FD_MIRROR_g)) { + H5FD_MIRROR_g = H5FD_register(&H5FD_mirror_g, sizeof(H5FD_class_t), + FALSE); + } + + ret_value = H5FD_MIRROR_g; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_mirror_init() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_term + * + * Purpose: Shut down the VFD + * + * Returns: SUCCEED (Can't fail) + * --------------------------------------------------------------------------- + */ +static herr_t +H5FD_mirror_term(void) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Reset VFL ID */ + H5FD_MIRROR_g = 0; + + LOG_OP_CALL("H5FD_mirror_term"); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD_mirror_term() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_decode_uint16 + * + * Purpose: Extract a 16-bit integer in "network" (Big-Endian) word order + * from the byte-buffer and return it with the local word order at + * the destination pointer. + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * Return: The number of bytes read from the buffer (2). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_decode_uint16(uint16_t *out, const unsigned char *_buf) +{ + uint16_t n = 0; + LOG_OP_CALL("H5FD__mirror_xmit_decode_uint16"); + HDassert(_buf && out); + HDmemcpy(&n, _buf, sizeof(n)); + *out = (uint16_t)HDntohs(n); + return 2; /* number of bytes eaten */ +} /* end H5FD__mirror_xmit_decode_uint16() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_decode_uint32 + * + * Purpose: Extract a 32-bit integer in "network" (Big-Endian) word order + * from the byte-buffer and return it with the local word order at + * the destination pointer. + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * Return: The number of bytes read from the buffer (4). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_decode_uint32(uint32_t *out, const unsigned char *_buf) +{ + uint32_t n = 0; + LOG_OP_CALL("H5FD__mirror_xmit_decode_uint32"); + HDassert(_buf && out); + HDmemcpy(&n, _buf, sizeof(n)); + *out = (uint32_t)HDntohl(n); + return 4; /* number of bytes eaten */ +} /* end H5FD__mirror_xmit_decode_uint32() */ + + + +/* --------------------------------------------------------------------------- + * Function: is_host_little_endian + * + * Purpose: Determine whether the host machine is is little-endian. + * + * Store an intger with a known value, re-map the memory to a + * character array, and inspect the array's contents. + * + * Return: The number of bytes written to the buffer (8). + * + * Programmer: Jacob Smith + * 2020-03-05 + * --------------------------------------------------------------------------- + */ +static hbool_t +is_host_little_endian(void) +{ + union { + uint32_t u32; + uint8_t u8[4]; + } echeck; + echeck.u32 = 0xA1B2C3D4; + if (echeck.u8[0] == 0xD4) { + return TRUE; + } + else { + return FALSE; + } +} /* end is_host_little_endian() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_decode_uint64 + * + * Purpose: Extract a 64-bit integer in "network" (Big-Endian) word order + * from the byte-buffer and return it with the local word order. + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * WARNING: Does not accommodate other forms of endianness, + * e.g. "middle-endian". + * + * Return: The number of bytes written to the buffer (8). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_decode_uint64(uint64_t *out, const unsigned char *_buf) +{ + uint64_t n = 0; + LOG_OP_CALL("H5FD__mirror_xmit_decode_uint64"); + HDassert(_buf && out); + HDmemcpy(&n, _buf, sizeof(n)); + if (TRUE == is_host_little_endian()) { + *out = _BSWAP_64(n); + } + else { + *out = n; + } + return 8; +} /* end H5FD__mirror_xmit_decode_uint64() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_decode_uint8 + * + * Purpose: Extract a 8-bit integer in "network" (Big-Endian) word order + * from the byte-buffer and return it with the local word order at + * the destination pointer. + * (yes, it's one byte). + * + * Return: The number of bytes read from the buffer (1). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_decode_uint8(uint8_t *out, const unsigned char *_buf) +{ + LOG_OP_CALL("H5FD__mirror_xmit_decode_uint8"); + HDassert(_buf && out); + HDmemcpy(out, _buf, sizeof(uint8_t)); + return 1; /* number of bytes eaten */ +} /* end H5FD__mirror_xmit_decode_uint8() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_encode_uint16 + * + * Purpose: Encode a 16-bit integer in "network" (Big-Endian) word order + * in place in the destination bytes-buffer. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer (2). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_encode_uint16(unsigned char *_dest, uint16_t v) +{ + uint16_t n = 0; + LOG_OP_CALL("H5FD__mirror_xmit_encode_uint16"); + HDassert(_dest); + n = (uint16_t)HDhtons(v); + HDmemcpy(_dest, &n, sizeof(n)); + return 2; +} /* end H5FD__mirror_xmit_encode_uint16() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_encode_uint32 + * + * Purpose: Encode a 32-bit integer in "network" (Big-Endian) word order + * in place in the destination bytes-buffer. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer (4). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_encode_uint32(unsigned char *_dest, uint32_t v) +{ + uint32_t n = 0; + LOG_OP_CALL("H5FD__mirror_xmit_encode_uint32"); + HDassert(_dest); + n = (uint32_t)HDhtonl(v); + HDmemcpy(_dest, &n, sizeof(n)); + return 4; +} /* end H5FD__mirror_xmit_encode_uint32() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_encode_uint64 + * + * Purpose: Encode a 64-bit integer in "network" (Big-Endian) word order + * in place in the destination bytes-buffer. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer (8). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_encode_uint64(unsigned char *_dest, uint64_t v) +{ + uint64_t n = v; + LOG_OP_CALL("H5FD__mirror_xmit_decode_uint64"); + HDassert(_dest); + if (TRUE == is_host_little_endian()) { + n = _BSWAP_64(v); + } + HDmemcpy(_dest, &n, sizeof(n)); + return 8; +} /* H5FD__mirror_xmit_encode_uint64() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_encode_uint8 + * + * Purpose: Encode a 8-bit integer in "network" (Big-Endian) word order + * in place in the destination bytes-buffer. + * (yes, it's one byte). + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes read from the buffer (1). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_encode_uint8(unsigned char *dest, uint8_t v) +{ + LOG_OP_CALL("H5FD__mirror_xmit_encode_uint8"); + HDassert(dest); + HDmemcpy(dest, &v, sizeof(v)); + return 1; +} /* end H5FD__mirror_xmit_encode_uint8() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_decode_header + * + * Purpose: Extract a mirror_xmit_t "header" from the bytes-buffer. + * + * Fields will be lifted from the buffer and stored in the + * target structure, using in the correct location (different + * systems may insert different padding between components) and + * word order (Big- vs Little-Endian). + * + * The resulting structure should be sanity-checked with + * H5FD_mirror_xmit_is_xmit() before use. + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * Return: The number of bytes consumed from the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_decode_header(H5FD_mirror_xmit_t *out, + const unsigned char *buf) +{ + size_t n_eaten = 0; + LOG_OP_CALL("H5FD_mirror_xmit_decode_header"); + HDassert(out && buf); + n_eaten += H5FD__mirror_xmit_decode_uint32(&(out->magic), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint8(&(out->version), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint32(&(out->session_token), + &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint32(&(out->xmit_count), + &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint8(&(out->op), &buf[n_eaten]); + HDassert(n_eaten == H5FD_MIRROR_XMIT_HEADER_SIZE); + return n_eaten; +} /* end H5FD_mirror_xmit_decode_header() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_decode_lock + * + * Purpose: Extract a mirror_xmit_lock_t from the bytes-buffer. + * + * Fields will be lifted from the buffer and stored in the + * target structure, using in the correct location (different + * systems may insert different padding between components) and + * word order (Big- vs Little-Endian). + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * The resulting structure should be sanity-checked with + * H5FD_mirror_xmit_is_lock() before use. + * + * Return: The number of bytes consumed from the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_decode_lock(H5FD_mirror_xmit_lock_t *out, + const unsigned char *buf) +{ + size_t n_eaten = 0; + LOG_OP_CALL("H5FD_mirror_xmit_decode_lock"); + HDassert(out && buf); + n_eaten += H5FD_mirror_xmit_decode_header(&(out->pub), buf); + n_eaten += H5FD__mirror_xmit_decode_uint64(&(out->rw), &buf[n_eaten]); + HDassert(n_eaten == H5FD_MIRROR_XMIT_LOCK_SIZE); + return n_eaten; +} /* end H5FD_mirror_xmit_decode_lock() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_decode_open + * + * Purpose: Extract a mirror_xmit_open_t from the bytes-buffer. + * + * Fields will be lifted from the buffer and stored in the + * target structure, using in the correct location (different + * systems may insert different padding between components) and + * word order (Big- vs Little-Endian). + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * The resulting structure should be sanity-checked with + * H5FD_mirror_xmit_is_open() before use. + * + * Return: The maximum number of bytes that this decoding operation might + * have consumed from the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_decode_open(H5FD_mirror_xmit_open_t *out, + const unsigned char *buf) +{ + size_t n_eaten = 0; + LOG_OP_CALL("H5FD_mirror_xmit_decode_open"); + HDassert(out && buf); + n_eaten += H5FD_mirror_xmit_decode_header(&(out->pub), buf); + n_eaten += H5FD__mirror_xmit_decode_uint32(&(out->flags), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint64(&(out->maxaddr), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint64(&(out->size_t_blob), + &buf[n_eaten]); + HDassert((H5FD_MIRROR_XMIT_OPEN_SIZE - H5FD_MIRROR_XMIT_FILEPATH_MAX) + == n_eaten); + HDstrncpy(out->filename, (const char *)&buf[n_eaten], + H5FD_MIRROR_XMIT_FILEPATH_MAX-1); + out->filename[H5FD_MIRROR_XMIT_FILEPATH_MAX-1] = 0; /* force final NULL */ + return H5FD_MIRROR_XMIT_OPEN_SIZE; +} /* end H5FD_mirror_xmit_decode_open() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_decode_reply + * + * Purpose: Extract a mirror_xmit_reply_t from the bytes-buffer. + * + * Fields will be lifted from the buffer and stored in the + * target structure, using in the correct location (different + * systems may insert different padding between components) and + * word order (Big- vs Little-Endian). + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * The resulting structure should be sanity-checked with + * H5FD_mirror_xmit_is_reply() before use. + * + * Return: The maximum number of bytes that this decoding operation might + * have consumed from the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_decode_reply(H5FD_mirror_xmit_reply_t *out, + const unsigned char *buf) +{ + size_t n_eaten = 0; + LOG_OP_CALL("H5FD_mirror_xmit_decode_reply"); + HDassert(out && buf); + n_eaten += H5FD_mirror_xmit_decode_header(&(out->pub), buf); + n_eaten += H5FD__mirror_xmit_decode_uint32(&(out->status), &buf[n_eaten]); + HDassert((H5FD_MIRROR_XMIT_REPLY_SIZE - H5FD_MIRROR_STATUS_MESSAGE_MAX) + == n_eaten); + HDstrncpy(out->message, (const char *)&buf[n_eaten], + H5FD_MIRROR_STATUS_MESSAGE_MAX-1); + out->message[H5FD_MIRROR_STATUS_MESSAGE_MAX-1] = 0; /* force NULL term */ + return H5FD_MIRROR_XMIT_REPLY_SIZE; +} /* end H5FD_mirror_xmit_decode_reply() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_decode_set_eoa + * + * Purpose: Extract a mirror_xmit_eoa_t from the bytes-buffer. + * + * Fields will be lifted from the buffer and stored in the + * target structure, using in the correct location (different + * systems may insert different padding between components) and + * word order (Big- vs Little-Endian). + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * The resulting structure should be sanity-checked with + * H5FD_mirror_xmit_is_set_eoa() before use. + * + * Return: The number of bytes consumed from the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_decode_set_eoa(H5FD_mirror_xmit_eoa_t *out, + const unsigned char *buf) +{ + size_t n_eaten = 0; + LOG_OP_CALL("H5FD_mirror_xmit_decode_set_eoa"); + HDassert(out && buf); + n_eaten += H5FD_mirror_xmit_decode_header(&(out->pub), buf); + n_eaten += H5FD__mirror_xmit_decode_uint8(&(out->type), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint64(&(out->eoa_addr), &buf[n_eaten]); + HDassert(n_eaten == H5FD_MIRROR_XMIT_EOA_SIZE); + return n_eaten; +} /* end H5FD_mirror_xmit_decode_set_eoa() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_decode_write + * + * Purpose: Extract a mirror_xmit_write_t from the bytes-buffer. + * + * Fields will be lifted from the buffer and stored in the + * target structure, using in the correct location (different + * systems may insert different padding between components) and + * word order (Big- vs Little-Endian). + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * The resulting structure should be sanity-checked with + * H5FD_mirror_xmit_is_write() before use. + * + * Return: The number of bytes consumed from the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_decode_write(H5FD_mirror_xmit_write_t *out, + const unsigned char *buf) +{ + size_t n_eaten = 0; + LOG_OP_CALL("H5FD_mirror_xmit_decode_write"); + HDassert(out && buf); + n_eaten += H5FD_mirror_xmit_decode_header(&(out->pub), buf); + n_eaten += H5FD__mirror_xmit_decode_uint8(&(out->type), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint64(&(out->offset), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint64(&(out->size), &buf[n_eaten]); + HDassert(n_eaten == H5FD_MIRROR_XMIT_WRITE_SIZE); + return n_eaten; +} /* end H5FD_mirror_xmit_decode_write() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_encode_header + * + * Purpose: Encode a mirror_xmit_t "header" to the bytes-buffer. + * + * Fields will be packed into the buffer in a predictable manner, + * any numbers stored in "network" (Big-Endian) word order. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_encode_header(unsigned char *dest, + const H5FD_mirror_xmit_t *x) +{ + size_t n_writ = 0; + LOG_OP_CALL("H5FD_mirror_xmit_encode_header"); + HDassert(dest && x); + n_writ += H5FD__mirror_xmit_encode_uint32((dest+n_writ), x->magic); + n_writ += H5FD__mirror_xmit_encode_uint8((dest+n_writ), x->version); + n_writ += H5FD__mirror_xmit_encode_uint32((dest+n_writ), x->session_token); + n_writ += H5FD__mirror_xmit_encode_uint32((dest+n_writ), x->xmit_count); + n_writ += H5FD__mirror_xmit_encode_uint8((dest+n_writ), x->op); + HDassert(n_writ == H5FD_MIRROR_XMIT_HEADER_SIZE); + return n_writ; +} /* end H5FD_mirror_xmit_encode_header() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_encode_lock + * + * Purpose: Encode a mirror_xmit_lock_t to the bytes-buffer. + * Fields will be packed into the buffer in a predictable manner, + * any numbers stored in "network" (Big-Endian) word order. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_encode_lock(unsigned char *dest, + const H5FD_mirror_xmit_lock_t *x) +{ + size_t n_writ = 0; + LOG_OP_CALL("H5FD_mirror_xmit_encode_lock"); + HDassert(dest && x); + n_writ += H5FD_mirror_xmit_encode_header(dest, + (const H5FD_mirror_xmit_t *)&(x->pub)); + n_writ += H5FD__mirror_xmit_encode_uint64(&dest[n_writ], x->rw); + HDassert(n_writ == H5FD_MIRROR_XMIT_LOCK_SIZE); + return n_writ; +} /* end H5FD_mirror_xmit_encode_lock() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_encode_open + * + * Purpose: Encode a mirror_xmit_open_t to the bytes-buffer. + * Fields will be packed into the buffer in a predictable manner, + * any numbers stored in "network" (Big-Endian) word order. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The maximum number of bytes that this decoding operation might + * have written into the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_encode_open(unsigned char *dest, + const H5FD_mirror_xmit_open_t *x) +{ + size_t n_writ = 0; + LOG_OP_CALL("H5FD_mirror_xmit_encode_open"); + HDassert(dest && x); + HDbzero(dest, H5FD_MIRROR_XMIT_OPEN_SIZE); + n_writ += H5FD_mirror_xmit_encode_header(dest, + (const H5FD_mirror_xmit_t *)&(x->pub)); + n_writ += H5FD__mirror_xmit_encode_uint32(&dest[n_writ], x->flags); + n_writ += H5FD__mirror_xmit_encode_uint64(&dest[n_writ], x->maxaddr); + n_writ += H5FD__mirror_xmit_encode_uint64(&dest[n_writ], x->size_t_blob); + HDassert((H5FD_MIRROR_XMIT_OPEN_SIZE - H5FD_MIRROR_XMIT_FILEPATH_MAX) + == n_writ); + HDstrncpy((char *)&dest[n_writ], x->filename, + H5FD_MIRROR_XMIT_FILEPATH_MAX); + return H5FD_MIRROR_XMIT_OPEN_SIZE; +} /* end H5FD_mirror_xmit_encode_open() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_encode_reply + * + * Purpose: Encode a mirror_xmit_reply_t to the bytes-buffer. + * + * Fields will be packed into the buffer in a predictable manner, + * any numbers stored in "network" (Big-Endian) word order. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The maximum number of bytes that this decoding operation might + * have written into the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_encode_reply(unsigned char *dest, + const H5FD_mirror_xmit_reply_t *x) +{ + size_t n_writ = 0; + LOG_OP_CALL("H5FD_mirror_xmit_encode_reply"); + HDassert(dest && x); + HDbzero(dest, H5FD_MIRROR_XMIT_REPLY_SIZE); + n_writ += H5FD_mirror_xmit_encode_header(dest, + (const H5FD_mirror_xmit_t *)&(x->pub)); + n_writ += H5FD__mirror_xmit_encode_uint32(&dest[n_writ], x->status); + HDassert((H5FD_MIRROR_XMIT_REPLY_SIZE - H5FD_MIRROR_STATUS_MESSAGE_MAX) + == n_writ); + HDstrncpy((char *)&dest[n_writ], x->message, + H5FD_MIRROR_STATUS_MESSAGE_MAX); + return H5FD_MIRROR_XMIT_REPLY_SIZE; +} /* end H5FD_mirror_xmit_encode_reply() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_encode_set_eoa + * + * Purpose: Encode a mirror_xmit_eoa_t to the bytes-buffer. + * + * Fields will be packed into the buffer in a predictable manner, + * any numbers stored in "network" (Big-Endian) word order. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_encode_set_eoa(unsigned char *dest, + const H5FD_mirror_xmit_eoa_t *x) +{ + size_t n_writ = 0; + LOG_OP_CALL("H5FD_mirror_xmit_encode_set_eoa"); + HDassert(dest && x); + n_writ += H5FD_mirror_xmit_encode_header(dest, + (const H5FD_mirror_xmit_t *)&(x->pub)); + n_writ += H5FD__mirror_xmit_encode_uint8(&dest[n_writ], x->type); + n_writ += H5FD__mirror_xmit_encode_uint64(&dest[n_writ], x->eoa_addr); + HDassert(n_writ == H5FD_MIRROR_XMIT_EOA_SIZE); + return n_writ; +} /* end H5FD_mirror_xmit_encode_set_eoa() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_encode_write + * + * Purpose: Encode a mirror_xmit_write_t to the bytes-buffer. + * + * Fields will be packed into the buffer in a predictable manner, + * any numbers stored in "network" (Big-Endian) word order. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_encode_write(unsigned char *dest, + const H5FD_mirror_xmit_write_t *x) +{ + size_t n_writ = 0; + LOG_OP_CALL("H5FD_mirror_xmit_encode_write"); + HDassert(dest && x); + n_writ += H5FD_mirror_xmit_encode_header(dest, + (const H5FD_mirror_xmit_t *)&(x->pub)); + n_writ += H5FD__mirror_xmit_encode_uint8(&dest[n_writ], x->type); + n_writ += H5FD__mirror_xmit_encode_uint64(&dest[n_writ], x->offset); + n_writ += H5FD__mirror_xmit_encode_uint64(&dest[n_writ], x->size); + HDassert(n_writ == H5FD_MIRROR_XMIT_WRITE_SIZE); + return n_writ; +} /* end H5FD_mirror_xmit_encode_write() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_close + * + * Purpose: Verify that a mirror_xmit_t is a valid CLOSE xmit. + * + * Checks header validity and op code. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +hbool_t +H5FD_mirror_xmit_is_close(const H5FD_mirror_xmit_t *xmit) +{ + LOG_OP_CALL("H5FD_mirror_xmit_is_close"); + HDassert(xmit); + if ( (TRUE == H5FD_mirror_xmit_is_xmit(xmit)) && + (H5FD_MIRROR_OP_CLOSE == xmit->op) ) + { + return TRUE; + } + return FALSE; +} /* end H5FD_mirror_xmit_is_close() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_lock + * + * Purpose: Verify that a mirror_xmit_lock_t is a valid LOCK xmit. + * + * Checks header validity and op code. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +hbool_t +H5FD_mirror_xmit_is_lock(const H5FD_mirror_xmit_lock_t *xmit) +{ + LOG_OP_CALL("H5FD_mirror_xmit_is_lock"); + HDassert(xmit); + if ( (TRUE == H5FD_mirror_xmit_is_xmit(&(xmit->pub))) && + (H5FD_MIRROR_OP_LOCK == xmit->pub.op) ) + { + return TRUE; + } + return FALSE; +} /* end H5FD_mirror_xmit_is_lock() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_open + * + * Purpose: Verify that a mirror_xmit_open_t is a valid OPEN xmit. + * + * Checks header validity and op code. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +hbool_t +H5FD_mirror_xmit_is_open(const H5FD_mirror_xmit_open_t *xmit) +{ + LOG_OP_CALL("H5FD_mirror_xmit_is_open"); + HDassert(xmit); + if ( (TRUE == H5FD_mirror_xmit_is_xmit(&(xmit->pub))) && + (H5FD_MIRROR_OP_OPEN == xmit->pub.op) ) + { + return TRUE; + } + return FALSE; +} /* end H5FD_mirror_xmit_is_open() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_eoa + * + * Purpose: Verify that a mirror_xmit_eoa_t is a valid SET-EOA xmit. + * + * Checks header validity and op code. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +hbool_t +H5FD_mirror_xmit_is_set_eoa(const H5FD_mirror_xmit_eoa_t *xmit) +{ + LOG_OP_CALL("H5FD_mirror_xmit_is_set_eoa"); + HDassert(xmit); + if ( (TRUE == H5FD_mirror_xmit_is_xmit(&(xmit->pub))) && + (H5FD_MIRROR_OP_SET_EOA == xmit->pub.op) ) + { + return TRUE; + } + return FALSE; +} /* end H5FD_mirror_xmit_is_eoa() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_reply + * + * Purpose: Verify that a mirror_xmit_reply_t is a valid REPLY xmit. + * + * Checks header validity and op code. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +hbool_t +H5FD_mirror_xmit_is_reply(const H5FD_mirror_xmit_reply_t *xmit) +{ + LOG_OP_CALL("H5FD_mirror_xmit_is_reply"); + HDassert(xmit); + if ( (TRUE == H5FD_mirror_xmit_is_xmit(&(xmit->pub))) && + (H5FD_MIRROR_OP_REPLY == xmit->pub.op) ) + { + return TRUE; + } + return FALSE; +} /* end H5FD_mirror_xmit_is_reply() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_write + * + * Purpose: Verify that a mirror_xmit_write_t is a valid WRITE xmit. + * + * Checks header validity and op code. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +hbool_t +H5FD_mirror_xmit_is_write(const H5FD_mirror_xmit_write_t *xmit) +{ + LOG_OP_CALL("H5FD_mirror_xmit_is_write"); + HDassert(xmit); + if ( (TRUE == H5FD_mirror_xmit_is_xmit(&(xmit->pub))) && + (H5FD_MIRROR_OP_WRITE == xmit->pub.op) ) + { + return TRUE; + } + return FALSE; +} /* end H5FD_mirror_xmit_is_write() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_xmit + * + * Purpose: Verify that a mirror_xmit_t is well-formed. + * + * Checks magic number and structure version. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +hbool_t +H5FD_mirror_xmit_is_xmit(const H5FD_mirror_xmit_t *xmit) +{ + LOG_OP_CALL("H5FD_mirror_xmit_is_xmit"); + HDassert(xmit); + if ( (H5FD_MIRROR_XMIT_MAGIC != xmit->magic) || + (H5FD_MIRROR_XMIT_CURR_VERSION != xmit->version) ) + { + return FALSE; + } + return TRUE; +} /* end H5FD_mirror_xmit_is_xmit() */ + + +/* ---------------------------------------------------------------------------- + * Function: H5FD__mirror_verify_reply + * + * Purpose: Wait for and read reply data from remote processes. + * Sanity-check that a reply is well-formed and valid. + * If all checks pass, inspect the reply contents and handle + * reported error, if not an OK reply. + * + * Return: SUCCEED if ok, else FAIL. + * ---------------------------------------------------------------------------- + */ +herr_t +H5FD__mirror_verify_reply(H5FD_mirror_t *file) +{ + char xmit_buf[H5FD_MIRROR_XMIT_REPLY_SIZE]; + struct H5FD_mirror_xmit_reply_t reply; + ssize_t read_ret = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT; + + LOG_OP_CALL("H5FD__mirror_verify_reply"); + + HDassert(file && file->sock_fd); + + read_ret = HDread(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_REPLY_SIZE); + if (read_ret < 0) { + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to read reply"); + } + if (read_ret != H5FD_MIRROR_XMIT_REPLY_SIZE) { + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unexpected read size"); + } + + LOG_XMIT_BYTES("reply", xmit_buf, read_ret); + + if (H5FD_mirror_xmit_decode_reply(&reply, (const unsigned char *)xmit_buf) + != H5FD_MIRROR_XMIT_REPLY_SIZE) + { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to decode reply xmit"); + } + + if (H5FD_mirror_xmit_is_reply(&reply) != TRUE) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "xmit op code was not REPLY"); + } + + if (reply.pub.session_token != file->xmit.session_token) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "wrong session"); + } + + if (reply.pub.xmit_count != (file->xmit_i)++) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "xmit out of sync"); + } + + if (reply.status != H5FD_MIRROR_STATUS_OK) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, + "%s", (const char *)(reply.message)); + } + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__mirror_verify_reply() */ + + +/* ------------------------------------------------------------------------- + * Function: H5FD_mirror_fapl_get + * + * Purpose: Get the file access propety list which could be used to create + * an identical file. + * + * Return: Success: pointer to the new file access property list value. + * Failure: NULL + * ------------------------------------------------------------------------- + */ +static void * +H5FD_mirror_fapl_get(H5FD_t *_file) +{ + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + H5FD_mirror_fapl_t *fa = NULL; + void *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + LOG_OP_CALL("H5FD_mirror_fapl_get"); + + fa = (H5FD_mirror_fapl_t *)H5MM_calloc(sizeof(H5FD_mirror_fapl_t)); + if (NULL == fa) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "calloc failed"); + } + + HDmemcpy(fa, &(file->fa), sizeof(H5FD_mirror_fapl_t)); + + ret_value = fa; + +done: + if (ret_value == NULL) { + if (fa != NULL) { + H5MM_xfree(fa); + } + } + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_mirror_fapl_get() */ + + +/* ------------------------------------------------------------------------- + * Function: H5FD_mirror_fapl_copy + * + * Purpose: Copies the mirror vfd-specific file access properties. + * + * Return: Success: Pointer to a new property list + * Failure: NULL + * ------------------------------------------------------------------------- + */ +static void * +H5FD_mirror_fapl_copy(const void *_old_fa) +{ + const H5FD_mirror_fapl_t *old_fa = (const H5FD_mirror_fapl_t *)_old_fa; + H5FD_mirror_fapl_t *new_fa = NULL; + void *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + LOG_OP_CALL("H5FD_mirror_fapl_copy"); + + new_fa = (H5FD_mirror_fapl_t *)H5MM_malloc(sizeof(H5FD_mirror_fapl_t)); + if (new_fa == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + "memory allocation failed"); + } + + HDmemcpy(new_fa, old_fa, sizeof(H5FD_mirror_fapl_t)); + ret_value = new_fa; + +done: + if (ret_value == NULL) { + if (new_fa != NULL) { + H5MM_xfree(new_fa); + } + } + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_mirror_fapl_copy() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_mirror_fapl_free + * + * Purpose: Frees the mirror VFD-specific file access properties. + * + * Return: SUCCEED (cannot fail) + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_mirror_fapl_free(void *_fa) +{ + H5FD_mirror_fapl_t *fa = (H5FD_mirror_fapl_t*)_fa; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + LOG_OP_CALL("H5FD_mirror_fapl_free"); + + /* sanity check */ + HDassert(fa != NULL); + HDassert(fa->magic == H5FD_MIRROR_FAPL_MAGIC); + + fa->magic += 1; /* invalidate */ + H5MM_xfree(fa); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD_mirror_fapl_free() */ + + +/* ------------------------------------------------------------------------- + * Function: H5Pget_fapl_mirror + * + * Purpose: Get the configuration information for this fapl. + * Data is memcopied into the fa_out pointer. + * + * Return: SUCCEED/FAIL + * ------------------------------------------------------------------------- + */ +herr_t +H5Pget_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa_out) +{ + const H5FD_mirror_fapl_t *fa = NULL; + H5P_genplist_t *plist = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*x", fapl_id, fa_out); + + LOG_OP_CALL("H5Pget_fapl_mirror"); + + if (NULL == fa_out) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fa_out is NULL"); + } + + plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); + if (NULL == plist) { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, + "not a file access property list"); + } + + if (H5P_peek_driver(plist) != H5FD_MIRROR) { + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver"); + } + + fa = (const H5FD_mirror_fapl_t *)H5P_peek_driver_info(plist); + if (NULL == fa) { + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info"); + } + + HDassert(fa->magic == H5FD_MIRROR_FAPL_MAGIC); /* sanity check */ + + HDmemcpy(fa_out, fa, sizeof(H5FD_mirror_fapl_t)); + +done: + FUNC_LEAVE_API(ret_value); +} /* end H5Pget_fapl_mirror() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_fapl_mirror + * + * Purpose: Modify the file access property list to use the mirror + * driver (H5FD_MIRROR) defined in this source file. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa) +{ + H5P_genplist_t *plist = NULL; + herr_t ret_value = FAIL; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*x", fapl_id, fa); + + LOG_OP_CALL("H5Pset_fapl_mirror"); + + plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); + if (NULL == plist) { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, + "not a file access property list"); + } + if (NULL == fa) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null fapl_t pointer"); + } + if (H5FD_MIRROR_FAPL_MAGIC != fa->magic) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid fapl_t magic"); + } + if (H5FD_MIRROR_CURR_FAPL_T_VERSION != fa->version) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unknown fapl_t version"); + } + + ret_value = H5P_set_driver(plist, H5FD_MIRROR, (const void *)fa); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_fapl_mirror() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_mirror_open + * + * Purpose: Create and/or opens a file as an HDF5 file. + * + * Initiate connection with remote Server/Writer. + * If successful, the remote file is open. + * + * Return: Success: A pointer to a new file data structure. The + * public fields will be initialized by the + * caller, which is always H5FD_open(). + * Failure: NULL + *------------------------------------------------------------------------- + */ +static H5FD_t * +H5FD_mirror_open(const char *name, + unsigned flags, + hid_t fapl_id, + haddr_t maxaddr) +{ +#define MIRR_OPEN_MAXBUF 16 /* local symbol to give meaning to magic number */ + /* #defined because it is needed at compile time */ + /* Large enough to hold a port number string */ + int live_socket = -1; + struct sockaddr_in target_addr; + socklen_t addr_size; + char xmit_buf[H5FD_MIRROR_XMIT_OPEN_SIZE]; + H5FD_mirror_fapl_t fa; + H5FD_mirror_t *file = NULL; + H5FD_mirror_xmit_open_t open_xmit; + H5FD_t *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + LOG_OP_CALL("H5FD_mirror_open"); + + /* --------------- */ + /* Check arguments */ + /* --------------- */ + + if (!name || !*name) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name"); + } + if (HDstrlen(name) >= H5FD_MIRROR_XMIT_FILEPATH_MAX) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "filename is too long"); + } + if (0 == maxaddr || HADDR_UNDEF == maxaddr) { + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr"); + } + if (ADDR_OVERFLOW(maxaddr)) { + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr"); + } + + if (H5Pget_fapl_mirror(fapl_id, &fa) == FAIL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "can't get config info"); + } + + if (H5FD_MIRROR_FAPL_MAGIC != fa.magic) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid fapl magic"); + } + + if (H5FD_MIRROR_CURR_FAPL_T_VERSION != fa.version) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid fapl version"); + } + + /* --------------------- */ + /* Handshake with remote */ + /* --------------------- */ + + live_socket = HDsocket(AF_INET, SOCK_STREAM, 0); + if (live_socket < 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "can't create socket"); + } + + target_addr.sin_family = AF_INET; + target_addr.sin_port = HDhtons((uint16_t)fa.handshake_port); + target_addr.sin_addr.s_addr = HDinet_addr(fa.remote_ip); + HDmemset(target_addr.sin_zero, '\0', sizeof target_addr.sin_zero); + + addr_size = sizeof(target_addr); + if (HDconnect(live_socket, (struct sockaddr *)&target_addr, addr_size) < 0) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "can't connect to remote server"); + } + + /* ------------- */ + /* Open the file */ + /* ------------- */ + + file = (H5FD_mirror_t *)H5FL_CALLOC(H5FD_mirror_t); + if (NULL == file) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + "unable to allocate file struct"); + } + + file->sock_fd = live_socket; + file->xmit_i = 0; + + file->xmit.magic = H5FD_MIRROR_XMIT_MAGIC; + file->xmit.version = H5FD_MIRROR_XMIT_CURR_VERSION; + file->xmit.xmit_count = file->xmit_i++; + file->xmit.session_token = (uint32_t)(0x01020304 ^ file->sock_fd); /* TODO: hashing? */ + /* int --> uint32_t may truncate on some systems... shouldn't matter? */ + + file->xmit.op = H5FD_MIRROR_OP_OPEN; + open_xmit.pub = file->xmit; + open_xmit.flags = (uint32_t)flags; + open_xmit.maxaddr = (uint64_t)maxaddr; + open_xmit.size_t_blob = (uint64_t)((size_t)(-1)); + HDsnprintf(open_xmit.filename, H5FD_MIRROR_XMIT_FILEPATH_MAX-1, "%s", name); + + if (H5FD_mirror_xmit_encode_open((unsigned char *)xmit_buf, + (const H5FD_mirror_xmit_open_t *)&open_xmit) + != H5FD_MIRROR_XMIT_OPEN_SIZE) + { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, NULL, "unable to encode open"); + } + + LOG_XMIT_BYTES("open", xmit_buf, H5FD_MIRROR_XMIT_OPEN_SIZE); + + if (HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_OPEN_SIZE) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, NULL, "unable to transmit open"); + } + + if (H5FD__mirror_verify_reply(file) == FAIL) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "invalid reply"); + } + + ret_value = (H5FD_t *)file; + +done: + if (NULL == ret_value) { + if (file) { + file = H5FL_FREE(H5FD_mirror_t, file); + } + if (live_socket >= 0 && HDclose(live_socket) < 0) { + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, + "can't close socket"); + } + } + FUNC_LEAVE_NOAPI(ret_value) +#undef MIRR_OPEN_MAXBUF +} /* end H5FD_mirror_open() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_mirror_close + * + * Purpose: Closes the HDF5 file. + * + * Tries to send a CLOSE op to the remote Writer and expects + * a valid reply, then closes the socket. + * In error, attempts to send a deliberately invalid xmit to the + * Writer to get it to close/abort, then attempts to close the + * socket. + * + * Return: Success: SUCCEED + * Failure: FAIL, file possibly not closed but resources freed. + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_mirror_close(H5FD_t *_file) +{ + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + unsigned char xmit_buf[H5FD_MIRROR_XMIT_HEADER_SIZE]; + int xmit_encoded = 0; /* monitor point of failure */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + LOG_OP_CALL("H5FD_mirror_close"); + + /* Sanity check */ + HDassert(file); + HDassert(file->sock_fd >= 0); + + file->xmit.xmit_count = (file->xmit_i)++; + file->xmit.op = H5FD_MIRROR_OP_CLOSE; + + if (H5FD_mirror_xmit_encode_header(xmit_buf, + (const H5FD_mirror_xmit_t *)&(file->xmit)) + != H5FD_MIRROR_XMIT_HEADER_SIZE) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to encode close"); + } + xmit_encoded = 1; + + LOG_XMIT_BYTES("close", xmit_buf, H5FD_MIRROR_XMIT_HEADER_SIZE); + + if (HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_HEADER_SIZE) < 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to transmit close"); + } + + if (H5FD__mirror_verify_reply(file) == FAIL) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + } + + if (HDclose(file->sock_fd) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "can't close socket"); + } + +done: + if (ret_value == FAIL) { + if (xmit_encoded == 0) { + /* Encode failed; send GOODBYE to force writer halt. + * We can ignore any response from the writer, if we receive + * any reply at all. + */ + if (HDwrite(file->sock_fd, "GOODBYE", HDstrlen("GOODBYE")) < 0) { + HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to transmit close"); + if (HDclose(file->sock_fd) < 0) { + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + "can't close socket"); + } + file->sock_fd = -1; /* invalidate for later */ + } /* end if problem writing goodbye; go down hard */ + else + if (HDshutdown(file->sock_fd, SHUT_WR) < 0) { + HDONE_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, + "can't shutdown socket write: %s", + HDstrerror(errno)); + } /* end else-if problem shutting down socket */ + } /* end if xmit encode failed */ + + if (file->sock_fd >= 0) { + if (HDclose(file->sock_fd) < 0) { + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + "can't close socket"); + } + } /* end if socket not closed by going down hard */ + } /* end if error */ + + file = H5FL_FREE(H5FD_mirror_t, file); /* always release resources */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_mirror_close() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_mirror_query + * + * Purpose: Get the driver feature flags implemented by the driver. + * + * Return: SUCCEED (non-negative) (can't fail) + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_mirror_query(const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR; + + LOG_OP_CALL("H5FD_mirror_query"); + + /* Notice: the Mirror VFD Writer currently uses only the Sec2 driver as + * the underying driver -- as such, the Mirror VFD implementation copies + * the Sec2 feature flags as its own. + * + * File pointer is always NULL/unused -- the H5FD_FEAT_IGNORE_DRVRINFO flag + * is never included. + * -- JOS 2020-01-13 + */ + if (flags) { + *flags = 0 \ + | H5FD_FEAT_AGGREGATE_METADATA \ + | H5FD_FEAT_ACCUMULATE_METADATA \ + | H5FD_FEAT_DATA_SIEVE \ + | H5FD_FEAT_AGGREGATE_SMALLDATA \ + | H5FD_FEAT_POSIX_COMPAT_HANDLE \ + | H5FD_FEAT_SUPPORTS_SWMR_IO \ + | H5FD_FEAT_DEFAULT_VFD_COMPATIBLE; + } + + FUNC_LEAVE_NOAPI(SUCCEED); +} /* end H5FD_mirror_query() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_mirror_get_eoa + * + * Purpose: Gets the end-of-address marker for the file. The EOA marker + * is the first address past the last byte allocated in the + * format address space. + * + * Required to register the driver. + * + * Return: The end-of-address marker. + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD_mirror_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_mirror_t *file = (const H5FD_mirror_t *)_file; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + LOG_OP_CALL("H5FD_mirror_get_eoa"); + + HDassert(file); + + FUNC_LEAVE_NOAPI(file->eoa) +} /* end H5FD_mirror_get_eoa() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_mirror_set_eoa + * + * Purpose: Set the end-of-address marker for the file. This function is + * called shortly after an existing HDF5 file is opened in order + * to tell the driver where the end of the HDF5 data is located. + * + * Return: SUCCEED / FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_mirror_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr) +{ + H5FD_mirror_xmit_eoa_t xmit_eoa; + unsigned char xmit_buf[H5FD_MIRROR_XMIT_EOA_SIZE]; + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + LOG_OP_CALL("H5FD_mirror_set_eoa"); + + HDassert(file); + + file->eoa = addr; /* local copy */ + + file->xmit.xmit_count = (file->xmit_i)++; + file->xmit.op = H5FD_MIRROR_OP_SET_EOA; + + xmit_eoa.pub = file->xmit; + xmit_eoa.type = (uint8_t)type; + xmit_eoa.eoa_addr = (uint64_t)addr; + + if (H5FD_mirror_xmit_encode_set_eoa(xmit_buf, + (const H5FD_mirror_xmit_eoa_t *)&xmit_eoa) + != H5FD_MIRROR_XMIT_EOA_SIZE) + { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to encode set-eoa"); + } + + LOG_XMIT_BYTES("set-eoa", xmit_buf, H5FD_MIRROR_XMIT_EOA_SIZE); + + if (HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_EOA_SIZE) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, + "unable to transmit set-eoa"); + } + + if (H5FD__mirror_verify_reply(file) == FAIL) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_mirror_set_eoa() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_mirror_get_eof + * + * Purpose: Returns the end-of-file marker, which is the greater of + * either the filesystem end-of-file or the HDF5 end-of-address + * markers. + * + * Required to register the driver. + * + * Return: End of file address, the first address past the end of the + * "file", either the filesystem file or the HDF5 file. + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD_mirror_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_mirror_t *file = (const H5FD_mirror_t *)_file; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + LOG_OP_CALL("H5FD_mirror_get_eof"); + + HDassert(file); + + FUNC_LEAVE_NOAPI(file->eof) +} /* end H5FD_mirror_get_eof() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_mirror_read + * + * Purpose: Required to register the driver. + * If called, MUST fail. + * + * Return: FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_mirror_read(H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED fapl_id, + haddr_t H5_ATTR_UNUSED addr, + size_t H5_ATTR_UNUSED size, + void H5_ATTR_UNUSED *buf) +{ + herr_t ret_value = FAIL; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + LOG_OP_CALL("H5FD_mirror_read"); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_mirror_read() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_mirror_write + * + * Purpose: Writes SIZE bytes of data to FILE beginning at address ADDR + * from buffer BUF according to data transfer properties in + * DXPL_ID. + * + * Send metadata regarding the write (location, size) to the + * remote Writer, then separately transmits the data. + * Both transmission expect an OK reply from the Writer. + * This two-exchange approach incurs significant overhead, + * but is a simple and modular approach. + * Start optimizations here. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_mirror_write(H5FD_t *_file, + H5FD_mem_t type, + hid_t H5_ATTR_UNUSED dxpl_id, + haddr_t addr, + size_t size, + const void *buf) +{ + H5FD_mirror_xmit_write_t xmit_write; + unsigned char xmit_buf[H5FD_MIRROR_XMIT_WRITE_SIZE]; + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + LOG_OP_CALL("H5FD_mirror_write"); + + HDassert(file); + HDassert(buf); + + file->xmit.xmit_count = (file->xmit_i)++; + file->xmit.op = H5FD_MIRROR_OP_WRITE; + + xmit_write.pub = file->xmit; + xmit_write.size = (uint64_t)size; + xmit_write.offset = (uint64_t)addr; + xmit_write.type = (uint8_t)type; + + + /* Notify Writer of incoming data to write. */ + if (H5FD_mirror_xmit_encode_write(xmit_buf, + (const H5FD_mirror_xmit_write_t *)&xmit_write) + != H5FD_MIRROR_XMIT_WRITE_SIZE) + { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to encode write"); + } + + LOG_XMIT_BYTES("write", xmit_buf, H5FD_MIRROR_XMIT_WRITE_SIZE); + + if (HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_WRITE_SIZE) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to transmit write"); + } + + /* Check that our write xmission was received */ + if (H5FD__mirror_verify_reply(file) == FAIL) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + } + + /* Send the data to be written */ + if (HDwrite(file->sock_fd, buf, size) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to transmit data"); + } + + /* Writer should reply that it got the data and is still okay/ready */ + if (H5FD__mirror_verify_reply(file) == FAIL) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_mirror_write() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_mirror_truncate + * + * Purpose: Makes sure that the true file size is the same (or larger) + * than the end-of-address. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_mirror_truncate(H5FD_t *_file, + hid_t H5_ATTR_UNUSED dxpl_id, + hbool_t H5_ATTR_UNUSED closing) +{ + unsigned char xmit_buf[H5FD_MIRROR_XMIT_HEADER_SIZE]; + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + LOG_OP_CALL("H5FD_mirror_truncate"); + + file->xmit.xmit_count = (file->xmit_i)++; + file->xmit.op = H5FD_MIRROR_OP_TRUNCATE; + + if (H5FD_mirror_xmit_encode_header(xmit_buf, &(file->xmit)) + != H5FD_MIRROR_XMIT_HEADER_SIZE) + { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to encode truncate"); + } + + LOG_XMIT_BYTES("truncate", xmit_buf, H5FD_MIRROR_XMIT_HEADER_SIZE); + + if (HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_HEADER_SIZE) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, + "unable to transmit truncate"); + } + + if (H5FD__mirror_verify_reply(file) == FAIL) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_mirror_truncate() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_mirror_lock + * + * Purpose: To place an advisory lock on a file. + * The lock type to apply depends on the parameter "rw": + * TRUE--opens for write: an exclusive lock + * FALSE--opens for read: a shared lock + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_mirror_lock(H5FD_t *_file, hbool_t rw) +{ + H5FD_mirror_xmit_lock_t xmit_lock; + unsigned char xmit_buf[H5FD_MIRROR_XMIT_LOCK_SIZE]; + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT; + + LOG_OP_CALL("H5FD_mirror_lock"); + + file->xmit.xmit_count = (file->xmit_i)++; + file->xmit.op = H5FD_MIRROR_OP_LOCK; + + xmit_lock.pub = file->xmit; + xmit_lock.rw = (uint64_t)rw; + + if (H5FD_mirror_xmit_encode_lock(xmit_buf, + (const H5FD_mirror_xmit_lock_t *)&xmit_lock) + != H5FD_MIRROR_XMIT_LOCK_SIZE) + { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to encode lock"); + } + + LOG_XMIT_BYTES("lock", xmit_buf, H5FD_MIRROR_XMIT_LOCK_SIZE); + + if (HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_LOCK_SIZE) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to transmit lock"); + } + + if (H5FD__mirror_verify_reply(file) == FAIL) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + } + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_mirror_lock */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_mirror_unlock + * + * Purpose: Remove the existing lock on the file. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_mirror_unlock(H5FD_t *_file) +{ + unsigned char xmit_buf[H5FD_MIRROR_XMIT_HEADER_SIZE]; + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT; + + LOG_OP_CALL("H5FD_mirror_unlock"); + + file->xmit.xmit_count = (file->xmit_i)++; + file->xmit.op = H5FD_MIRROR_OP_UNLOCK; + + if (H5FD_mirror_xmit_encode_header(xmit_buf, &(file->xmit)) + != H5FD_MIRROR_XMIT_HEADER_SIZE) + { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to encode unlock"); + } + + LOG_XMIT_BYTES("unlock", xmit_buf, H5FD_MIRROR_XMIT_HEADER_SIZE); + + if (HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_HEADER_SIZE) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to transmit unlock"); + } + + if (H5FD__mirror_verify_reply(file) == FAIL) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + } + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_mirror_unlock */ + +#endif /* H5_HAVE_MIRROR_VFD */ + diff --git a/src/H5FDmirror.h b/src/H5FDmirror.h new file mode 100644 index 0000000..fb66b7b --- /dev/null +++ b/src/H5FDmirror.h @@ -0,0 +1,371 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Public, shared definitions for Mirror VFD & remote Writer. + */ + +#ifndef H5FDmirror_H +#define H5FDmirror_H + +#ifdef H5_HAVE_MIRROR_VFD + +#define H5FD_MIRROR (H5FD_mirror_init()) + +#ifdef __cplusplus +extern "C" { +#endif + +/* ============================================================================ + * Mirror VFD use and operation. + * ============================================================================ + */ + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_fapl_t + * + * Used to pass configuraiton information to the Mirror VFD. + * Populate components as appropriate and pass structure pointer to + * `H5Pset_fapl_mirror()`. + * + * `magic` (uint32_t) + * Semi-unique number to sanity-check pointers to this structure type. + * MUST equal H5FD_MIRROR_FAPL_MAGIC to be considered valid. + * + * `version` (uint32_t) + * Indicates expected components of the structure. + * + * `handshake_port (int) + * Port number to expect to reach the "Mirror Server" on the remote host. + * + * `remote_ip` (char[]) + * IP address string of "Mirror Server" remote host. + * --------------------------------------------------------------------------- + */ +#define H5FD_MIRROR_FAPL_MAGIC 0xF8DD514C +#define H5FD_MIRROR_CURR_FAPL_T_VERSION 1 +#define H5FD_MIRROR_MAX_IP_LEN 32 +typedef struct H5FD_mirror_fapl_t { + uint32_t magic; + uint32_t version; + int handshake_port; + char remote_ip[H5FD_MIRROR_MAX_IP_LEN + 1]; +} H5FD_mirror_fapl_t; + +H5_DLL hid_t H5FD_mirror_init(void); +H5_DLL herr_t H5Pget_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa_out); +H5_DLL herr_t H5Pset_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa); + +/* = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = + * IPC - Mirror VFD and Remote Worker application. + * = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = + */ + + +/* The maximum allowed size for a receiving buffer when accepting bytes to + * write. Writes larger than this size are performed by multiple accept-write + * steps by the Writer. */ +#define H5FD_MIRROR_DATA_BUFFER_MAX H5_GB /* 1 Gigabyte */ + +#define H5FD_MIRROR_XMIT_CURR_VERSION 1 +#define H5FD_MIRROR_XMIT_MAGIC 0x87F8005B + +#define H5FD_MIRROR_OP_OPEN 1 +#define H5FD_MIRROR_OP_CLOSE 2 +#define H5FD_MIRROR_OP_WRITE 3 +#define H5FD_MIRROR_OP_TRUNCATE 4 +#define H5FD_MIRROR_OP_REPLY 5 +#define H5FD_MIRROR_OP_SET_EOA 6 +#define H5FD_MIRROR_OP_LOCK 7 +#define H5FD_MIRROR_OP_UNLOCK 8 + +#define H5FD_MIRROR_STATUS_OK 0 +#define H5FD_MIRROR_STATUS_ERROR 1 +#define H5FD_MIRROR_STATUS_MESSAGE_MAX 256 /* Dedicated error message size */ + +/* Maximum length of a path/filename string, including the NULL-terminator. + * Must not be smaller than H5FD_SPLITTER_PATH_MAX. */ +#define H5FD_MIRROR_XMIT_FILEPATH_MAX 4097 + +/* Define the exact sizes of the various xmit blobs as sent over the wire. + * This is used to minimize the number of bytes transmitted as well as to + * sanity-check received bytes. + * Any modifications to the xmit structures and/or the encode/decode functions + * must be reflected here. + * */ +#define H5FD_MIRROR_XMIT_HEADER_SIZE 14 +#define H5FD_MIRROR_XMIT_EOA_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 9) +#define H5FD_MIRROR_XMIT_LOCK_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 8) +#define H5FD_MIRROR_XMIT_OPEN_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 20 + H5FD_MIRROR_XMIT_FILEPATH_MAX) +#define H5FD_MIRROR_XMIT_REPLY_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 4 + H5FD_MIRROR_STATUS_MESSAGE_MAX) +#define H5FD_MIRROR_XMIT_WRITE_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 17) + +/* Maximum length of any xmit. */ +#define H5FD_MIRROR_XMIT_BUFFER_MAX MAX2( MAX3(H5FD_MIRROR_XMIT_HEADER_SIZE, \ + H5FD_MIRROR_XMIT_EOA_SIZE, \ + H5FD_MIRROR_XMIT_LOCK_SIZE), \ + MAX3(H5FD_MIRROR_XMIT_OPEN_SIZE, \ + H5FD_MIRROR_XMIT_REPLY_SIZE, \ + H5FD_MIRROR_XMIT_WRITE_SIZE) ) \ + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_xmit_t + * + * Common structure 'header' for all mirror VFD/worker IPC. + * Must be the first component of a derived operation xmit structure, + * such as file-open or write command. + * + * `magic` (uint32_t) + * A "unique" number identifying the structure and endianness of + * transmitting maching. + * Must be set to H5FD_MIRROR_XMIT_MAGIC native to the VFD "sender". + * + * `version` (uint8_t) + * Number used to identify the structure membership. + * Allows sane modifications to this structure in the future. + * Must be set to H5FD_MIRROR_XMIT_CURR_VERSION. + * + * `session_token` (uint32_t) + * A "unique" number identifying the session between VFD sender and + * remote receiver/worker/writer. Exists to help sanity-check. + * + * `xmit_count` (uint32_t) + * Which transmission this is since the session began. + * Used to sanity-check transmission errors. + * First xmit (file-open) must be 0. + * + * `op` (uint8_t) + * Number identifying which operation to perform. + * Corresponds with the extended structure outside of this xmit header. + * Possible values are all defined H5FD_MIRROR_OP_* constants. + * + * --------------------------------------------------------------------------- + */ +typedef struct H5FD_mirror_xmit_t { + uint32_t magic; + uint8_t version; + uint32_t session_token; + uint32_t xmit_count; + uint8_t op; +} H5FD_mirror_xmit_t; + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_xmit_eoa_t + * + * Structure containing eoa-set information from VFD sender. + * + * `pub` (H5FD_mirror_xmit_t) + * Common transmission header, containing session information. + * Must be first. + * + * `type` (uint8_t) + * System-independent alias for H5F[D]_mem_t. + * Specifies datatype to be written. + * + * `eoa_addr` (uint64_t) + * New address for eoa. + * (Natively 'haddr_t', always a 64-bit field) + * + * --------------------------------------------------------------------------- + */ +typedef struct H5FD_mirror_xmit_eoa_t { + H5FD_mirror_xmit_t pub; + uint8_t type; + uint64_t eoa_addr; +} H5FD_mirror_xmit_eoa_t; + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_xmit_lock_t + * + * Structure containing eoa-set information from VFD sender. + * + * `pub` (H5FD_mirror_xmit_t) + * Common transmission header, containing session information. + * Must be first. + * + * `rw` (uint64_t) + * The Read/Write mode flag passed into H5FDlock(). + * (Natively `hbool_t`, an 'int') TODO: native int may be 64-bit? + * + * --------------------------------------------------------------------------- + */ +typedef struct H5FD_mirror_xmit_lock_t { + H5FD_mirror_xmit_t pub; + uint64_t rw; +} H5FD_mirror_xmit_lock_t; + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_xmit_open_t + * + * Structure containing file-open information from the VFD sender. + * + * `pub` (H5FD_mirror_xmit_t) + * Common transmission header, containing session information. + * Must be first. + * + * `flags` (uint32_t) + * VFL-layer file-open flags passed directly to H5FDopen(). + * (Natively 'unsigned [int]') TODO: native int may be 64-bit? + * + * `maxaddr` (uint64_t) + * VFL-layer maximum allowed address space for the file to open passed + * directly to H5FDopen(). + * (Natively 'haddr_t', always a 64-bit field) + * + * `size_t_blob` (uint64_t) + * A number indicating how large a size_t is on the sending system. + * Must be set to (uint64_t)((size_t)(-1)) + * (maximum possible value of size_t, cast to uint64_t). + * The receiving system inspects this value -- if the local (remote) + * size_t is smaller than that of the Sender, issues a warning. + * Not an error, as: + * 1. It is assumed that underlying file systems/drivers have become + * smart enough to handle file sizes that otherwise might be + * constrained. + * 2. The Mirror Writer ingests bytes to write multiple 'slices' if the + * size is greater than H5FD_MIRROR_DATA_BUFFER_MAX, regardless of + * any size_t storage size disparity. + * + * `filename` (char[]) + * String giving the filename and path of file to open. + * + * --------------------------------------------------------------------------- + */ +typedef struct H5FD_mirror_xmit_open_t { + H5FD_mirror_xmit_t pub; + uint32_t flags; + uint64_t maxaddr; + uint64_t size_t_blob; + char filename[H5FD_MIRROR_XMIT_FILEPATH_MAX]; +} H5FD_mirror_xmit_open_t; + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_xmit_reply_t + * + * Structure used by the remote receiver/worker/writer to respond to + * a command from the VFD sender. + * + * `pub` (H5FD_mirror_xmit_t) + * Common transmission header, containing session information. + * Must be first. + * + * `status` (uint32_t) + * Number indicating whether the command was successful or if an + * occured. + * Allowed values are H5FD_MIRROR_STATUS_OK and + * H5FD_MIRROR_STATUS_ERROR. + * + * `message` (char[]) + * Error message. Populated if and only if there was a problem. + * It is possible that a message may reach the end of the alloted + * space without a NULL terminator -- the onus is on the programmer to + * handle this situation. + * + * --------------------------------------------------------------------------- + */ +typedef struct H5FD_mirror_xmit_reply_t { + H5FD_mirror_xmit_t pub; + uint32_t status; + char message[H5FD_MIRROR_STATUS_MESSAGE_MAX]; +} H5FD_mirror_xmit_reply_t; + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_xmit_write_t + * + * Structure containing data-write information from VFD sender. + * + * The data to be written is transmitted in subsequent, packets + * and may be broken up into more than one transmission buffer. + * The VFD sender and remote receiver/worker/writer must coordinate + * the receipt of data. + * + * `pub` (H5FD_mirror_xmit_t) + * Common transmission header, containing session information. + * Must be first. + * + * `type` (uint8_t) + * Specifies datatype to be written. + * (Natively 'H5FD_mem_t', an enumerated type in H5Fpublic.h) + * + * `offset` (uint64_t) + * Start location of write in file. + * (Natively 'haddr_t', always a 64-bit field) + * + * `size` (uint64_t) + * Size of the data to be written, in bytes. + * (Natively 'size_t', accommodate the largest possible as 64-bits) + * + * --------------------------------------------------------------------------- + */ +typedef struct H5FD_mirror_xmit_write_t { + H5FD_mirror_xmit_t pub; + uint8_t type; + uint64_t offset; + uint64_t size; +} H5FD_mirror_xmit_write_t; + + + +/* Encode/decode routines are required to "pack" the xmit data into a known + * byte format for transmission over the wire. + * + * All component numbers must be stored in "network" word order (Big-Endian). + * + * All components must be packed in the order given in the structure definition. + * + * All components must be packed with zero padding between. + */ + +H5_DLL size_t H5FD__mirror_xmit_decode_uint16(uint16_t *out, const unsigned char *buf); +H5_DLL size_t H5FD__mirror_xmit_decode_uint32(uint32_t *out, const unsigned char *buf); +H5_DLL size_t H5FD__mirror_xmit_decode_uint64(uint64_t *out, const unsigned char *buf); +H5_DLL size_t H5FD__mirror_xmit_decode_uint8(uint8_t *out, const unsigned char *buf); +H5_DLL size_t H5FD__mirror_xmit_encode_uint16(unsigned char *dest, uint16_t v); +H5_DLL size_t H5FD__mirror_xmit_encode_uint32(unsigned char *dest, uint32_t v); +H5_DLL size_t H5FD__mirror_xmit_encode_uint64(unsigned char *dest, uint64_t v); +H5_DLL size_t H5FD__mirror_xmit_encode_uint8(unsigned char *dest, uint8_t v); + +H5_DLL size_t H5FD_mirror_xmit_decode_header(H5FD_mirror_xmit_t *out, const unsigned char *buf); +H5_DLL size_t H5FD_mirror_xmit_decode_lock(H5FD_mirror_xmit_lock_t *out, const unsigned char *buf); +H5_DLL size_t H5FD_mirror_xmit_decode_open(H5FD_mirror_xmit_open_t *out, const unsigned char *buf); +H5_DLL size_t H5FD_mirror_xmit_decode_reply(H5FD_mirror_xmit_reply_t *out, const unsigned char *buf); +H5_DLL size_t H5FD_mirror_xmit_decode_set_eoa(H5FD_mirror_xmit_eoa_t *out, const unsigned char *buf); +H5_DLL size_t H5FD_mirror_xmit_decode_write(H5FD_mirror_xmit_write_t *out, const unsigned char *buf); + +H5_DLL size_t H5FD_mirror_xmit_encode_header(unsigned char *dest, const H5FD_mirror_xmit_t *x); +H5_DLL size_t H5FD_mirror_xmit_encode_lock(unsigned char *dest, const H5FD_mirror_xmit_lock_t *x); +H5_DLL size_t H5FD_mirror_xmit_encode_open(unsigned char *dest, const H5FD_mirror_xmit_open_t *x); +H5_DLL size_t H5FD_mirror_xmit_encode_reply(unsigned char *dest, const H5FD_mirror_xmit_reply_t *x); +H5_DLL size_t H5FD_mirror_xmit_encode_set_eoa(unsigned char *dest, const H5FD_mirror_xmit_eoa_t *x); +H5_DLL size_t H5FD_mirror_xmit_encode_write(unsigned char *dest, const H5FD_mirror_xmit_write_t *x); + +H5_DLL hbool_t H5FD_mirror_xmit_is_close(const H5FD_mirror_xmit_t *xmit); +H5_DLL hbool_t H5FD_mirror_xmit_is_lock(const H5FD_mirror_xmit_lock_t *xmit); +H5_DLL hbool_t H5FD_mirror_xmit_is_open(const H5FD_mirror_xmit_open_t *xmit); +H5_DLL hbool_t H5FD_mirror_xmit_is_reply(const H5FD_mirror_xmit_reply_t *xmit); +H5_DLL hbool_t H5FD_mirror_xmit_is_set_eoa(const H5FD_mirror_xmit_eoa_t *xmit); +H5_DLL hbool_t H5FD_mirror_xmit_is_write(const H5FD_mirror_xmit_write_t *xmit); +H5_DLL hbool_t H5FD_mirror_xmit_is_xmit(const H5FD_mirror_xmit_t *xmit); + +#ifdef __cplusplus +} +#endif + +#else /* H5_HAVE_MIRROR_VFD */ + +#define H5FD_MIRROR (H5I_INAVLID_HID) + +#endif /* H5_HAVE_MIRROR_VFD */ + +#endif /* H5FDmirror_H */ + + diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h index 514d1bf..124bac6 100644 --- a/src/H5FDpublic.h +++ b/src/H5FDpublic.h @@ -255,6 +255,8 @@ typedef enum H5F_mem_t H5FD_mem_t; * that creates a file which is compatible with the default VFD. * Generally, this means that the VFD creates a single file that follows * the canonical HDF5 file format. + * Regarding the Splitter VFD specifically, only drivers with this flag + * enabled may be used as the Write-Only (W/O) channel driver. */ #define H5FD_FEAT_DEFAULT_VFD_COMPATIBLE 0x00008000 diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c index a393490..d718a93 100644 --- a/src/H5FDsec2.c +++ b/src/H5FDsec2.c @@ -521,6 +521,12 @@ H5FD_sec2_query(const H5FD_t *_file, unsigned long *flags /* out */) FUNC_ENTER_NOAPI_NOINIT_NOERR /* Set the VFL feature flags that this driver supports */ + /* Notice: the Mirror VFD Writer currently uses only the Sec2 driver as + * the underying driver -- as such, the Mirror VFD implementation copies + * these feature flags as its own. Any modifications made here must be + * reflected in H5FDmirror.c + * -- JOS 2020-01-13 + */ if(flags) { *flags = 0; *flags |= H5FD_FEAT_AGGREGATE_METADATA; /* OK to aggregate metadata allocations */ diff --git a/src/H5FDsplitter.c b/src/H5FDsplitter.c new file mode 100644 index 0000000..13816a5 --- /dev/null +++ b/src/H5FDsplitter.c @@ -0,0 +1,1467 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: The Splitter VFD implements a file driver which relays all the + * VFD calls to an underlying VFD, and send all the write calls to + * another underlying VFD. Maintains two files simultaneously. + */ + +/* This source code file is part of the H5FD driver module */ +#include "H5FDdrvr_module.h" + +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File access */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5FDsplitter.h" /* Splitter file driver */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5FDsec2.h" /* Generic Functions */ +#include "H5FDstdio.h" /* Generic Functions */ +#include "H5Pprivate.h" /* Property lists */ + +/* The driver identification number, initialized at runtime */ +static hid_t H5FD_SPLITTER_g = 0; + +/* Driver-specific file access properties */ +typedef struct H5FD_splitter_fapl_t { + hid_t rw_fapl_id; /* fapl for the R/W channel */ + hid_t wo_fapl_id; /* fapl for the W/O channel */ + char wo_path[H5FD_SPLITTER_PATH_MAX + 1]; /* file name for the W/O channel */ + char log_file_path[H5FD_SPLITTER_PATH_MAX + 1]; /* file to record errors reported by the W/O channel */ + hbool_t ignore_wo_errs; /* TRUE to ignore errors on the W/O channel */ +} H5FD_splitter_fapl_t; + +/* The information of this splitter */ +typedef struct H5FD_splitter_t { + H5FD_t pub; /* public stuff, must be first */ + unsigned version; /* version of the H5FD_splitter_vfd_config_t structure used */ + H5FD_splitter_fapl_t fa; /* driver-specific file access properties */ + H5FD_t *rw_file; /* pointer of R/W channel */ + H5FD_t *wo_file; /* pointer of W/O channel */ + FILE *logfp; /* Log file pointer */ +} H5FD_splitter_t; + +/* + * These macros check for overflow of various quantities. These macros + * assume that HDoff_t is signed and haddr_t and size_t are unsigned. + * + * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t' + * is too large to be represented by the second argument + * of the file seek function. + * + * SIZE_OVERFLOW: Checks whether a buffer size of type `hsize_t' is too + * large to be represented by the `size_t' type. + * + * REGION_OVERFLOW: Checks whether an address and size pair describe data + * which can be addressed entirely by the second + * argument of the file seek function. + */ +#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) +#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR)) +#define SIZE_OVERFLOW(Z) ((Z) & ~(hsize_t)MAXADDR) +#define REGION_OVERFLOW(A,Z) (ADDR_OVERFLOW(A) || SIZE_OVERFLOW(Z) || \ + HADDR_UNDEF==(A)+(Z) || \ + (HDoff_t)((A)+(Z))<(HDoff_t)(A)) + +/* This macro provides a wrapper for shared fail-log-ignore behavior + * for errors arising in the splitter's W/O channel. + * Logs an error entry in a log file, if the file exists. + * If not set to ignore errors, registers an error with the library. + */ +#define H5FD_SPLITTER_WO_ERROR(file, funcname, errmajor, errminor, ret, mesg) \ +{ \ + H5FD__splitter_log_error((file), (funcname), (mesg)); \ + if (FALSE == (file)->fa.ignore_wo_errs) { \ + HGOTO_ERROR((errmajor), (errminor), (ret), (mesg)) \ + } \ +} + +#define H5FD_SPLITTER_DEBUG_OP_CALLS 0 /* debugging print toggle; 0 disables */ + +#if H5FD_SPLITTER_DEBUG_OP_CALLS +#define H5FD_SPLITTER_LOG_CALL(name) do { \ + HDprintf("called %s()\n", (name)); \ + fflush(stdout); \ +} while (0) +#else +#define H5FD_SPLITTER_LOG_CALL(name) /* no-op */ +#endif /* H5FD_SPLITTER_DEBUG_OP_CALLS */ + +/* Private functions */ + +/* Print error messages from W/O channel to log file */ +static herr_t H5FD__splitter_log_error(const H5FD_splitter_t *file, const char *atfunc, const char *msg); + +static int H5FD__copy_plist(hid_t fapl_id, hid_t *id_out_ptr); + +/* Prototypes */ +static herr_t H5FD_splitter_term(void); +static hsize_t H5FD_splitter_sb_size(H5FD_t *_file); +static herr_t H5FD_splitter_sb_encode(H5FD_t *_file, char *name/*out*/, unsigned char *buf/*out*/); +static herr_t H5FD_splitter_sb_decode(H5FD_t *_file, const char *name, const unsigned char *buf); +static void *H5FD_splitter_fapl_get(H5FD_t *_file); +static void *H5FD_splitter_fapl_copy(const void *_old_fa); +static herr_t H5FD_splitter_fapl_free(void *_fapl); +static H5FD_t *H5FD_splitter_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); +static herr_t H5FD_splitter_close(H5FD_t *_file); +static int H5FD_splitter_cmp(const H5FD_t *_f1, const H5FD_t *_f2); +static herr_t H5FD_splitter_query(const H5FD_t *_file, unsigned long *flags /* out */); +static herr_t H5FD_splitter_get_type_map(const H5FD_t *_file, H5FD_mem_t *type_map); +static haddr_t H5FD_splitter_alloc(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size); +static herr_t H5FD_splitter_free(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t size); +static haddr_t H5FD_splitter_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type); +static herr_t H5FD_splitter_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr); +static haddr_t H5FD_splitter_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type); +static herr_t H5FD_splitter_get_handle(H5FD_t *_file, hid_t H5_ATTR_UNUSED fapl, void** file_handle); +static herr_t H5FD_splitter_read(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, void *buf); +static herr_t H5FD_splitter_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, const void *buf); +static herr_t H5FD_splitter_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); +static herr_t H5FD_splitter_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); +static herr_t H5FD_splitter_lock(H5FD_t *_file, hbool_t rw); +static herr_t H5FD_splitter_unlock(H5FD_t *_file); + +static const H5FD_class_t H5FD_splitter_g = { + "splitter", /* name */ + MAXADDR, /* maxaddr */ + H5F_CLOSE_WEAK, /* fc_degree */ + H5FD_splitter_term, /* terminate */ + H5FD_splitter_sb_size, /* sb_size */ + H5FD_splitter_sb_encode, /* sb_encode */ + H5FD_splitter_sb_decode, /* sb_decode */ + sizeof(H5FD_splitter_fapl_t), /* fapl_size */ + H5FD_splitter_fapl_get, /* fapl_get */ + H5FD_splitter_fapl_copy, /* fapl_copy */ + H5FD_splitter_fapl_free, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD_splitter_open, /* open */ + H5FD_splitter_close, /* close */ + H5FD_splitter_cmp, /* cmp */ + H5FD_splitter_query, /* query */ + H5FD_splitter_get_type_map, /* get_type_map */ + H5FD_splitter_alloc, /* alloc */ + H5FD_splitter_free, /* free */ + H5FD_splitter_get_eoa, /* get_eoa */ + H5FD_splitter_set_eoa, /* set_eoa */ + H5FD_splitter_get_eof, /* get_eof */ + H5FD_splitter_get_handle, /* get_handle */ + H5FD_splitter_read, /* read */ + H5FD_splitter_write, /* write */ + H5FD_splitter_flush, /* flush */ + H5FD_splitter_truncate, /* truncate */ + H5FD_splitter_lock, /* lock */ + H5FD_splitter_unlock, /* unlock */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ +}; + +/* Declare a free list to manage the H5FD_splitter_t struct */ +H5FL_DEFINE_STATIC(H5FD_splitter_t); + + +/*------------------------------------------------------------------------- + * Function: H5FD__init_package + * + * Purpose: Initializes any interface-specific data or routines. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__init_package(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL("H5FD__init_package"); + + if (H5FD_splitter_init() < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize splitter VFD") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__init_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_init + * + * Purpose: Initialize the splitter driver by registering it with the + * library. + * + * Return: Success: The driver ID for the splitter driver. + * Failure: Negative + *------------------------------------------------------------------------- + */ +hid_t +H5FD_splitter_init(void) +{ + hid_t ret_value = H5I_INVALID_HID; + + FUNC_ENTER_NOAPI(FAIL) + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_init"); + + if (H5I_VFL != H5I_get_type(H5FD_SPLITTER_g)) { + H5FD_SPLITTER_g = H5FDregister(&H5FD_splitter_g); + } + + ret_value = H5FD_SPLITTER_g; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_init() */ + + +/*--------------------------------------------------------------------------- + * Function: H5FD_splitter_term + * + * Purpose: Shut down the splitter VFD. + * + * Returns: SUCCEED (Can't fail) + *--------------------------------------------------------------------------- + */ +static herr_t +H5FD_splitter_term(void) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_term"); + + /* Reset VFL ID */ + H5FD_SPLITTER_g = 0; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD_splitter_term() */ + + + /*------------------------------------------------------------------------- + * Function: H5FD__copy_plist + * + * Purpose: Sanity-wrapped H5P_copy_plist() for each channel. + * Utility function for operation in multiple locations. + * + * Return: 0 on success, -1 on error. + *------------------------------------------------------------------------- + */ +static int +H5FD__copy_plist(hid_t fapl_id, + hid_t *id_out_ptr) +{ + int ret_value = 0; + H5P_genplist_t *plist_ptr = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD__copy_plist"); + + HDassert(id_out_ptr != NULL); + + if (FALSE == H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, -1, "not a file access property list"); + } + + plist_ptr = (H5P_genplist_t *)H5I_object(fapl_id); + if (NULL == plist_ptr) { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, -1, "unable to get property list"); + } + + *id_out_ptr = H5P_copy_plist(plist_ptr, FALSE); + if (H5I_INVALID_HID == *id_out_ptr) { + HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, -1, "unable to copy file access property list"); + } + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__copy_plist() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_fapl_splitter + * + * Purpose: Sets the file access property list to use the + * splitter driver. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *vfd_config) +{ + H5FD_splitter_fapl_t info; + H5P_genplist_t *plist_ptr = NULL; + herr_t ret_value = SUCCEED; + + H5Eclear2(H5E_DEFAULT); + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*Dr", fapl_id, vfd_config); + + H5FD_SPLITTER_LOG_CALL("H5Pset_fapl_splitter"); + + if (H5FD_SPLITTER_MAGIC != vfd_config->magic) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid configuration (magic number mismatch)") + } + if (H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION != vfd_config->version) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invlaid config (version number mismatch)") + } + + if (NULL == (plist_ptr = (H5P_genplist_t *)H5I_object(fapl_id))) { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid property list") + } + + + /* Make sure that the W/O channel supports write-only capability. + * Some drivers (e.g. family or multi) do revision of the superblock + * in-memory, causing problems in that channel. + * Uses the feature flag H5FD_FEAT_DEFAULT_VFD_COMPATIBLE as the + * determining attribute. + */ + if (H5P_DEFAULT != vfd_config->wo_fapl_id) { + H5FD_class_t *wo_driver = NULL; + H5FD_driver_prop_t wo_driver_prop; + H5P_genplist_t *wo_plist_ptr = NULL; + unsigned long wo_driver_flags = 0; + + wo_plist_ptr = (H5P_genplist_t *)H5I_object(vfd_config->wo_fapl_id); + if (NULL == wo_plist_ptr) { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + } + if (H5P_peek(wo_plist_ptr, H5F_ACS_FILE_DRV_NAME, &wo_driver_prop) < 0) { + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get driver ID & info") + } + wo_driver = (H5FD_class_t *)H5I_object(wo_driver_prop.driver_id); + if (NULL == wo_driver) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid driver ID in file access property list") + } + if (H5FD_driver_query(wo_driver, &wo_driver_flags) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't query VFD flags") + } + if (0 == (H5FD_FEAT_DEFAULT_VFD_COMPATIBLE & wo_driver_flags)) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unsuitable W/O driver") + } + } /* end if W/O VFD is non-default */ + + + info.ignore_wo_errs = vfd_config->ignore_wo_errs; + HDstrncpy(info.wo_path, vfd_config->wo_path, H5FD_SPLITTER_PATH_MAX); + HDstrncpy(info.log_file_path, vfd_config->log_file_path, H5FD_SPLITTER_PATH_MAX); + info.rw_fapl_id = H5P_FILE_ACCESS_DEFAULT; /* pre-set value */ + info.wo_fapl_id = H5P_FILE_ACCESS_DEFAULT; /* pre-set value */ + + /* Set non-default channel FAPL IDs in splitter configuration info */ + if (H5P_DEFAULT != vfd_config->rw_fapl_id) { + if (FALSE == H5P_isa_class(vfd_config->rw_fapl_id, H5P_FILE_ACCESS)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list") + } + info.rw_fapl_id = vfd_config->rw_fapl_id; + } + if (H5P_DEFAULT != vfd_config->wo_fapl_id) { + if (FALSE == H5P_isa_class(vfd_config->wo_fapl_id, H5P_FILE_ACCESS)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list") + } + info.wo_fapl_id = vfd_config->wo_fapl_id; + } + + ret_value = H5P_set_driver(plist_ptr, H5FD_SPLITTER, &info); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_fapl_splitter() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_fapl_splitter + * + * Purpose: Returns information about the splitter file access property + * list through the structure config_out. + * + * Will fail if config_out is received without pre-set valid + * magic and version information. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *config_out) +{ + const H5FD_splitter_fapl_t *fapl_ptr = NULL; + H5P_genplist_t *plist_ptr = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*Dr", fapl_id, config_out); + + H5FD_SPLITTER_LOG_CALL("H5Pget_fapl_splitter"); + + /* Check arguments */ + if (TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + } + if (config_out == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "config_out pointer is null") + } + if (H5FD_SPLITTER_MAGIC != config_out->magic) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "info-out pointer invalid (magic number mismatch)") + } + if (H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION != config_out->version) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "info-out pointer invalid (version unsafe)") + } + + /* Pre-set out FAPL IDs with intent to replace these values */ + config_out->rw_fapl_id = H5I_INVALID_HID; + config_out->wo_fapl_id = H5I_INVALID_HID; + + /* Check and get the splitter fapl */ + if (NULL == (plist_ptr = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + } + if (H5FD_SPLITTER != H5P_peek_driver(plist_ptr)) { + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") + } + if (NULL == (fapl_ptr = (const H5FD_splitter_fapl_t *)H5P_peek_driver_info(plist_ptr))) { + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "unable to get specific-driver info") + } + + HDstrncpy(config_out->wo_path, fapl_ptr->wo_path, H5FD_SPLITTER_PATH_MAX); + HDstrncpy(config_out->log_file_path, fapl_ptr->log_file_path, H5FD_SPLITTER_PATH_MAX); + config_out->ignore_wo_errs = fapl_ptr->ignore_wo_errs; + + /* Copy R/W and W/O FAPLs */ + if (H5FD__copy_plist(fapl_ptr->rw_fapl_id, &(config_out->rw_fapl_id)) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't copy R/W FAPL"); + } + if (H5FD__copy_plist(fapl_ptr->wo_fapl_id, &(config_out->wo_fapl_id)) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't copy W/O FAPL"); + } + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_fapl_splitter() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_flush + * + * Purpose: Flushes all data to disk for both channels. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_splitter_flush(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t closing) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_flush"); + + /* Public API for dxpl "context" */ + if (H5FDflush(file->rw_file, dxpl_id, closing) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTFLUSH, FAIL, "unable to flush R/W file") + } + if (H5FDflush(file->wo_file, dxpl_id, closing) < 0) { + H5FD_SPLITTER_WO_ERROR(file, "H5FD_splitter_flush", + H5E_VFL, H5E_CANTFLUSH, FAIL, + "unable to flush W/O file") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_flush() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_read + * + * Purpose: Reads SIZE bytes of data from the R/W channel, beginning at + * address ADDR into buffer BUF according to data transfer + * properties in DXPL_ID. + * + * Return: Success: SUCCEED + * The read result is written into the BUF buffer + * which should be allocated by the caller. + * Failure: FAIL + * The contents of BUF are undefined. + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_splitter_read( + H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED dxpl_id, + haddr_t addr, + size_t size, + void *buf /*out*/) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_read"); + + HDassert(file && file->pub.cls); + HDassert(buf); + + /* Check for overflow conditions */ + if (!H5F_addr_defined(addr)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "addr undefined, addr = %llu", (unsigned long long)addr) + } + if (REGION_OVERFLOW(addr, size)) { + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, addr = %llu", (unsigned long long)addr) + } + + /* Only read from R/W channel */ + /* Public API for dxpl "context" */ + if (H5FDread(file->rw_file, type, dxpl_id, addr, size, buf) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "Reading from R/W channel failed") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_read() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_write + * + * Purpose: Writes SIZE bytes of data to R/W and W/O channels, beginning + * at address ADDR from buffer BUF according to data transfer + * properties in DXPL_ID. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_splitter_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, const void *buf) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + H5P_genplist_t *plist_ptr = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_write"); + + if (NULL == (plist_ptr = (H5P_genplist_t *)H5I_object(dxpl_id))) { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") + } + + /* Write to each file */ + /* Public API for dxpl "context" */ + if (H5FDwrite(file->rw_file, type, dxpl_id, addr, size, buf) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "R/W file write failed") + } + if (H5FDwrite(file->wo_file, type, dxpl_id, addr, size, buf) < 0) { + H5FD_SPLITTER_WO_ERROR(file, "H5FD_splitter_write", + H5E_VFL, H5E_WRITEERROR, FAIL, + "unable to write W/O file") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_write() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_fapl_get + * + * Purpose: Returns a file access property list which indicates how the + * specified file is being accessed. The return list could be + * used to access another file the same way. + * + * Return: Success: Ptr to new file access property list with all + * members copied from the file struct. + * Failure: NULL + *------------------------------------------------------------------------- + */ +static void * +H5FD_splitter_fapl_get(H5FD_t *_file) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + void *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_fapl_get"); + + ret_value = H5FD_splitter_fapl_copy(&(file->fa)); + + FUNC_LEAVE_NOAPI(ret_value) +} + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_fapl_copy + * + * Purpose: Copies the file access properties. + * + * Return: Success: Pointer to a new property list info structure. + * Failure: NULL + *------------------------------------------------------------------------- + */ +static void * +H5FD_splitter_fapl_copy(const void *_old_fa) +{ + const H5FD_splitter_fapl_t *old_fa_ptr = (const H5FD_splitter_fapl_t *)_old_fa; + H5FD_splitter_fapl_t *new_fa_ptr = NULL; + void *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_fapl_copy"); + + HDassert(old_fa_ptr); + + new_fa_ptr = (H5FD_splitter_fapl_t *)H5MM_calloc(sizeof(H5FD_splitter_fapl_t)); + if (NULL == new_fa_ptr) { + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate log file FAPL") + } + + if (HDmemcpy(new_fa_ptr, old_fa_ptr, sizeof(H5FD_splitter_fapl_t)) == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, NULL, "unable to shallow-copy info") + } + if (HDstrncpy(new_fa_ptr->wo_path, old_fa_ptr->wo_path, H5FD_SPLITTER_PATH_MAX) == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, NULL, "unable to copy write-only channel file path") + } + if (HDstrncpy(new_fa_ptr->log_file_path, old_fa_ptr->log_file_path, H5FD_SPLITTER_PATH_MAX) == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, NULL, "unable to copy log file path") + } + + /* Copy R/W and W/O FAPLs */ + if (H5FD__copy_plist(old_fa_ptr->rw_fapl_id, &(new_fa_ptr->rw_fapl_id)) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't copy R/W FAPL"); + } + if (H5FD__copy_plist(old_fa_ptr->wo_fapl_id, &(new_fa_ptr->wo_fapl_id)) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't copy W/O FAPL"); + } + + ret_value = (void *)new_fa_ptr; + +done: + if (NULL == ret_value) { + if (new_fa_ptr) { + H5MM_free(new_fa_ptr); + } + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_fapl_copy() */ + + +/*-------------------------------------------------------------------------- + * Function: H5FD_splitter_fapl_free + * + * Purpose: Releases the file access lists + * + * Return: SUCCEED/FAIL + *-------------------------------------------------------------------------- + */ +static herr_t +H5FD_splitter_fapl_free(void *_fapl) +{ + H5FD_splitter_fapl_t *fapl = (H5FD_splitter_fapl_t*)_fapl; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_fapl_free"); + + /* Check arguments */ + HDassert(fapl); + + if (H5I_dec_ref(fapl->rw_fapl_id) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTDEC, FAIL, "can't close R/W FAPL ID") + } + if (H5I_dec_ref(fapl->wo_fapl_id) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTDEC, FAIL, "can't close W/O FAPL ID") + } + + /* Free the property list */ + H5MM_free(fapl); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_fapl_free() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_open + * + * Purpose: Create and/or opens a file as an HDF5 file. + * + * Return: Success: A pointer to a new file data structure. The + * public fields will be initialized by the + * caller, which is always H5FD_open(). + * Failure: NULL + *------------------------------------------------------------------------- + */ +static H5FD_t * +H5FD_splitter_open(const char *name, unsigned flags, hid_t splitter_fapl_id, haddr_t maxaddr) +{ + H5FD_splitter_t *file_ptr = NULL; /* Splitter VFD info */ + const H5FD_splitter_fapl_t *fapl_ptr = NULL; /* Driver-specific property list */ + H5P_genplist_t *plist_ptr = NULL; + H5FD_t *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_open"); + + /* Check arguments */ + if (!name || !*name) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name") + } + if (0 == maxaddr || HADDR_UNDEF == maxaddr) { + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr") + } + if (ADDR_OVERFLOW(maxaddr)) { + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr") + } + if ( (H5P_FILE_ACCESS_DEFAULT == splitter_fapl_id) || + (H5FD_SPLITTER != H5Pget_driver(splitter_fapl_id)) ) + { + /* presupposes that H5P_FILE_ACCESS_DEFAULT is not a splitter */ + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "driver is not splitter") + } + + file_ptr = (H5FD_splitter_t *)H5FL_CALLOC(H5FD_splitter_t); + if (NULL == file_ptr) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate file struct") + } + file_ptr->fa.rw_fapl_id = H5I_INVALID_HID; + file_ptr->fa.wo_fapl_id = H5I_INVALID_HID; + + /* Get the driver-specific file access properties */ + plist_ptr = (H5P_genplist_t *)H5I_object(splitter_fapl_id); + if (NULL == plist_ptr) { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list") + } + fapl_ptr = (const H5FD_splitter_fapl_t *)H5P_peek_driver_info(plist_ptr); + if (NULL == fapl_ptr) { + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "unable to get VFL driver info") + } + + /* Copy simpler info */ + if (HDstrncpy(file_ptr->fa.wo_path, fapl_ptr->wo_path, H5FD_SPLITTER_PATH_MAX) == NULL) { + HGOTO_ERROR(H5E_VFL, H5E_CANTCOPY, NULL, "unable to copy write-only path") + } + if (HDstrncpy(file_ptr->fa.log_file_path, fapl_ptr->log_file_path, H5FD_SPLITTER_PATH_MAX) == NULL) { + HGOTO_ERROR(H5E_VFL, H5E_CANTCOPY, NULL, "unable to copy logfile path") + } + file_ptr->fa.ignore_wo_errs = fapl_ptr->ignore_wo_errs; + + /* Copy R/W and W/O channel FAPLs. */ + if (H5FD__copy_plist(fapl_ptr->rw_fapl_id, &(file_ptr->fa.rw_fapl_id)) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't copy R/W FAPL"); + } + if (H5FD__copy_plist(fapl_ptr->wo_fapl_id, &(file_ptr->fa.wo_fapl_id)) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't copy W/O FAPL"); + } + + /* Prepare log file if necessary. + * If application wants to ignore the errors from W/O channel and + * provided a name for the log file, then open it + */ + if (!file_ptr->logfp) { + if (file_ptr->fa.log_file_path[0] != '\0') { + file_ptr->logfp = HDfopen(file_ptr->fa.log_file_path, "w"); + if (file_ptr->logfp == NULL) { + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "unable to open log file") + } + } /* end if logfile path given */ + } /* end if logfile pointer/handle does not exist */ + + file_ptr->rw_file = H5FD_open(name, flags, fapl_ptr->rw_fapl_id, HADDR_UNDEF); + if (!file_ptr->rw_file) { + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "unable to open R/W file") + } + + file_ptr->wo_file = H5FD_open(fapl_ptr->wo_path, flags, fapl_ptr->wo_fapl_id, HADDR_UNDEF); + if (!file_ptr->wo_file) { + H5FD_SPLITTER_WO_ERROR(file_ptr, "H5FD_splitter_open", + H5E_VFL, H5E_CANTOPENFILE, NULL, + "unable to open W/O file") + } + + ret_value = (H5FD_t*)file_ptr; + +done: + if (NULL == ret_value) { + if (file_ptr) { + if (H5I_INVALID_HID != file_ptr->fa.rw_fapl_id) { + H5I_dec_ref(file_ptr->fa.rw_fapl_id); + } + if (H5I_INVALID_HID != file_ptr->fa.wo_fapl_id) { + H5I_dec_ref(file_ptr->fa.wo_fapl_id); + } + if (file_ptr->rw_file) { + H5FD_close(file_ptr->rw_file); + } + if (file_ptr->wo_file) { + H5FD_close(file_ptr->wo_file); + } + H5FL_FREE(H5FD_splitter_t, file_ptr); + } + } /* end if error */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_open() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_close + * + * Purpose: Closes files on both read-write and write-only channels. + * + * Return: Success: SUCCEED + * Failure: FAIL, file not closed. + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_splitter_close(H5FD_t *_file) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_close"); + + /* Sanity check */ + HDassert(file); + + if (H5I_dec_ref(file->fa.rw_fapl_id) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_ARGS, FAIL, "can't close R/W FAPL") + } + if (H5I_dec_ref(file->fa.wo_fapl_id) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_ARGS, FAIL, "can't close W/O FAPL") + } + + if (file->rw_file) { + if (H5FD_close(file->rw_file) == FAIL) { + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close R/W file") + } + } + if (file->wo_file) { + if (H5FD_close(file->wo_file) == FAIL) { + H5FD_SPLITTER_WO_ERROR(file, "H5FD_splitter_close", + H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + "unable to close W/O file") + } + } + + if (file->logfp) { + HDfclose(file->logfp); + file->logfp = NULL; + } + + /* Release the file info */ + file = H5FL_FREE(H5FD_splitter_t, file); + file = NULL; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_close() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_get_eoa + * + * Purpose: Returns the end-of-address marker for the file. The EOA + * marker is the first address past the last byte allocated in + * the format address space. + * + * Return: Success: The end-of-address-marker + * + * Failure: HADDR_UNDEF + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD_splitter_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_splitter_t *file = (const H5FD_splitter_t *)_file; + haddr_t ret_value = HADDR_UNDEF; + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_get_eoa"); + + /* Sanity check */ + HDassert(file); + HDassert(file->rw_file); + + if ((ret_value = H5FD_get_eoa(file->rw_file, type)) == HADDR_UNDEF) { + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, HADDR_UNDEF, "unable to get eoa") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_get_eoa */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_set_eoa + * + * Purpose: Set the end-of-address marker for the file. This function is + * called shortly after an existing HDF5 file is opened in order + * to tell the driver where the end of the HDF5 data is located. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_splitter_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_set_eoa";) + + /* Sanity check */ + HDassert(file); + HDassert(file->rw_file); + HDassert(file->wo_file); + + if (H5FD_set_eoa(file->rw_file, type, addr) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "H5FDset_eoa failed for R/W file") + } + + if (H5FD_set_eoa(file->wo_file, type, addr) < 0) { + H5FD_SPLITTER_WO_ERROR(file, "H5FD_splitter_set_eoa", + H5E_VFL, H5E_CANTSET, FAIL, + "unable to set EOA for W/O file") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_set_eoa() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_get_eof + * + * Purpose: Returns the end-of-address marker for the file. The EOA + * marker is the first address past the last byte allocated in + * the format address space. + * + * Return: Success: The end-of-address-marker + * + * Failure: HADDR_UNDEF + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD_splitter_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_splitter_t *file = (const H5FD_splitter_t *)_file; + haddr_t ret_value = HADDR_UNDEF; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_get_eof"); + + /* Sanity check */ + HDassert(file); + HDassert(file->rw_file); + + if (HADDR_UNDEF == (ret_value = H5FD_get_eof(file->rw_file, type))) { + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, HADDR_UNDEF, "unable to get eof") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_get_eof */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_truncate + * + * Purpose: Notify driver to truncate the file back to the allocated size. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_splitter_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_truncate"); + + HDassert(file); + HDassert(file->rw_file); + HDassert(file->wo_file); + + if (H5FDtruncate(file->rw_file, dxpl_id, closing) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "unable to truncate R/W file") + } + + if (H5FDtruncate(file->wo_file, dxpl_id, closing) < 0) { + H5FD_SPLITTER_WO_ERROR(file, "H5FD_splitter_truncate", + H5E_VFL, H5E_CANTUPDATE, FAIL, + "unable to truncate W/O file") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_truncate */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_sb_size + * + * Purpose: Obtains the number of bytes required to store the driver file + * access data in the HDF5 superblock. + * + * Return: Success: Number of bytes required. + * + * Failure: 0 if an error occurs or if the driver has no + * data to store in the superblock. + * + * NOTE: no public API for H5FD_sb_size, it needs to be added + *------------------------------------------------------------------------- + */ +static hsize_t +H5FD_splitter_sb_size(H5FD_t *_file) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + hsize_t ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_sb_size"); + + /* Sanity check */ + HDassert(file); + HDassert(file->rw_file); + + if (file->rw_file) { + ret_value = H5FD_sb_size(file->rw_file); + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_sb_size */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_sb_encode + * + * Purpose: Encode driver-specific data into the output arguments. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_splitter_sb_encode(H5FD_t *_file, char *name/*out*/, unsigned char *buf/*out*/) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_sb_encode"); + + /* Sanity check */ + HDassert(file); + HDassert(file->rw_file); + + if (file->rw_file && H5FD_sb_encode(file->rw_file, name, buf) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTENCODE, FAIL, "unable to encode the superblock in R/W file") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_sb_encode */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_sb_decode + * + * Purpose: Decodes the driver information block. + * + * Return: SUCCEED/FAIL + * + * NOTE: no public API for H5FD_sb_size, need to add + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_splitter_sb_decode(H5FD_t *_file, const char *name, const unsigned char *buf) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_sb_decode"); + + /* Sanity check */ + HDassert(file); + HDassert(file->rw_file); + + if (H5FD_sb_load(file->rw_file, name, buf) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "unable to decode the superblock in R/W file") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_sb_decode */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_cmp + * + * Purpose: Compare the keys of two files. + * + * Return: Success: A value like strcmp() + * Failure: Must never fail + *------------------------------------------------------------------------- + */ +static int +H5FD_splitter_cmp(const H5FD_t *_f1, const H5FD_t *_f2) +{ + const H5FD_splitter_t *f1 = (const H5FD_splitter_t *)_f1; + const H5FD_splitter_t *f2 = (const H5FD_splitter_t *)_f2; + herr_t ret_value = 0; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_cmp"); + + HDassert(f1); + HDassert(f2); + + ret_value = H5FD_cmp(f1->rw_file, f2->rw_file); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_cmp */ + + +/*-------------------------------------------------------------------------- + * Function: H5FD_splitter_get_handle + * + * Purpose: Returns a pointer to the file handle of low-level virtual + * file driver. + * + * Return: SUCCEED/FAIL + *-------------------------------------------------------------------------- + */ +static herr_t +H5FD_splitter_get_handle( + H5FD_t *_file, + hid_t H5_ATTR_UNUSED fapl, + void **file_handle) +{ + H5FD_splitter_t *file = (H5FD_splitter_t*)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_get_handle"); + + /* Check arguments */ + HDassert(file); + HDassert(file->rw_file); + HDassert(file_handle); + + /* Only do for R/W channel */ + if (H5FD_get_vfd_handle(file->rw_file, file->fa.rw_fapl_id, file_handle) < 0) + { + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "unable to get handle of R/W file") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_get_handle */ + + +/*-------------------------------------------------------------------------- + * Function: H5FD_splitter_lock + * + * Purpose: Sets a file lock. + * + * Return: SUCCEED/FAIL + *-------------------------------------------------------------------------- + */ +static herr_t +H5FD_splitter_lock(H5FD_t *_file, hbool_t rw) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; /* VFD file struct */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_lock"); + + HDassert(file); + HDassert(file->rw_file); + + /* Place the lock on each file */ + if (H5FD_lock(file->rw_file, rw) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCK, FAIL, "unable to lock R/W file") + } + if (file->wo_file != NULL) { + if (H5FD_lock(file->wo_file, rw) < 0) { + H5FD_SPLITTER_WO_ERROR(file, "H5FD_splitter_lock", + H5E_VFL, H5E_CANTLOCK, FAIL, + "unable to lock W/O file") + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_lock */ + + +/*-------------------------------------------------------------------------- + * Function: H5FD_splitter_unlock + * + * Purpose: Removes a file lock. + * + * Return: SUCCEED/FAIL + *-------------------------------------------------------------------------- + */ +static herr_t +H5FD_splitter_unlock(H5FD_t *_file) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; /* VFD file struct */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_unlock"); + + /* Check arguments */ + HDassert(file); + HDassert(file->rw_file); + + /* Remove the lock on each file */ + if (H5FD_unlock(file->rw_file) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCK, FAIL, "unable to unlock R/W file") + } + if (file->wo_file != NULL) { + if (H5FD_unlock(file->wo_file) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCK, FAIL, + "unable to unlock W/O file") + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_unlock */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_query + * + * Purpose: Set the flags that this VFL driver is capable of supporting. + * (listed in H5FDpublic.h) + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_splitter_query(const H5FD_t *_file, unsigned long *flags /* out */) +{ + const H5FD_splitter_t *file = (const H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_query"); + + if (file) { + HDassert(file); + HDassert(file->rw_file); + + if (H5FDquery(file->rw_file, flags) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCK, FAIL, + "unable to query R/W file"); + } + } + else { + /* There is no file. Because this is a pure passthrough VFD, + * it has no features of its own. + */ + if (flags) { + *flags = 0; + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_query() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_alloc + * + * Purpose: Allocate file memory. + * + * Return: Address of allocated space (HADDR_UNDEF if error). + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD_splitter_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; /* VFD file struct */ + haddr_t ret_value = HADDR_UNDEF; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_alloc"); + + /* Check arguments */ + HDassert(file); + HDassert(file->rw_file); + + /* Allocate memory for each file, only return the return value for R/W file. */ + if ((ret_value = H5FDalloc(file->rw_file, type, dxpl_id, size)) == HADDR_UNDEF) { + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "unable to allocate for R/W file") + } + + if (H5FDalloc(file->wo_file, type, dxpl_id, size) == HADDR_UNDEF) { + H5FD_SPLITTER_WO_ERROR(file, "H5FD_splitter_alloc", + H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, + "unable to alloc for W/O file") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_alloc() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_get_type_map + * + * Purpose: Retrieve the memory type mapping for this file + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_splitter_get_type_map(const H5FD_t *_file, H5FD_mem_t *type_map) +{ + const H5FD_splitter_t *file = (const H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_get_type_map"); + + /* Check arguments */ + HDassert(file); + HDassert(file->rw_file); + + /* Retrieve memory type mapping for R/W channel only */ + if (H5FD_get_fs_type_map(file->rw_file, type_map) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "unable to allocate for R/W file") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_get_type_map() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_free + * + * Purpose: Free the resources for the splitter VFD. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_splitter_free(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t size) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; /* VFD file struct */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + H5FD_SPLITTER_LOG_CALL("H5FD_splitter_free"); + + /* Check arguments */ + HDassert(file); + HDassert(file->rw_file); + + if (H5FDfree(file->rw_file, type, dxpl_id, addr, size) < 0) { + HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to free for R/W file") + } + + if (H5FDfree(file->wo_file, type, dxpl_id, addr, size) < 0) { + H5FD_SPLITTER_WO_ERROR(file, "H5FD_splitter_free", + H5E_VFL, H5E_CANTINIT, FAIL, + "unable to free for W/O file") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_free() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_log_error + * + * Purpose: Log an error from the W/O channel appropriately. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_log_error(const H5FD_splitter_t *file, const char *atfunc, const char *msg) +{ + size_t size = 0; + char *s = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC_NOERR + + H5FD_SPLITTER_LOG_CALL("H5FD__splitter_log_error"); + + /* Check arguments */ + HDassert(file); + HDassert(atfunc && *atfunc); + HDassert(msg && *msg); + + if (file->logfp != NULL) { + size = strlen(atfunc) + strlen(msg) + 3; /* ':', ' ', '\n' */ + s = (char *)malloc(sizeof(char) * (size+1)); + if (NULL == s) { + ret_value = FAIL; + } + else + if (size < (size_t)HDsnprintf(s, size+1, "%s: %s\n", atfunc, msg)) { + ret_value = FAIL; + } + else + if (size != HDfwrite(s, 1, size, file->logfp)) { + ret_value = FAIL; + } + HDfree(s); + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_log_error() */ + diff --git a/src/H5FDsplitter.h b/src/H5FDsplitter.h new file mode 100644 index 0000000..5a5ef29 --- /dev/null +++ b/src/H5FDsplitter.h @@ -0,0 +1,99 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: The public header file for the "splitter" driver. + */ + +#ifndef H5FDsplitter_H +#define H5FDsplitter_H + +#define H5FD_SPLITTER (H5FD_splitter_init()) + +/* The version of the H5FD_splitter_vfd_config_t structure used */ +#define H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION 1 + +/* Maximum length of a filename/path string in the Write-Only channel, + * including the NULL-terminator. + */ +#define H5FD_SPLITTER_PATH_MAX 4096 + +/* Semi-unique constant used to help identify structure pointers */ +#define H5FD_SPLITTER_MAGIC 0x2B916880 + +/* ---------------------------------------------------------------------------- + * Structure: H5FD_spliiter_vfd_config_t + * + * One-stop shopping for configuring a Splitter VFD (rather than many + * paramaters passed into H5Pset/get functions). + * + * magic (int32_t) + * Semi-unique number, used to sanity-check that a given pointer is + * likely (or not) to be this structure type. MUST be first. + * If magic is not H5FD_SPLITTER_MAGIC, the structure (and/or pointer to) + * must be considered invalid. + * + * version (unsigned int) + * Version number of this structure -- informs component membership. + * If not H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION, the structure (and/or + * pointer to) must be considered invalid. + * + * rw_fapl_id (hid_t) + * Library-given identification number of the Read/Write channel driver + * File Access Property List. + * The driver must support read/write access. + * Must be set to H5P_DEFAULT or a valid FAPL ID. + * + * wo_fapl_id (hid_t) + * Library-given identification number of the Read/Write channel driver + * File Access Property List. + * The driver feature flags must include H5FD_FEAT_DEFAULT_VFD_COMPAITBLE. + * Must be set to H5P_DEFAULT or a valid FAPL ID. + * + * wo_file_path (char[H5FD_SPLITTER_PATH_MAX + 1]) + * String buffer for the Write-Only channel target file. + * Must be null-terminated, cannot be empty. + * + * log_file_path (char[H5FD_SPLITTER_PATH_MAX + 1]) + * String buffer for the Splitter VFD logging output. + * Must be null-terminated. + * If null, no logfile is created. + * + * ignore_wo_errors (hbool_t) + * Toggle flag for how judiciously to respond to errors on the Write-Only + * channel. + * + * ---------------------------------------------------------------------------- + */ +typedef struct H5FD_splitter_vfd_config_t { + int32_t magic; + unsigned int version; + hid_t rw_fapl_id; + hid_t wo_fapl_id; + char wo_path[H5FD_SPLITTER_PATH_MAX + 1]; + char log_file_path[H5FD_SPLITTER_PATH_MAX + 1]; + hbool_t ignore_wo_errs; +} H5FD_splitter_vfd_config_t; + +#ifdef __cplusplus +extern "C" { +#endif +H5_DLL hid_t H5FD_splitter_init(void); +H5_DLL herr_t H5Pset_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *config_ptr); +H5_DLL herr_t H5Pget_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *config_ptr); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/src/H5private.h b/src/H5private.h index f570723..f7fb40b 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -365,6 +365,22 @@ #endif /* __cplusplus */ /* + * Networking headers used by the mirror VFD and related tests and utilities. + */ +#ifdef H5_HAVE_ARPA_INET_H +# include <arpa/inet.h> +#endif +#ifdef H5_HAVE_NETDB_H +# include <netdb.h> +#endif +#ifdef H5_HAVE_NETINET_IN_H +# include <netinet/in.h> +#endif +#ifdef H5_HAVE_SYS_SOCKET_H +# include <sys/socket.h> +#endif + +/* * Status return values for the `herr_t' type. * Since some unix/c routines use 0 and -1 (or more precisely, non-negative * vs. negative) as their return code, and some assumption had been made in @@ -646,6 +662,9 @@ typedef struct { #ifndef HDabs #define HDabs(X) abs(X) #endif /* HDabs */ +#ifndef HDaccept + #define HDaccept(A,B,C) accept((A),(B),(C)) /* mirror VFD */ +#endif /* HDaccept */ #ifndef HDaccess #define HDaccess(F,M) access(F, M) #endif /* HDaccess */ @@ -692,9 +711,15 @@ typedef struct { #ifndef HDatoll #define HDatoll(S) atoll(S) #endif /* HDatol */ +#ifndef HDbind + #define HDbind(A,B,C) bind((A),(B),(C)) /* mirror VFD */ +#endif /* HDbind */ #ifndef HDbsearch #define HDbsearch(K,B,N,Z,F) bsearch(K,B,N,Z,F) #endif /* HDbsearch */ +#ifndef HDbzero + #define HDbzero(A,B) bzero((A),(B)) /* mirror VFD */ +#endif /* HDbzero */ #ifndef HDcalloc #define HDcalloc(N,Z) calloc(N,Z) #endif /* HDcalloc */ @@ -734,6 +759,9 @@ typedef struct { #ifndef HDclosedir #define HDclosedir(D) closedir(D) #endif /* HDclosedir */ +#ifndef HDconnect + #define HDconnect(A,B,C) connect((A),(B),(C)) /* mirror VFD */ +#endif /* HDconnect */ #ifndef HDcos #define HDcos(X) cos(X) #endif /* HDcos */ @@ -978,9 +1006,12 @@ typedef off_t h5_stat_size_t; #ifndef HDgetgroups #define HDgetgroups(Z,G) getgroups(Z,G) #endif /* HDgetgroups */ +#ifndef HDgethostbyaddr + #define HDgethostbyaddr(A,B,C) gethostbyaddr((A),(B),(C)) /* mirror VFD */ +#endif /* HDgethostbyaddr */ #ifndef HDgethostname #define HDgethostname(N,L) gethostname(N,L) -#endif /* HDgetlogin */ +#endif /* HDgethostname */ #ifndef HDgetlogin #define HDgetlogin() getlogin() #endif /* HDgetlogin */ @@ -1014,6 +1045,18 @@ typedef off_t h5_stat_size_t; #ifndef HDgmtime #define HDgmtime(T) gmtime(T) #endif /* HDgmtime */ +#ifndef HDhtonl + #define HDhtonl(X) htonl((X)) /* mirror VFD */ +#endif /* HDhtonl */ +#ifndef HDhtons + #define HDhtons(X) htons((X)) /* mirror VFD */ +#endif /* HDhtons */ +#ifndef HDinet_addr + #define HDinet_addr(C) inet_addr((C)) /* mirror VFD */ +#endif /* HDinet_addr */ +#ifndef HDinet_ntoa + #define HDinet_ntoa(C) inet_ntoa((C)) /* mirror VFD */ +#endif /* HDinet_ntoa */ #ifndef HDisalnum #define HDisalnum(C) isalnum((int)(C)) /*cast for solaris warning*/ #endif /* HDisalnum */ @@ -1068,6 +1111,9 @@ typedef off_t h5_stat_size_t; #ifndef HDlink #define HDlink(OLD,NEW) link(OLD,NEW) #endif /* HDlink */ +#ifndef HDlisten + #define HDlisten(A,B) listen((A),(B)) /* mirror VFD */ +#endif /* HDlisten */ #ifndef HDllround #define HDllround(V) llround(V) #endif /* HDround */ @@ -1149,6 +1195,12 @@ typedef off_t h5_stat_size_t; #ifndef HDnanosleep #define HDnanosleep(N, O) nanosleep(N, O) #endif /* HDnanosleep */ +#ifndef HDntohl + #define HDntohl(A) ntohl((A)) /* mirror VFD */ +#endif /* HDntohl */ +#ifndef HDntohs + #define HDntohs(A) ntohs((A)) /* mirror VFD */ +#endif /* HDntohs */ #ifndef HDopen #define HDopen(F,...) open(F,__VA_ARGS__) #endif /* HDopen */ @@ -1296,12 +1348,21 @@ typedef off_t h5_stat_size_t; #ifndef HDsetsid #define HDsetsid() setsid() #endif /* HDsetsid */ +#ifndef HDsetsockopt + #define HDsetsockopt(A,B,C,D,E) setsockopt((A),(B),(C),(D),(E)) /* mirror VFD */ +#endif /* HDsetsockopt */ #ifndef HDsetuid #define HDsetuid(U) setuid(U) #endif /* HDsetuid */ #ifndef HDsetvbuf #define HDsetvbuf(F,S,M,Z) setvbuf(F,S,M,Z) #endif /* HDsetvbuf */ +#ifndef HDshutdown + #define HDshutdown(A, B) shutdown((A),(B)) /* mirror VFD */ +#endif /* HDshutdown */ +#ifndef HDsigaction + #define HDsigaction(S,A,O) sigaction((S),(A),(O)) +#endif /* HDsigaction */ #ifndef HDsigaddset #define HDsigaddset(S,N) sigaddset(S,N) #endif /* HDsigaddset */ @@ -1347,6 +1408,9 @@ typedef off_t h5_stat_size_t; #ifndef HDsnprintf #define HDsnprintf snprintf /*varargs*/ #endif /* HDsnprintf */ +#ifndef HDsocket + #define HDsocket(A,B,C) socket((A),(B),(C)) /* mirror VFD */ +#endif /* HDsocket */ #ifndef HDsprintf #define HDsprintf sprintf /*varargs*/ #endif /* HDsprintf */ diff --git a/src/Makefile.am b/src/Makefile.am index 0c07c1b..4f40e60 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -62,7 +62,8 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5lib_settings.c H5system.c \ H5FA.c H5FAcache.c H5FAdbg.c H5FAdblock.c H5FAdblkpage.c H5FAhdr.c \ H5FAint.c H5FAstat.c H5FAtest.c \ H5FD.c H5FDcore.c H5FDfamily.c H5FDhdfs.c H5FDint.c H5FDlog.c \ - H5FDmulti.c H5FDsec2.c H5FDspace.c H5FDstdio.c H5FDtest.c \ + H5FDmirror.c H5FDmulti.c H5FDros3.c H5FDsec2.c H5FDspace.c \ + H5FDsplitter.c H5FDstdio.c H5FDtest.c \ H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSint.c H5FSsection.c \ H5FSstat.c H5FStest.c \ H5G.c H5Gbtree2.c H5Gcache.c H5Gcompact.c H5Gdense.c H5Gdeprec.c \ @@ -135,9 +136,9 @@ include_HEADERS = hdf5.h H5api_adpt.h H5overflow.h H5pubconf.h H5public.h H5vers H5Apublic.h H5ACpublic.h \ H5Cpublic.h H5Dpublic.h \ H5Epubgen.h H5Epublic.h H5ESpublic.h H5Fpublic.h \ - H5FDpublic.h H5FDcore.h H5FDdirect.h \ - H5FDfamily.h H5FDhdfs.h H5FDlog.h H5FDmpi.h H5FDmpio.h \ - H5FDmulti.h H5FDros3.h H5FDsec2.h H5FDstdio.h H5FDwindows.h \ + H5FDpublic.h H5FDcore.h H5FDdirect.h H5FDfamily.h H5FDhdfs.h \ + H5FDlog.h H5FDmirror.h H5FDmpi.h H5FDmpio.h H5FDmulti.h H5FDros3.h \ + H5FDsec2.h H5FDsplitter.h H5FDstdio.h H5FDwindows.h \ H5Gpublic.h H5Ipublic.h H5Lpublic.h \ H5Mpublic.h H5MMpublic.h H5Opublic.h H5Ppublic.h \ H5PLextern.h H5PLpublic.h \ @@ -46,10 +46,12 @@ #include "H5FDfamily.h" /* File families */ #include "H5FDhdfs.h" /* Hadoop HDFS */ #include "H5FDlog.h" /* sec2 driver with I/O logging (for debugging) */ +#include "H5FDmirror.h" /* Mirror VFD and IPC definitions */ #include "H5FDmpi.h" /* MPI-based file drivers */ #include "H5FDmulti.h" /* Usage-partitioned file family */ #include "H5FDros3.h" /* R/O S3 "file" I/O */ #include "H5FDsec2.h" /* POSIX unbuffered file I/O */ +#include "H5FDsplitter.h" /* Twin-channel (R/W & R/O) I/O passthrough */ #include "H5FDstdio.h" /* Standard C buffered I/O */ #ifdef H5_HAVE_WINDOWS #include "H5FDwindows.h" /* Win32 I/O */ diff --git a/src/libhdf5.settings.in b/src/libhdf5.settings.in index 1591bed..df7ddf2 100644 --- a/src/libhdf5.settings.in +++ b/src/libhdf5.settings.in @@ -80,6 +80,7 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ MPE: @MPE@ Map (H5M) API: @MAP_API@ Direct VFD: @DIRECT_VFD@ + Mirror VFD: @MIRROR_VFD@ (Read-Only) S3 VFD: @ROS3_VFD@ (Read-Only) HDFS VFD: @HAVE_LIBHDFS@ dmalloc: @HAVE_DMALLOC@ |