From 355cb72ff12dc58c8ff6de63592dbe10ad84a085 Mon Sep 17 00:00:00 2001 From: mainzer Date: Mon, 16 Aug 2021 14:40:59 -0500 Subject: Added "ctl" callback to the VFD interfance, and the associated H5FDctl() and H5FD_ctl() calls. Modified the MPIO VFD accordingly -- specifically: Added ctl() call with op-code support to expose rank, size, and communicator. Modified H5FD_mpi_get_rank(), H5FD_mpi_get_size(), and H5FD_mpi_get_comm() to use the new ctl() callback. In passing removed the const qualifier from the file parameter of these functions, as the file parameter of the ctl callback is not const. Deleted the old H5FD__mpio_mpi_rank(), H5FD__mpio_mpi_size(), and H5FD__mpio_communicator() calls from the MPIO VFD. Deleted H5FD_class_mpi_t from H5FDprivate.h, and modified the MPIO VFD accordingly. Note that all VFDs now use H5FD_class_t, with no special class for VFDs that that support MPI. Some minor touch ups to the Neil's selection I/O mods in passing. Tested serial and parallel, debug and production on charis and jelly. --- src/H5FD.c | 108 ++++++++++++++++++++++++ src/H5FDcore.c | 1 + src/H5FDdirect.c | 1 + src/H5FDfamily.c | 1 + src/H5FDhdfs.c | 1 + src/H5FDint.c | 2 +- src/H5FDlog.c | 1 + src/H5FDmirror.c | 1 + src/H5FDmpi.c | 90 +++++++++++++++----- src/H5FDmpio.c | 236 ++++++++++++++++++++++++++--------------------------- src/H5FDmulti.c | 1 + src/H5FDprivate.h | 19 ++--- src/H5FDpublic.h | 65 +++++++++++++++ src/H5FDros3.c | 1 + src/H5FDsec2.c | 1 + src/H5FDsplitter.c | 1 + src/H5FDstdio.c | 1 + 17 files changed, 377 insertions(+), 154 deletions(-) diff --git a/src/H5FD.c b/src/H5FD.c index 62d5c38..3cc5e82 100644 --- a/src/H5FD.c +++ b/src/H5FD.c @@ -1645,6 +1645,8 @@ H5FDread_selection(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, uint32_t count, herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) + H5TRACE9("e", "*#MtiIu*i*i*a*zx", file, type, dxpl_id, count, mem_space_ids, file_space_ids, offsets, + element_sizes, bufs); /* Check arguments */ if (!file) @@ -1744,6 +1746,8 @@ H5FDwrite_selection(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, uint32_t count herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) + H5TRACE9("e", "*#MtiIu*i*i*a*z**x", file, type, dxpl_id, count, mem_space_ids, file_space_ids, offsets, + element_sizes, bufs); /* Check arguments */ if (!file) @@ -2049,6 +2053,110 @@ done: } /* end H5FD_unlock() */ /*------------------------------------------------------------------------- + * Function: H5FDctl + * + * Purpose: Perform a CTL operation. + * + * The desired operation is specified by the op_code + * parameter. + * + * The flags parameter controls management of op_codes that + * are unknown to the callback + * + * The input and output parameters allow op_code specific + * input and output + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: JRM -- 8/3/21 + * + *------------------------------------------------------------------------- + */ +herr_t +H5FDctl(H5FD_t *file, uint64_t op_code, uint64_t flags, const void *input, void **output) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE5("e", "*#ULUL*x**x", file, op_code, flags, input, output); + + /* Check arguments */ + if (!file) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file pointer cannot be NULL") + + if (!file->cls) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file class pointer cannot be NULL") + + /* Don't attempt to validate the op code. If appropriate, that will + * be done by the underlying VFD callback, along with the input and + * output parameters. + */ + + /* Call private function */ + if (H5FD_ctl(file, op_code, flags, input, output) < 0) + HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "VFD ctl request failed") + +done: + + FUNC_LEAVE_API(ret_value) + +} /* end H5FDctl() */ + +/*------------------------------------------------------------------------- + * Function: H5FD_ctl + * + * Purpose: Private version of H5FDctl() + * + * The desired operation is specified by the op_code + * parameter. + * + * The flags parameter controls management of op_codes that + * are unknown to the callback + * + * The input and output parameters allow op_code specific + * input and output + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: JRM -- 8/3/21 + * + *------------------------------------------------------------------------- + */ +herr_t +H5FD_ctl(H5FD_t *file, uint64_t op_code, uint64_t flags, const void *input, void **output) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(file); + HDassert(file->cls); + + /* Dispatch to driver if the ctl function exists. + * + * If it doesn't, fail if the H5FD_CTL__FAIL_IF_UNKNOWN_FLAG is set. + * + * Otherwise, report success. + */ + if (file->cls->ctl) { + + if ((file->cls->ctl)(file, op_code, flags, input, output) < 0) + + HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "VFD ctl request failed") + } + else if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG) { + + HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "VFD ctl request failed (no ctl and fail if unknown)") + } + +done: + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_ctl() */ + +/*------------------------------------------------------------------------- * Function: H5FD_get_fileno * * Purpose: Quick and dirty routine to retrieve the file's 'fileno' value diff --git a/src/H5FDcore.c b/src/H5FDcore.c index 4dea126..5fe845c 100644 --- a/src/H5FDcore.c +++ b/src/H5FDcore.c @@ -185,6 +185,7 @@ static const H5FD_class_t H5FD_core_g = { H5FD__core_truncate, /* truncate */ H5FD__core_lock, /* lock */ H5FD__core_unlock, /* unlock */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; diff --git a/src/H5FDdirect.c b/src/H5FDdirect.c index b777496..21a46da 100644 --- a/src/H5FDdirect.c +++ b/src/H5FDdirect.c @@ -174,6 +174,7 @@ static const H5FD_class_t H5FD_direct_g = { H5FD__direct_truncate, /* truncate */ H5FD__direct_lock, /* lock */ H5FD__direct_unlock, /* unlock */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; diff --git a/src/H5FDfamily.c b/src/H5FDfamily.c index c7ca09c..ac3f54d 100644 --- a/src/H5FDfamily.c +++ b/src/H5FDfamily.c @@ -139,6 +139,7 @@ static const H5FD_class_t H5FD_family_g = { H5FD__family_truncate, /* truncate */ H5FD__family_lock, /* lock */ H5FD__family_unlock, /* unlock */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; diff --git a/src/H5FDhdfs.c b/src/H5FDhdfs.c index 63d20a7..e48d62e 100644 --- a/src/H5FDhdfs.c +++ b/src/H5FDhdfs.c @@ -313,6 +313,7 @@ static const H5FD_class_t H5FD_hdfs_g = { H5FD__hdfs_truncate, /* truncate */ NULL, /* lock */ NULL, /* unlock */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; diff --git a/src/H5FDint.c b/src/H5FDint.c index 0972bf1..6111781 100644 --- a/src/H5FDint.c +++ b/src/H5FDint.c @@ -2088,7 +2088,7 @@ H5FD__vsrt_tmp_cmp(const void *element_1, const void *element_2) herr_t H5FD_sort_vector_io_req(hbool_t *vector_was_sorted, uint32_t count, H5FD_mem_t types[], haddr_t addrs[], - size_t sizes[], void *bufs[], H5FD_mem_t **s_types_ptr, haddr_t **s_addrs_ptr, + size_t sizes[], const void *bufs[], H5FD_mem_t **s_types_ptr, haddr_t **s_addrs_ptr, size_t **s_sizes_ptr, void ***s_bufs_ptr) { herr_t ret_value = SUCCEED; /* Return value */ diff --git a/src/H5FDlog.c b/src/H5FDlog.c index 1a4f8b0..ea18b02 100644 --- a/src/H5FDlog.c +++ b/src/H5FDlog.c @@ -214,6 +214,7 @@ static const H5FD_class_t H5FD_log_g = { H5FD__log_truncate, /* truncate */ H5FD__log_lock, /* lock */ H5FD__log_unlock, /* unlock */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; diff --git a/src/H5FDmirror.c b/src/H5FDmirror.c index 39fc7eb..4b7cb38 100644 --- a/src/H5FDmirror.c +++ b/src/H5FDmirror.c @@ -195,6 +195,7 @@ static const H5FD_class_t H5FD_mirror_g = { H5FD__mirror_truncate, /* truncate */ H5FD__mirror_lock, /* lock */ H5FD__mirror_unlock, /* unlock */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; diff --git a/src/H5FDmpi.c b/src/H5FDmpi.c index 048b8f3..7eb1463 100644 --- a/src/H5FDmpi.c +++ b/src/H5FDmpi.c @@ -41,26 +41,42 @@ * Programmer: Quincey Koziol * Friday, January 30, 2004 * + * Changes: Reworked function to use the ctl callback so we can get + * rid of H5FD_class_mpi_t. Since there are no real limits + * on what the ctl callback can do, its file parameter can't + * be constant. Thus, I had to remove the const qualifier + * on this functions file parameter as well. Note also the + * circumlocution required to use the ctl callbacks output + * parameter to pass back the rank without introducing + * compiler warnings. + * JRM -- 8/13/21 + * *------------------------------------------------------------------------- */ int -H5FD_mpi_get_rank(const H5FD_t *file) +H5FD_mpi_get_rank(H5FD_t *file) { - const H5FD_class_mpi_t *cls; - - int ret_value; + const H5FD_class_t *cls; + uint64_t flags = H5FD_CTL__FAIL_IF_UNKNOWN_FLAG | H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG; + int rank = -1; + void * rank_ptr = (void *)(&rank); + int ret_value; FUNC_ENTER_NOAPI(FAIL) HDassert(file); - cls = (const H5FD_class_mpi_t *)(file->cls); + cls = (const H5FD_class_t *)(file->cls); HDassert(cls); - HDassert(cls->get_rank); /* All MPI drivers must implement this */ + HDassert(cls->ctl); /* All MPI drivers must implement this */ /* Dispatch to driver */ - if ((ret_value = (cls->get_rank)(file)) < 0) + if ((cls->ctl)(file, H5FD_CTL__GET_MPI_RANK_OPCODE, flags, NULL, &rank_ptr) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "driver get_rank request failed") + HDassert(rank >= 0); + + ret_value = rank; + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_mpi_get_rank() */ @@ -77,25 +93,43 @@ done: * Programmer: Quincey Koziol * Friday, January 30, 2004 * + * Changes: Reworked function to use the ctl callback so we can get + * rid of H5FD_class_mpi_t. Since there are no real limits + * on what the ctl callback can do, its file parameter can't + * be constant. Thus, I had to remove the const qualifier + * on this functions file parameter as well. Note also the + * circumlocution required to use the ctl callbacks output + * parameter to pass back the rank without introducing + * compiler warnings. + * JRM -- 8/13/21 + * *------------------------------------------------------------------------- */ int -H5FD_mpi_get_size(const H5FD_t *file) +H5FD_mpi_get_size(H5FD_t *file) { - const H5FD_class_mpi_t *cls; - int ret_value; + const H5FD_class_t *cls; + uint64_t flags = H5FD_CTL__FAIL_IF_UNKNOWN_FLAG | H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG; + int size = 0; + void * size_ptr = (void *)(&size); + int ret_value; FUNC_ENTER_NOAPI(FAIL) HDassert(file); - cls = (const H5FD_class_mpi_t *)(file->cls); + cls = (const H5FD_class_t *)(file->cls); HDassert(cls); - HDassert(cls->get_size); /* All MPI drivers must implement this */ + HDassert(cls->ctl); /* All MPI drivers must implement this */ /* Dispatch to driver */ - if ((ret_value = (cls->get_size)(file)) < 0) + if ((cls->ctl)(file, H5FD_CTL__GET_MPI_SIZE_OPCODE, flags, NULL, &size_ptr) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "driver get_size request failed") + if (0 >= size) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "driver get_size request returned bad value") + + ret_value = size; + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_mpi_get_size() */ @@ -112,25 +146,43 @@ done: * Programmer: Quincey Koziol * Friday, January 30, 2004 * + * Changes: Reworked function to use the ctl callback so we can get + * rid of H5FD_class_mpi_t. Since there are no real limits + * on what the ctl callback can do, its file parameter can't + * be constant. Thus, I had to remove the const qualifier + * on this functions file parameter as well. Note also the + * circumlocution required to use the ctl callbacks output + * parameter to pass back the rank without introducing + * compiler warnings. + * JRM -- 8/13/21 + * *------------------------------------------------------------------------- */ MPI_Comm -H5FD_mpi_get_comm(const H5FD_t *file) +H5FD_mpi_get_comm(H5FD_t *file) { - const H5FD_class_mpi_t *cls; - MPI_Comm ret_value; + const H5FD_class_t *cls; + uint64_t flags = H5FD_CTL__FAIL_IF_UNKNOWN_FLAG | H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG; + MPI_Comm comm = MPI_COMM_NULL; + void * comm_ptr = (void *)(&comm); + MPI_Comm ret_value; FUNC_ENTER_NOAPI(MPI_COMM_NULL) HDassert(file); - cls = (const H5FD_class_mpi_t *)(file->cls); + cls = (const H5FD_class_t *)(file->cls); HDassert(cls); - HDassert(cls->get_comm); /* All MPI drivers must implement this */ + HDassert(cls->ctl); /* All MPI drivers must implement this */ /* Dispatch to driver */ - if ((ret_value = (cls->get_comm)(file)) == MPI_COMM_NULL) + if ((cls->ctl)(file, H5FD_CTL__GET_MPI_COMMUNICATOR_OPCODE, flags, NULL, &comm_ptr) < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTGET, MPI_COMM_NULL, "driver get_comm request failed") + if (comm == MPI_COMM_NULL) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, MPI_COMM_NULL, "driver get_comm request failed -- bad comm") + + ret_value = comm; + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_mpi_get_comm() */ diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index ce6d696..1db700a 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -72,70 +72,67 @@ typedef struct H5FD_mpio_t { /* Private Prototypes */ /* Callbacks */ -static herr_t H5FD__mpio_term(void); -static H5FD_t * H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); -static herr_t H5FD__mpio_close(H5FD_t *_file); -static herr_t H5FD__mpio_query(const H5FD_t *_f1, unsigned long *flags); -static haddr_t H5FD__mpio_get_eoa(const H5FD_t *_file, H5FD_mem_t type); -static herr_t H5FD__mpio_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); -static haddr_t H5FD__mpio_get_eof(const H5FD_t *_file, H5FD_mem_t type); -static herr_t H5FD__mpio_get_handle(H5FD_t *_file, hid_t fapl, void **file_handle); -static herr_t H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, - void *buf); -static herr_t H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, - const void *buf); -static herr_t H5FD__mpio_read_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t count, - H5FD_mem_t types[], haddr_t addrs[], size_t sizes[], void *bufs[]); -static herr_t H5FD__mpio_write_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t count, - H5FD_mem_t types[], haddr_t addrs[], size_t sizes[], void *bufs[]); -static herr_t H5FD__mpio_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); -static herr_t H5FD__mpio_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); -static int H5FD__mpio_mpi_rank(const H5FD_t *_file); -static int H5FD__mpio_mpi_size(const H5FD_t *_file); -static MPI_Comm H5FD__mpio_communicator(const H5FD_t *_file); +static herr_t H5FD__mpio_term(void); +static H5FD_t *H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); +static herr_t H5FD__mpio_close(H5FD_t *_file); +static herr_t H5FD__mpio_query(const H5FD_t *_f1, unsigned long *flags); +static haddr_t H5FD__mpio_get_eoa(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD__mpio_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); +static haddr_t H5FD__mpio_get_eof(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD__mpio_get_handle(H5FD_t *_file, hid_t fapl, void **file_handle); +static herr_t H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, + void *buf); +static herr_t H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, + const void *buf); +static herr_t H5FD__mpio_read_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t count, + H5FD_mem_t types[], haddr_t addrs[], size_t sizes[], void *bufs[]); +static herr_t H5FD__mpio_write_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t count, + H5FD_mem_t types[], haddr_t addrs[], size_t sizes[], + const void *bufs[]); +static herr_t H5FD__mpio_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); +static herr_t H5FD__mpio_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); +static herr_t H5FD__mpio_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, + const void H5_ATTR_UNUSED *input, void **output); /* The MPIO file driver information */ -static const H5FD_class_mpi_t H5FD_mpio_g = { - { - /* Start of superclass information */ - "mpio", /*name */ - HADDR_MAX, /*maxaddr */ - H5F_CLOSE_SEMI, /*fc_degree */ - H5FD__mpio_term, /*terminate */ - NULL, /*sb_size */ - NULL, /*sb_encode */ - NULL, /*sb_decode */ - 0, /*fapl_size */ - NULL, /*fapl_get */ - NULL, /*fapl_copy */ - NULL, /*fapl_free */ - 0, /*dxpl_size */ - NULL, /*dxpl_copy */ - NULL, /*dxpl_free */ - H5FD__mpio_open, /*open */ - H5FD__mpio_close, /*close */ - NULL, /*cmp */ - H5FD__mpio_query, /*query */ - NULL, /*get_type_map */ - NULL, /*alloc */ - NULL, /*free */ - H5FD__mpio_get_eoa, /*get_eoa */ - H5FD__mpio_set_eoa, /*set_eoa */ - H5FD__mpio_get_eof, /*get_eof */ - H5FD__mpio_get_handle, /*get_handle */ - H5FD__mpio_read, /*read */ - H5FD__mpio_write, /*write */ - H5FD__mpio_read_vector, /*read_vector */ - H5FD__mpio_write_vector, /*write_vector */ - H5FD__mpio_flush, /*flush */ - H5FD__mpio_truncate, /*truncate */ - NULL, /*lock */ - NULL, /*unlock */ - H5FD_FLMAP_DICHOTOMY /*fl_map */ - }, /* End of superclass information */ - H5FD__mpio_mpi_rank, /*get_rank */ - H5FD__mpio_mpi_size, /*get_size */ - H5FD__mpio_communicator /*get_comm */ +static const H5FD_class_t H5FD_mpio_g = { + "mpio", /*name */ + HADDR_MAX, /*maxaddr */ + H5F_CLOSE_SEMI, /*fc_degree */ + H5FD__mpio_term, /*terminate */ + NULL, /*sb_size */ + NULL, /*sb_encode */ + NULL, /*sb_decode */ + 0, /*fapl_size */ + NULL, /*fapl_get */ + NULL, /*fapl_copy */ + NULL, /*fapl_free */ + 0, /*dxpl_size */ + NULL, /*dxpl_copy */ + NULL, /*dxpl_free */ + H5FD__mpio_open, /*open */ + H5FD__mpio_close, /*close */ + NULL, /*cmp */ + H5FD__mpio_query, /*query */ + NULL, /*get_type_map */ + NULL, /*alloc */ + NULL, /*free */ + H5FD__mpio_get_eoa, /*get_eoa */ + H5FD__mpio_set_eoa, /*set_eoa */ + H5FD__mpio_get_eof, /*get_eof */ + H5FD__mpio_get_handle, /*get_handle */ + H5FD__mpio_read, /*read */ + H5FD__mpio_write, /*write */ + H5FD__mpio_read_vector, /*read_vector */ + H5FD__mpio_write_vector, /*write_vector */ + NULL, /*read_selection */ + NULL, /*write_selection */ + H5FD__mpio_flush, /*flush */ + H5FD__mpio_truncate, /*truncate */ + NULL, /*lock */ + NULL, /*unlock */ + H5FD__mpio_ctl, /*ctl */ + H5FD_FLMAP_DICHOTOMY /*fl_map */ }; #ifdef H5FDmpio_DEBUG @@ -214,7 +211,7 @@ H5FD_mpio_init(void) /* Register the MPI-IO VFD, if it isn't already */ if (H5I_VFL != H5I_get_type(H5FD_MPIO_g)) - H5FD_MPIO_g = H5FD_register((const H5FD_class_t *)&H5FD_mpio_g, sizeof(H5FD_class_mpi_t), FALSE); + H5FD_MPIO_g = H5FD_register((const H5FD_class_t *)&H5FD_mpio_g, sizeof(H5FD_class_t), FALSE); /* Allow MPI buf-and-file-type optimizations? */ s = HDgetenv("HDF5_MPI_OPT_TYPES"); @@ -1968,7 +1965,7 @@ done: */ static herr_t H5FD__mpio_write_vector(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, uint32_t count, H5FD_mem_t types[], - haddr_t addrs[], size_t sizes[], void *bufs[]) + haddr_t addrs[], size_t sizes[], const void *bufs[]) { H5FD_mpio_t * file = (H5FD_mpio_t *)_file; hbool_t vector_was_sorted = TRUE; @@ -2497,84 +2494,79 @@ done: } /* end H5FD__mpio_truncate() */ /*------------------------------------------------------------------------- - * Function: H5FD__mpio_mpi_rank + * Function: H5FD__mpio_ctl * - * Purpose: Returns the MPI rank for a process + * Purpose: MPIO version of the ctl callback. * - * Return: Success: non-negative - * Failure: negative + * The desired operation is specified by the op_code + * parameter. * - * Programmer: Quincey Koziol - * Thursday, May 16, 2002 + * The flags parameter controls management of op_codes that + * are unknown to the callback * - *------------------------------------------------------------------------- - */ -static int -H5FD__mpio_mpi_rank(const H5FD_t *_file) -{ - const H5FD_mpio_t *file = (const H5FD_mpio_t *)_file; - - FUNC_ENTER_STATIC_NOERR - - /* Sanity checks */ - HDassert(file); - HDassert(H5FD_MPIO == file->pub.driver_id); - - FUNC_LEAVE_NOAPI(file->mpi_rank) -} /* end H5FD__mpio_mpi_rank() */ - -/*------------------------------------------------------------------------- - * Function: H5FD__mpio_mpi_size + * The input and output parameters allow op_code specific + * input and output * - * Purpose: Returns the number of MPI processes + * At present, the supported op codes are: * - * Return: Success: non-negative - * Failure: negative + * H5FD_CTL__GET_MPI_COMMUNICATOR_OPCODE + * H5FD_CTL__GET_MPI_RANK_OPCODE + * H5FD_CTL__GET_MPI_SIZE_OPCODE * - * Programmer: Quincey Koziol - * Thursday, May 16, 2002 + * Note that these opcodes must be supported by all VFDs that + * support MPI. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: JRM -- 8/3/21 * *------------------------------------------------------------------------- */ -static int -H5FD__mpio_mpi_size(const H5FD_t *_file) +static herr_t +H5FD__mpio_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_ATTR_UNUSED *input, + void **output) { - const H5FD_mpio_t *file = (const H5FD_mpio_t *)_file; + H5FD_mpio_t *file = (H5FD_mpio_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC_NOERR + FUNC_ENTER_NOAPI(FAIL) /* Sanity checks */ HDassert(file); HDassert(H5FD_MPIO == file->pub.driver_id); - FUNC_LEAVE_NOAPI(file->mpi_size) -} /* end H5FD__mpio_mpi_size() */ + switch (op_code) { -/*------------------------------------------------------------------------- - * Function: H5FD__mpio_communicator - * - * Purpose: Returns the MPI communicator for the file. - * - * Return: Success: The communicator - * Failure: Can't fail - * - * Programmer: Robb Matzke - * Monday, August 9, 1999 - * - *------------------------------------------------------------------------- - */ -static MPI_Comm -H5FD__mpio_communicator(const H5FD_t *_file) -{ - const H5FD_mpio_t *file = (const H5FD_mpio_t *)_file; + case H5FD_CTL__GET_MPI_COMMUNICATOR_OPCODE: + HDassert(output); + HDassert(*output); + **((MPI_Comm **)output) = file->comm; + break; - FUNC_ENTER_STATIC_NOERR + case H5FD_CTL__GET_MPI_RANK_OPCODE: + HDassert(output); + HDassert(*output); + **((int **)output) = file->mpi_rank; + break; - /* Sanity checks */ - HDassert(file); - HDassert(H5FD_MPIO == file->pub.driver_id); + case H5FD_CTL__GET_MPI_SIZE_OPCODE: + HDassert(output); + HDassert(*output); + **((int **)output) = file->mpi_size; + break; + + default: /* unknown op code */ + if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG) { + + HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "unknown op_code and fail if unknown") + } + break; + } + +done: + + FUNC_LEAVE_NOAPI(ret_value) - FUNC_LEAVE_NOAPI(file->comm) -} /* end H5FD__mpio_communicator() */ +} /* end H5FD__mpio_ctl() */ #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5FDmulti.c b/src/H5FDmulti.c index 644e2f5..8527c60 100644 --- a/src/H5FDmulti.c +++ b/src/H5FDmulti.c @@ -172,6 +172,7 @@ static const H5FD_class_t H5FD_multi_g = { H5FD_multi_truncate, /*truncate */ H5FD_multi_lock, /*lock */ H5FD_multi_unlock, /*unlock */ + NULL, /*ctl */ H5FD_FLMAP_DEFAULT /*fl_map */ }; diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h index e0af020..c8b429f 100644 --- a/src/H5FDprivate.h +++ b/src/H5FDprivate.h @@ -45,13 +45,6 @@ /* Definitions for file MPI type property */ #define H5FD_MPI_XFER_FILE_MPI_TYPE_NAME "H5FD_mpi_file_mpi_type" -/* Sub-class the H5FD_class_t to add more specific functions for MPI-based VFDs */ -typedef struct H5FD_class_mpi_t { - H5FD_class_t super; /* Superclass information & methods */ - int (*get_rank)(const H5FD_t *file); /* Get the MPI rank of a process */ - int (*get_size)(const H5FD_t *file); /* Get the MPI size of a communicator */ - MPI_Comm (*get_comm)(const H5FD_t *file); /* Get the communicator for a file */ -} H5FD_class_mpi_t; #endif /****************************/ @@ -152,6 +145,7 @@ H5_DLL herr_t H5FD_flush(H5FD_t *file, hbool_t closing); H5_DLL herr_t H5FD_truncate(H5FD_t *file, hbool_t closing); H5_DLL herr_t H5FD_lock(H5FD_t *file, hbool_t rw); H5_DLL herr_t H5FD_unlock(H5FD_t *file); +H5_DLL herr_t H5FD_ctl(H5FD_t *file, uint64_t op_code, uint64_t flags, const void *input, void **output); H5_DLL herr_t H5FD_get_fileno(const H5FD_t *file, unsigned long *filenum); H5_DLL herr_t H5FD_get_vfd_handle(H5FD_t *file, hid_t fapl, void **file_handle); H5_DLL herr_t H5FD_set_base_addr(H5FD_t *file, haddr_t base_addr); @@ -159,8 +153,9 @@ H5_DLL haddr_t H5FD_get_base_addr(const H5FD_t *file); H5_DLL herr_t H5FD_set_paged_aggr(H5FD_t *file, hbool_t paged); H5_DLL herr_t H5FD_sort_vector_io_req(hbool_t *vector_was_sorted, uint32_t count, H5FD_mem_t types[], - haddr_t addrs[], size_t sizes[], void *bufs[], H5FD_mem_t **s_types_ptr, - haddr_t **s_addrs_ptr, size_t **s_sizes_ptr, void ***s_bufs_ptr); + haddr_t addrs[], size_t sizes[], const void *bufs[], + H5FD_mem_t **s_types_ptr, haddr_t **s_addrs_ptr, size_t **s_sizes_ptr, + void ***s_bufs_ptr); /* Function prototypes for MPI based VFDs*/ #ifdef H5_HAVE_PARALLEL @@ -175,9 +170,9 @@ H5_DLL herr_t H5FD_set_mpio_atomicity(H5FD_t *file, hbool_t flag); H5_DLL herr_t H5FD_get_mpio_atomicity(H5FD_t *file, hbool_t *flag); /* Driver specific methods */ -H5_DLL int H5FD_mpi_get_rank(const H5FD_t *file); -H5_DLL int H5FD_mpi_get_size(const H5FD_t *file); -H5_DLL MPI_Comm H5FD_mpi_get_comm(const H5FD_t *_file); +H5_DLL int H5FD_mpi_get_rank(H5FD_t *file); +H5_DLL int H5FD_mpi_get_size(H5FD_t *file); +H5_DLL MPI_Comm H5FD_mpi_get_comm(H5FD_t *file); #endif /* H5_HAVE_PARALLEL */ #endif /* !_H5FDprivate_H */ diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h index 8483667..eed41c2 100644 --- a/src/H5FDpublic.h +++ b/src/H5FDpublic.h @@ -263,6 +263,69 @@ typedef enum H5F_mem_t H5FD_mem_t; */ #define H5FD_FEAT_DEFAULT_VFD_COMPATIBLE 0x00008000 +/* ctl function op codes: */ +#define H5FD_CTL__INVALID_OPCODE 0 +#define H5FD_CTL__GET_MPI_COMMUNICATOR_OPCODE 1 +#define H5FD_CTL__GET_MPI_RANK_OPCODE 2 +#define H5FD_CTL__GET_MPI_SIZE_OPCODE 3 +#define H5FD_CTL__NUM_OPCODES 4 /* must be last */ + +/* ctl function flags: */ + +/* Definitions: + * + * WARNING: While the following definitions of Terminal + * and Passthrough VFDs should be workable for now, they + * have to be adjusted as our use cases for VFDs expand. + * + * JRM -- 8/4/21 + * + * + * Terminal VFD: Lowest VFD in the VFD stack through + * which all VFD calls pass. Note that this definition + * is situational. For example, the sec2 VFD is typically + * terminal. However, in the context of the family file + * driver, it is not -- the family file driver is the + * bottom VFD through which all VFD calls pass, and thus + * it is terminal. + * + * Similarly, on the splitter VFD, a sec2 VFD on the + * R/W channel is terminal, but a sec2 VFD on the W/O + * channel is not. + * + * + * Pass through VFD: Any VFD that relays all VFD calls + * (with the possible exception of some non-I/O related + * calls) to underlying VFD(s). + */ + +/* Unknown op codes should be ignored silently unless the + * H5FD_CTL__FAIL_IF_UNKNOWN_FLAG is set. + * + * On terminal VFDs, unknown op codes should generate an + * error unconditionally if this flag is set. + * + * On pass through VFDs, unknown op codes should be routed + * to the underlying VFD(s) as indicated by any routing + * flags. In the absence of such flags, the VFD should + * generate an error. + */ +#define H5FD_CTL__FAIL_IF_UNKNOWN_FLAG 0x0001 + +/* The H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG is used only + * by non-ternminal VFDs, and only applies to unknown + * opcodes. (known op codes should be handled as + * appropriate.) + * + * If this flag is set for an uknown op code, that + * op code should be passed to the next VFD down + * the VFD stack en-route to the terminal VFD. + * If that VFD does not support the ctl call, the + * pass through VFD should fail or succeed as directed + * by the H5FD_CTL__FAIL_IF_UNKNOWN_FLAG. + */ +#define H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG 0x0002 + /* Forward declaration */ typedef struct H5FD_t H5FD_t; @@ -309,6 +372,7 @@ typedef struct H5FD_class_t { herr_t (*truncate)(H5FD_t *file, hid_t dxpl_id, hbool_t closing); herr_t (*lock)(H5FD_t *file, hbool_t rw); herr_t (*unlock)(H5FD_t *file); + herr_t (*ctl)(H5FD_t *file, uint64_t op_code, uint64_t flags, const void *input, void **output); H5FD_mem_t fl_map[H5FD_MEM_NTYPES]; } H5FD_class_t; @@ -397,6 +461,7 @@ H5_DLL herr_t H5FDflush(H5FD_t *file, hid_t dxpl_id, hbool_t closing); H5_DLL herr_t H5FDtruncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing); H5_DLL herr_t H5FDlock(H5FD_t *file, hbool_t rw); H5_DLL herr_t H5FDunlock(H5FD_t *file); +H5_DLL herr_t H5FDctl(H5FD_t *file, uint64_t op_code, uint64_t flags, const void *input, void **output); /* Allows querying a VFD ID for features before the file is opened */ H5_DLL herr_t H5FDdriver_query(hid_t driver_id, unsigned long *flags /*out*/); diff --git a/src/H5FDros3.c b/src/H5FDros3.c index ed5fb03..165667c 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -270,6 +270,7 @@ static const H5FD_class_t H5FD_ros3_g = { H5FD__ros3_truncate, /* truncate */ NULL, /* lock */ NULL, /* unlock */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c index 03f8c2d..95a5414 100644 --- a/src/H5FDsec2.c +++ b/src/H5FDsec2.c @@ -175,6 +175,7 @@ static const H5FD_class_t H5FD_sec2_g = { H5FD__sec2_truncate, /* truncate */ H5FD__sec2_lock, /* lock */ H5FD__sec2_unlock, /* unlock */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; diff --git a/src/H5FDsplitter.c b/src/H5FDsplitter.c index 9aec6ac..045277e 100644 --- a/src/H5FDsplitter.c +++ b/src/H5FDsplitter.c @@ -166,6 +166,7 @@ static const H5FD_class_t H5FD_splitter_g = { H5FD__splitter_truncate, /* truncate */ H5FD__splitter_lock, /* lock */ H5FD__splitter_unlock, /* unlock */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; diff --git a/src/H5FDstdio.c b/src/H5FDstdio.c index a2be248..bb287b5 100644 --- a/src/H5FDstdio.c +++ b/src/H5FDstdio.c @@ -217,6 +217,7 @@ static const H5FD_class_t H5FD_stdio_g = { H5FD_stdio_truncate, /* truncate */ H5FD_stdio_lock, /* lock */ H5FD_stdio_unlock, /* unlock */ + NULL, /* ctl */ H5FD_FLMAP_DICHOTOMY /* fl_map */ }; -- cgit v0.12