VFD ctl feature (#981)

* Added "ctl" callback to the VFD interface, and the associated
H5FDctl() and H5FD_ctl() calls.

Modified the MPIO VFD accordingly -- specifically:

Added ctl() call with op-code support to expose rank, size,
and communicator.

Modified H5FD_mpi_get_rank(), H5FD_mpi_get_size(),
and H5FD_mpi_get_comm() to use the new ctl() callback.  In passing
removed the const qualifier from the file parameter of these
functions, as the file parameter of the ctl callback is not
const.

Deleted the old  H5FD__mpio_mpi_rank(), H5FD__mpio_mpi_size(), and
H5FD__mpio_communicator() calls from the MPIO VFD.

Deleted H5FD_class_mpi_t from H5FDprivate.h, and modified the
MPIO VFD accordingly.  Note that all VFDs now use H5FD_class_t,
with no special class for VFDs that that support MPI.

Some minor touch ups to the Neil's selection I/O mods in passing.

Tested serial and parallel, debug and production on charis and
jelly.

* Reserve a range of VFD "ctl" opcodes for library and experimental usage

* Add "ctl" callbacks to passthrough VFDs

* Add RELEASE.txt entry for "ctl" callback

* Use H5FDopen with H5F_ACC_RDWR flag instead of H5F_ACC_TRUNC in vfd test

* Remove handling of passthrough "ctl" flag from multi VFD

* Move logic for testing H5FD_CTL__TEST_OPCODE into a testing VFD

Revise description of "ctl" callback in RELEASE.txt

Remove unused H5FD_CTL__NUM_OPCODES definition

Fix some warnings in multi VFD

Co-authored-by: mainzer <mainzer#hdfgroup.org>

17 files changed, 526 insertions, 146 deletions

diff --git a/src/H5FD.c b/src/H5FD.c
index abda921..d3f72e8 100644
--- a/src/H5FD.c
+++ b/src/H5FD.c
@@ -1688,6 +1688,111 @@ done:
 } /* end H5FD_unlock() */
 
 /*-------------------------------------------------------------------------
+ * Function:    H5FDctl
+ *
+ * Purpose:     Perform a CTL operation.
+ *
+ *              The desired operation is specified by the op_code
+ *              parameter.
+ *
+ *              The flags parameter controls management of op_codes that
+ *              are unknown to the callback
+ *
+ *              The input and output parameters allow op_code specific
+ *              input and output
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Programmer:  JRM -- 8/3/21
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5FDctl(H5FD_t *file, uint64_t op_code, uint64_t flags, const void *input, void **output)
+{
+    herr_t ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_API(FAIL)
+    H5TRACE5("e", "*#ULUL*x**x", file, op_code, flags, input, output);
+
+    /* Check arguments */
+    if (!file)
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file pointer cannot be NULL")
+
+    if (!file->cls)
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file class pointer cannot be NULL")
+
+    /* Don't attempt to validate the op code.  If appropriate, that will
+     * be done by the underlying VFD callback, along with the input and
+     * output parameters.
+     */
+
+    /* Call private function */
+    if (H5FD_ctl(file, op_code, flags, input, output) < 0)
+        HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "VFD ctl request failed")
+
+done:
+
+    FUNC_LEAVE_API(ret_value)
+
+} /* end H5FDctl() */
+
+/*-------------------------------------------------------------------------
+ * Function:    H5FD_ctl
+ *
+ * Purpose:     Private version of H5FDctl()
+ *
+ *              The desired operation is specified by the op_code
+ *              parameter.
+ *
+ *              The flags parameter controls management of op_codes that
+ *              are unknown to the callback
+ *
+ *              The input and output parameters allow op_code specific
+ *              input and output
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Programmer:  JRM -- 8/3/21
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5FD_ctl(H5FD_t *file, uint64_t op_code, uint64_t flags, const void *input, void **output)
+{
+    herr_t ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /* Sanity checks */
+    HDassert(file);
+    HDassert(file->cls);
+
+    /* Dispatch to driver if the ctl function exists.
+     *
+     * If it doesn't, fail if the H5FD_CTL__FAIL_IF_UNKNOWN_FLAG is set.
+     *
+     * Otherwise, report success.
+     */
+    if (file->cls->ctl) {
+
+        if ((file->cls->ctl)(file, op_code, flags, input, output) < 0)
+
+            HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "VFD ctl request failed")
+    }
+    else if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG) {
+
+        HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL,
+                    "VFD ctl request failed (no ctl callback and fail if unknown flag is set)")
+    }
+
+done:
+
+    FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD_ctl() */
+
+/*-------------------------------------------------------------------------
  * Function:    H5FD_get_fileno
  *
  * Purpose:     Quick and dirty routine to retrieve the file's 'fileno' value
diff --git a/src/H5FDcore.c b/src/H5FDcore.c
index 50288c4..820c74f 100644
--- a/src/H5FDcore.c
+++ b/src/H5FDcore.c
@@ -183,6 +183,7 @@ static const H5FD_class_t H5FD_core_g = {
     H5FD__core_lock,          /* lock                 */
     H5FD__core_unlock,        /* unlock               */
     H5FD__core_delete,        /* del                  */
+    NULL,                     /* ctl                  */
     H5FD_FLMAP_DICHOTOMY      /* fl_map               */
 };
 
diff --git a/src/H5FDdevelop.h b/src/H5FDdevelop.h
index 57f0a42..4895658 100644
--- a/src/H5FDdevelop.h
+++ b/src/H5FDdevelop.h
@@ -192,6 +192,7 @@ typedef struct H5FD_class_t {
     herr_t (*lock)(H5FD_t *file, hbool_t rw);
     herr_t (*unlock)(H5FD_t *file);
     herr_t (*del)(const char *name, hid_t fapl);
+    herr_t (*ctl)(H5FD_t *file, uint64_t op_code, uint64_t flags, const void *input, void **output);
     H5FD_mem_t fl_map[H5FD_MEM_NTYPES];
 } H5FD_class_t;
 
@@ -254,6 +255,7 @@ H5_DLL herr_t  H5FDtruncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing);
 H5_DLL herr_t  H5FDlock(H5FD_t *file, hbool_t rw);
 H5_DLL herr_t  H5FDunlock(H5FD_t *file);
 H5_DLL herr_t  H5FDdelete(const char *name, hid_t fapl_id);
+H5_DLL herr_t  H5FDctl(H5FD_t *file, uint64_t op_code, uint64_t flags, const void *input, void **output);
 
 #ifdef __cplusplus
 }
diff --git a/src/H5FDdirect.c b/src/H5FDdirect.c
index a1b7b7e..7cca09f 100644
--- a/src/H5FDdirect.c
+++ b/src/H5FDdirect.c
@@ -172,6 +172,7 @@ static const H5FD_class_t H5FD_direct_g = {
     H5FD__direct_lock,          /* lock                 */
     H5FD__direct_unlock,        /* unlock               */
     H5FD__direct_delete,        /* del                  */
+    NULL,                       /* ctl                  */
     H5FD_FLMAP_DICHOTOMY        /* fl_map               */
 };
 
diff --git a/src/H5FDfamily.c b/src/H5FDfamily.c
index 8cf9f9e..af67c78 100644
--- a/src/H5FDfamily.c
+++ b/src/H5FDfamily.c
@@ -137,6 +137,7 @@ static const H5FD_class_t H5FD_family_g = {
     H5FD__family_lock,          /* lock                 */
     H5FD__family_unlock,        /* unlock               */
     H5FD__family_delete,        /* del                  */
+    NULL,                       /* ctl                  */
     H5FD_FLMAP_DICHOTOMY        /* fl_map               */
 };
 
diff --git a/src/H5FDhdfs.c b/src/H5FDhdfs.c
index 2c4bff6..ac48b42 100644
--- a/src/H5FDhdfs.c
+++ b/src/H5FDhdfs.c
@@ -310,6 +310,7 @@ static const H5FD_class_t H5FD_hdfs_g = {
     NULL,                     /* lock                 */
     NULL,                     /* unlock               */
     NULL,                     /* del                  */
+    NULL,                     /* ctl                  */
     H5FD_FLMAP_DICHOTOMY      /* fl_map               */
 };
 
diff --git a/src/H5FDlog.c b/src/H5FDlog.c
index f996b9e..87871ab 100644
--- a/src/H5FDlog.c
+++ b/src/H5FDlog.c
@@ -212,6 +212,7 @@ static const H5FD_class_t H5FD_log_g = {
     H5FD__log_lock,          /* lock                */
     H5FD__log_unlock,        /* unlock              */
     H5FD__log_delete,        /* del                 */
+    NULL,                    /* ctl                 */
     H5FD_FLMAP_DICHOTOMY     /* fl_map              */
 };
 
diff --git a/src/H5FDmirror.c b/src/H5FDmirror.c
index 8cbeff6..d539f4d 100644
--- a/src/H5FDmirror.c
+++ b/src/H5FDmirror.c
@@ -192,6 +192,7 @@ static const H5FD_class_t H5FD_mirror_g = {
     H5FD__mirror_lock,      /* lock                 */
     H5FD__mirror_unlock,    /* unlock               */
     NULL,                   /* del                  */
+    NULL,                   /* ctl                  */
     H5FD_FLMAP_DICHOTOMY    /* fl_map               */
 };
 
diff --git a/src/H5FDmpi.c b/src/H5FDmpi.c
index 048b8f3..7eb1463 100644
--- a/src/H5FDmpi.c
+++ b/src/H5FDmpi.c
@@ -41,26 +41,42 @@
  * Programmer:	Quincey Koziol
  *              Friday, January 30, 2004
  *
+ * Changes:     Reworked function to use the ctl callback so we can get
+ *              rid of H5FD_class_mpi_t.  Since there are no real limits
+ *              on what the ctl callback can do, its file parameter can't
+ *              be constant.  Thus, I had to remove the const qualifier
+ *              on this functions file parameter as well.  Note also the
+ *              circumlocution required to use the ctl callbacks output
+ *              parameter to pass back the rank without introducing
+ *              compiler warnings.
+ *                                             JRM -- 8/13/21
+ *
  *-------------------------------------------------------------------------
  */
 int
-H5FD_mpi_get_rank(const H5FD_t *file)
+H5FD_mpi_get_rank(H5FD_t *file)
 {
-    const H5FD_class_mpi_t *cls;
-
-    int ret_value;
+    const H5FD_class_t *cls;
+    uint64_t            flags    = H5FD_CTL__FAIL_IF_UNKNOWN_FLAG | H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG;
+    int                 rank     = -1;
+    void *              rank_ptr = (void *)(&rank);
+    int                 ret_value;
 
     FUNC_ENTER_NOAPI(FAIL)
 
     HDassert(file);
-    cls = (const H5FD_class_mpi_t *)(file->cls);
+    cls = (const H5FD_class_t *)(file->cls);
     HDassert(cls);
-    HDassert(cls->get_rank); /* All MPI drivers must implement this */
+    HDassert(cls->ctl); /* All MPI drivers must implement this */
 
     /* Dispatch to driver */
-    if ((ret_value = (cls->get_rank)(file)) < 0)
+    if ((cls->ctl)(file, H5FD_CTL__GET_MPI_RANK_OPCODE, flags, NULL, &rank_ptr) < 0)
         HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "driver get_rank request failed")
 
+    HDassert(rank >= 0);
+
+    ret_value = rank;
+
 done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5FD_mpi_get_rank() */
@@ -77,25 +93,43 @@ done:
  * Programmer:	Quincey Koziol
  *              Friday, January 30, 2004
  *
+ * Changes:     Reworked function to use the ctl callback so we can get
+ *              rid of H5FD_class_mpi_t.  Since there are no real limits
+ *              on what the ctl callback can do, its file parameter can't
+ *              be constant.  Thus, I had to remove the const qualifier
+ *              on this functions file parameter as well.  Note also the
+ *              circumlocution required to use the ctl callbacks output
+ *              parameter to pass back the rank without introducing
+ *              compiler warnings.
+ *                                             JRM -- 8/13/21
+ *
  *-------------------------------------------------------------------------
  */
 int
-H5FD_mpi_get_size(const H5FD_t *file)
+H5FD_mpi_get_size(H5FD_t *file)
 {
-    const H5FD_class_mpi_t *cls;
-    int                     ret_value;
+    const H5FD_class_t *cls;
+    uint64_t            flags    = H5FD_CTL__FAIL_IF_UNKNOWN_FLAG | H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG;
+    int                 size     = 0;
+    void *              size_ptr = (void *)(&size);
+    int                 ret_value;
 
     FUNC_ENTER_NOAPI(FAIL)
 
     HDassert(file);
-    cls = (const H5FD_class_mpi_t *)(file->cls);
+    cls = (const H5FD_class_t *)(file->cls);
     HDassert(cls);
-    HDassert(cls->get_size); /* All MPI drivers must implement this */
+    HDassert(cls->ctl); /* All MPI drivers must implement this */
 
     /* Dispatch to driver */
-    if ((ret_value = (cls->get_size)(file)) < 0)
+    if ((cls->ctl)(file, H5FD_CTL__GET_MPI_SIZE_OPCODE, flags, NULL, &size_ptr) < 0)
         HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "driver get_size request failed")
 
+    if (0 >= size)
+        HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "driver get_size request returned bad value")
+
+    ret_value = size;
+
 done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5FD_mpi_get_size() */
@@ -112,25 +146,43 @@ done:
  * Programmer:	Quincey Koziol
  *              Friday, January 30, 2004
  *
+ * Changes:     Reworked function to use the ctl callback so we can get
+ *              rid of H5FD_class_mpi_t.  Since there are no real limits
+ *              on what the ctl callback can do, its file parameter can't
+ *              be constant.  Thus, I had to remove the const qualifier
+ *              on this functions file parameter as well.  Note also the
+ *              circumlocution required to use the ctl callbacks output
+ *              parameter to pass back the rank without introducing
+ *              compiler warnings.
+ *                                             JRM -- 8/13/21
+ *
  *-------------------------------------------------------------------------
  */
 MPI_Comm
-H5FD_mpi_get_comm(const H5FD_t *file)
+H5FD_mpi_get_comm(H5FD_t *file)
 {
-    const H5FD_class_mpi_t *cls;
-    MPI_Comm                ret_value;
+    const H5FD_class_t *cls;
+    uint64_t            flags    = H5FD_CTL__FAIL_IF_UNKNOWN_FLAG | H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG;
+    MPI_Comm            comm     = MPI_COMM_NULL;
+    void *              comm_ptr = (void *)(&comm);
+    MPI_Comm            ret_value;
 
     FUNC_ENTER_NOAPI(MPI_COMM_NULL)
 
     HDassert(file);
-    cls = (const H5FD_class_mpi_t *)(file->cls);
+    cls = (const H5FD_class_t *)(file->cls);
     HDassert(cls);
-    HDassert(cls->get_comm); /* All MPI drivers must implement this */
+    HDassert(cls->ctl); /* All MPI drivers must implement this */
 
     /* Dispatch to driver */
-    if ((ret_value = (cls->get_comm)(file)) == MPI_COMM_NULL)
+    if ((cls->ctl)(file, H5FD_CTL__GET_MPI_COMMUNICATOR_OPCODE, flags, NULL, &comm_ptr) < 0)
         HGOTO_ERROR(H5E_VFL, H5E_CANTGET, MPI_COMM_NULL, "driver get_comm request failed")
 
+    if (comm == MPI_COMM_NULL)
+        HGOTO_ERROR(H5E_VFL, H5E_CANTGET, MPI_COMM_NULL, "driver get_comm request failed -- bad comm")
+
+    ret_value = comm;
+
 done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5FD_mpi_get_comm() */
diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c
index dd40399..7c85897 100644
--- a/src/H5FDmpio.c
+++ b/src/H5FDmpio.c
@@ -72,66 +72,60 @@ typedef struct H5FD_mpio_t {
 /* Private Prototypes */
 
 /* Callbacks */
-static herr_t   H5FD__mpio_term(void);
-static H5FD_t * H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr);
-static herr_t   H5FD__mpio_close(H5FD_t *_file);
-static herr_t   H5FD__mpio_query(const H5FD_t *_f1, unsigned long *flags);
-static haddr_t  H5FD__mpio_get_eoa(const H5FD_t *_file, H5FD_mem_t type);
-static herr_t   H5FD__mpio_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr);
-static haddr_t  H5FD__mpio_get_eof(const H5FD_t *_file, H5FD_mem_t type);
-static herr_t   H5FD__mpio_get_handle(H5FD_t *_file, hid_t fapl, void **file_handle);
-static herr_t   H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size,
-                                void *buf);
-static herr_t   H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size,
-                                 const void *buf);
-static herr_t   H5FD__mpio_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closing);
-static herr_t   H5FD__mpio_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing);
-static herr_t   H5FD__mpio_delete(const char *filename, hid_t fapl_id);
-static int      H5FD__mpio_mpi_rank(const H5FD_t *_file);
-static int      H5FD__mpio_mpi_size(const H5FD_t *_file);
-static MPI_Comm H5FD__mpio_communicator(const H5FD_t *_file);
+static herr_t  H5FD__mpio_term(void);
+static H5FD_t *H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr);
+static herr_t  H5FD__mpio_close(H5FD_t *_file);
+static herr_t  H5FD__mpio_query(const H5FD_t *_f1, unsigned long *flags);
+static haddr_t H5FD__mpio_get_eoa(const H5FD_t *_file, H5FD_mem_t type);
+static herr_t  H5FD__mpio_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr);
+static haddr_t H5FD__mpio_get_eof(const H5FD_t *_file, H5FD_mem_t type);
+static herr_t  H5FD__mpio_get_handle(H5FD_t *_file, hid_t fapl, void **file_handle);
+static herr_t  H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size,
+                               void *buf);
+static herr_t  H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size,
+                                const void *buf);
+static herr_t  H5FD__mpio_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closing);
+static herr_t  H5FD__mpio_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing);
+static herr_t  H5FD__mpio_delete(const char *filename, hid_t fapl_id);
+static herr_t  H5FD__mpio_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input,
+                              void **output);
 
 /* The MPIO file driver information */
-static const H5FD_class_mpi_t H5FD_mpio_g = {
-    {
-        /* Start of superclass information */
-        "mpio",                /* name                  */
-        HADDR_MAX,             /* maxaddr               */
-        H5F_CLOSE_SEMI,        /* fc_degree             */
-        H5FD__mpio_term,       /* terminate             */
-        NULL,                  /* sb_size               */
-        NULL,                  /* sb_encode             */
-        NULL,                  /* sb_decode             */
-        0,                     /* fapl_size             */
-        NULL,                  /* fapl_get              */
-        NULL,                  /* fapl_copy             */
-        NULL,                  /* fapl_free             */
-        0,                     /* dxpl_size             */
-        NULL,                  /* dxpl_copy             */
-        NULL,                  /* dxpl_free             */
-        H5FD__mpio_open,       /* open                  */
-        H5FD__mpio_close,      /* close                 */
-        NULL,                  /* cmp                   */
-        H5FD__mpio_query,      /* query                 */
-        NULL,                  /* get_type_map          */
-        NULL,                  /* alloc                 */
-        NULL,                  /* free                  */
-        H5FD__mpio_get_eoa,    /* get_eoa               */
-        H5FD__mpio_set_eoa,    /* set_eoa               */
-        H5FD__mpio_get_eof,    /* get_eof               */
-        H5FD__mpio_get_handle, /* get_handle            */
-        H5FD__mpio_read,       /* read                  */
-        H5FD__mpio_write,      /* write                 */
-        H5FD__mpio_flush,      /* flush                 */
-        H5FD__mpio_truncate,   /* truncate              */
-        NULL,                  /* lock                  */
-        NULL,                  /* unlock                */
-        H5FD__mpio_delete,     /* del                   */
-        H5FD_FLMAP_DICHOTOMY   /* fl_map                */
-    },                         /* End of superclass information */
-    H5FD__mpio_mpi_rank,       /* get_rank              */
-    H5FD__mpio_mpi_size,       /* get_size              */
-    H5FD__mpio_communicator    /* get_comm              */
+static const H5FD_class_t H5FD_mpio_g = {
+    "mpio",                /* name                  */
+    HADDR_MAX,             /* maxaddr               */
+    H5F_CLOSE_SEMI,        /* fc_degree             */
+    H5FD__mpio_term,       /* terminate             */
+    NULL,                  /* sb_size               */
+    NULL,                  /* sb_encode             */
+    NULL,                  /* sb_decode             */
+    0,                     /* fapl_size             */
+    NULL,                  /* fapl_get              */
+    NULL,                  /* fapl_copy             */
+    NULL,                  /* fapl_free             */
+    0,                     /* dxpl_size             */
+    NULL,                  /* dxpl_copy             */
+    NULL,                  /* dxpl_free             */
+    H5FD__mpio_open,       /* open                  */
+    H5FD__mpio_close,      /* close                 */
+    NULL,                  /* cmp                   */
+    H5FD__mpio_query,      /* query                 */
+    NULL,                  /* get_type_map          */
+    NULL,                  /* alloc                 */
+    NULL,                  /* free                  */
+    H5FD__mpio_get_eoa,    /* get_eoa               */
+    H5FD__mpio_set_eoa,    /* set_eoa               */
+    H5FD__mpio_get_eof,    /* get_eof               */
+    H5FD__mpio_get_handle, /* get_handle            */
+    H5FD__mpio_read,       /* read                  */
+    H5FD__mpio_write,      /* write                 */
+    H5FD__mpio_flush,      /* flush                 */
+    H5FD__mpio_truncate,   /* truncate              */
+    NULL,                  /* lock                  */
+    NULL,                  /* unlock                */
+    H5FD__mpio_delete,     /* del                   */
+    H5FD__mpio_ctl,        /* ctl                   */
+    H5FD_FLMAP_DICHOTOMY   /* fl_map                */
 };
 
 #ifdef H5FDmpio_DEBUG
@@ -245,7 +239,7 @@ H5FD_mpio_init(void)
 
     /* Register the MPI-IO VFD, if it isn't already */
     if (H5I_VFL != H5I_get_type(H5FD_MPIO_g))
-        H5FD_MPIO_g = H5FD_register((const H5FD_class_t *)&H5FD_mpio_g, sizeof(H5FD_class_mpi_t), FALSE);
+        H5FD_MPIO_g = H5FD_register((const H5FD_class_t *)&H5FD_mpio_g, sizeof(H5FD_class_t), FALSE);
 
     if (!H5FD_mpio_Debug_inited) {
         const char *s; /* String for environment variables */
@@ -1802,83 +1796,78 @@ done:
 } /* end H5FD__mpio_delete() */
 
 /*-------------------------------------------------------------------------
- * Function:    H5FD__mpio_mpi_rank
+ * Function:    H5FD__mpio_ctl
  *
- * Purpose:     Returns the MPI rank for a process
+ * Purpose:     MPIO version of the ctl callback.
  *
- * Return:      Success: non-negative
- *              Failure: negative
+ *              The desired operation is specified by the op_code
+ *              parameter.
  *
- * Programmer:  Quincey Koziol
- *              Thursday, May 16, 2002
+ *              The flags parameter controls management of op_codes that
+ *              are unknown to the callback
  *
- *-------------------------------------------------------------------------
- */
-static int
-H5FD__mpio_mpi_rank(const H5FD_t *_file)
-{
-    const H5FD_mpio_t *file = (const H5FD_mpio_t *)_file;
-
-    FUNC_ENTER_STATIC_NOERR
-
-    /* Sanity checks */
-    HDassert(file);
-    HDassert(H5FD_MPIO == file->pub.driver_id);
-
-    FUNC_LEAVE_NOAPI(file->mpi_rank)
-} /* end H5FD__mpio_mpi_rank() */
-
-/*-------------------------------------------------------------------------
- * Function:    H5FD__mpio_mpi_size
+ *              The input and output parameters allow op_code specific
+ *              input and output
  *
- * Purpose:     Returns the number of MPI processes
+ *              At present, the supported op codes are:
  *
- * Return:      Success: non-negative
- *              Failure: negative
+ *                  H5FD_CTL__GET_MPI_COMMUNICATOR_OPCODE
+ *                  H5FD_CTL__GET_MPI_RANK_OPCODE
+ *                  H5FD_CTL__GET_MPI_SIZE_OPCODE
  *
- * Programmer:  Quincey Koziol
- *              Thursday, May 16, 2002
+ *              Note that these opcodes must be supported by all VFDs that
+ *              support MPI.
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Programmer:  JRM -- 8/3/21
  *
  *-------------------------------------------------------------------------
  */
-static int
-H5FD__mpio_mpi_size(const H5FD_t *_file)
+static herr_t
+H5FD__mpio_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_ATTR_UNUSED *input,
+               void **output)
 {
-    const H5FD_mpio_t *file = (const H5FD_mpio_t *)_file;
+    H5FD_mpio_t *file      = (H5FD_mpio_t *)_file;
+    herr_t       ret_value = SUCCEED; /* Return value */
 
-    FUNC_ENTER_STATIC_NOERR
+    FUNC_ENTER_NOAPI(FAIL)
 
     /* Sanity checks */
     HDassert(file);
     HDassert(H5FD_MPIO == file->pub.driver_id);
 
-    FUNC_LEAVE_NOAPI(file->mpi_size)
-} /* end H5FD__mpio_mpi_size() */
+    switch (op_code) {
 
-/*-------------------------------------------------------------------------
- * Function:    H5FD__mpio_communicator
- *
- * Purpose:     Returns the MPI communicator for the file.
- *
- * Return:      Success:    The communicator
- *              Failure:    Can't fail
- *
- * Programmer:  Robb Matzke
- *              Monday, August  9, 1999
- *
- *-------------------------------------------------------------------------
- */
-static MPI_Comm
-H5FD__mpio_communicator(const H5FD_t *_file)
-{
-    const H5FD_mpio_t *file = (const H5FD_mpio_t *)_file;
+        case H5FD_CTL__GET_MPI_COMMUNICATOR_OPCODE:
+            HDassert(output);
+            HDassert(*output);
+            **((MPI_Comm **)output) = file->comm;
+            break;
 
-    FUNC_ENTER_STATIC_NOERR
+        case H5FD_CTL__GET_MPI_RANK_OPCODE:
+            HDassert(output);
+            HDassert(*output);
+            **((int **)output) = file->mpi_rank;
+            break;
 
-    /* Sanity checks */
-    HDassert(file);
-    HDassert(H5FD_MPIO == file->pub.driver_id);
+        case H5FD_CTL__GET_MPI_SIZE_OPCODE:
+            HDassert(output);
+            HDassert(*output);
+            **((int **)output) = file->mpi_size;
+            break;
+
+        default: /* unknown op code */
+            if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG) {
+
+                HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "unknown op_code and fail if unknown")
+            }
+            break;
+    }
+
+done:
+
+    FUNC_LEAVE_NOAPI(ret_value)
 
-    FUNC_LEAVE_NOAPI(file->comm)
-} /* end H5FD__mpio_communicator() */
+} /* end H5FD__mpio_ctl() */
 #endif /* H5_HAVE_PARALLEL */
diff --git a/src/H5FDmulti.c b/src/H5FDmulti.c
index cae4174..0d1967d 100644
--- a/src/H5FDmulti.c
+++ b/src/H5FDmulti.c
@@ -166,6 +166,8 @@ static herr_t  H5FD_multi_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing
 static herr_t  H5FD_multi_lock(H5FD_t *_file, hbool_t rw);
 static herr_t  H5FD_multi_unlock(H5FD_t *_file);
 static herr_t  H5FD_multi_delete(const char *filename, hid_t fapl_id);
+static herr_t  H5FD_multi_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input,
+                              void **output);
 
 /* The class struct */
 static const H5FD_class_t H5FD_multi_g = {
@@ -201,6 +203,7 @@ static const H5FD_class_t H5FD_multi_g = {
     H5FD_multi_lock,           /* lock              */
     H5FD_multi_unlock,         /* unlock            */
     H5FD_multi_delete,         /* del               */
+    H5FD_multi_ctl,            /* ctl               */
     H5FD_FLMAP_DEFAULT         /* fl_map            */
 };
 
@@ -2070,6 +2073,54 @@ H5FD_multi_delete(const char *filename, hid_t fapl_id)
 } /* end H5FD_multi_delete() */
 H5_MULTI_GCC_DIAG_ON("format-nonliteral")
 
+/*-------------------------------------------------------------------------
+ * Function:    H5FD_multi_ctl
+ *
+ * Purpose:     Multi VFD version of the ctl callback.
+ *
+ *              The desired operation is specified by the op_code
+ *              parameter.
+ *
+ *              The flags parameter controls management of op_codes that
+ *              are unknown to the callback
+ *
+ *              The input and output parameters allow op_code specific
+ *              input and output
+ *
+ *              At present, this VFD supports no op codes of its own.
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_multi_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input, void **output)
+{
+    H5FD_multi_t *     file      = (H5FD_multi_t *)_file;
+    static const char *func      = "H5FD_multi_ctl"; /* Function Name for error reporting */
+    herr_t             ret_value = 0;
+
+    /* Silence compiler */
+    (void)file;
+    (void)input;
+    (void)output;
+
+    /* Clear the error stack */
+    H5Eclear2(H5E_DEFAULT);
+
+    switch (op_code) {
+        /* Unknown op code */
+        default:
+            if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG)
+                H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_FCNTL,
+                            "VFD ctl request failed (unknown op code and fail if unknown flag is set)", -1);
+
+            break;
+    }
+
+    return ret_value;
+} /* end H5FD_multi_ctl() */
+
 #ifdef H5private_H
 /*
  * This is not related to the functionality of the driver code.
diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h
index c00c123..6dbd483 100644
--- a/src/H5FDprivate.h
+++ b/src/H5FDprivate.h
@@ -45,13 +45,6 @@
 /* Definitions for file MPI type property */
 #define H5FD_MPI_XFER_FILE_MPI_TYPE_NAME "H5FD_mpi_file_mpi_type"
 
-/* Sub-class the H5FD_class_t to add more specific functions for MPI-based VFDs */
-typedef struct H5FD_class_mpi_t {
-    H5FD_class_t super;                       /* Superclass information & methods */
-    int (*get_rank)(const H5FD_t *file);      /* Get the MPI rank of a process */
-    int (*get_size)(const H5FD_t *file);      /* Get the MPI size of a communicator */
-    MPI_Comm (*get_comm)(const H5FD_t *file); /* Get the communicator for a file */
-} H5FD_class_mpi_t;
 #endif
 
 /****************************/
@@ -137,6 +130,7 @@ H5_DLL herr_t  H5FD_truncate(H5FD_t *file, hbool_t closing);
 H5_DLL herr_t  H5FD_lock(H5FD_t *file, hbool_t rw);
 H5_DLL herr_t  H5FD_unlock(H5FD_t *file);
 H5_DLL herr_t  H5FD_delete(const char *name, hid_t fapl_id);
+H5_DLL herr_t  H5FD_ctl(H5FD_t *file, uint64_t op_code, uint64_t flags, const void *input, void **output);
 H5_DLL herr_t  H5FD_get_fileno(const H5FD_t *file, unsigned long *filenum);
 H5_DLL herr_t  H5FD_get_vfd_handle(H5FD_t *file, hid_t fapl, void **file_handle);
 H5_DLL herr_t  H5FD_set_base_addr(H5FD_t *file, haddr_t base_addr);
@@ -156,9 +150,9 @@ H5_DLL herr_t H5FD_set_mpio_atomicity(H5FD_t *file, hbool_t flag);
 H5_DLL herr_t H5FD_get_mpio_atomicity(H5FD_t *file, hbool_t *flag);
 
 /* Driver specific methods */
-H5_DLL int      H5FD_mpi_get_rank(const H5FD_t *file);
-H5_DLL int      H5FD_mpi_get_size(const H5FD_t *file);
-H5_DLL MPI_Comm H5FD_mpi_get_comm(const H5FD_t *_file);
+H5_DLL int      H5FD_mpi_get_rank(H5FD_t *file);
+H5_DLL int      H5FD_mpi_get_size(H5FD_t *file);
+H5_DLL MPI_Comm H5FD_mpi_get_comm(H5FD_t *file);
 #endif /* H5_HAVE_PARALLEL */
 
 #endif /* H5FDprivate_H */
diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h
index 0cfb072..b8f4f12 100644
--- a/src/H5FDpublic.h
+++ b/src/H5FDpublic.h
@@ -138,6 +138,77 @@
  */
 #define H5FD_FEAT_DEFAULT_VFD_COMPATIBLE 0x00008000
 
+/* ctl function definitions: */
+#define H5FD_CTL_OPC_RESERVED 512 /* Opcodes below this value are reserved for library use */
+#define H5FD_CTL_OPC_EXPER_MIN                                                                               \
+    H5FD_CTL_OPC_RESERVED /* Minimum opcode value available for experimental use                             \
+                           */
+#define H5FD_CTL_OPC_EXPER_MAX                                                                               \
+    (H5FD_CTL_OPC_RESERVED + 511) /* Maximum opcode value available for experimental use */
+
+/* ctl function op codes: */
+#define H5FD_CTL__INVALID_OPCODE              0
+#define H5FD_CTL__TEST_OPCODE                 1
+#define H5FD_CTL__GET_MPI_COMMUNICATOR_OPCODE 2
+#define H5FD_CTL__GET_MPI_RANK_OPCODE         3
+#define H5FD_CTL__GET_MPI_SIZE_OPCODE         4
+
+/* ctl function flags: */
+
+/* Definitions:
+ *
+ * WARNING: While the following definitions of Terminal
+ * and Passthrough VFDs should be workable for now, they
+ * have to be adjusted as our use cases for VFDs expand.
+ *
+ *                                   JRM -- 8/4/21
+ *
+ *
+ * Terminal VFD: Lowest VFD in the VFD stack through
+ * which all VFD calls pass.  Note that this definition
+ * is situational.  For example, the sec2 VFD is typically
+ * terminal.  However, in the context of the family file
+ * driver, it is not -- the family file driver is the
+ * bottom VFD through which all VFD calls pass, and thus
+ * it is terminal.
+ *
+ * Similarly, on the splitter VFD, a sec2 VFD on the
+ * R/W channel is terminal, but a sec2 VFD on the W/O
+ * channel is not.
+ *
+ *
+ * Pass through VFD:  Any VFD that relays all VFD calls
+ * (with the possible exception of some non-I/O related
+ * calls) to underlying VFD(s).
+ */
+
+/* Unknown op codes should be ignored silently unless the
+ * H5FD_CTL__FAIL_IF_UNKNOWN_FLAG is set.
+ *
+ * On terminal VFDs, unknown op codes should generate an
+ * error unconditionally if this flag is set.
+ *
+ * On pass through VFDs, unknown op codes should be routed
+ * to the underlying VFD(s) as indicated by any routing
+ * flags.  In the absence of such flags, the VFD should
+ * generate an error.
+ */
+#define H5FD_CTL__FAIL_IF_UNKNOWN_FLAG 0x0001
+
+/* The H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG is used only
+ * by non-ternminal VFDs, and only applies to unknown
+ * opcodes. (known op codes should be handled as
+ * appropriate.)
+ *
+ * If this flag is set for an uknown op code, that
+ * op code should be passed to the next VFD down
+ * the VFD stack en-route to the terminal VFD.
+ * If that VFD does not support the ctl call, the
+ * pass through VFD should fail or succeed as directed
+ * by the  H5FD_CTL__FAIL_IF_UNKNOWN_FLAG.
+ */
+#define H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG 0x0002
+
 /*******************/
 /* Public Typedefs */
 /*******************/
diff --git a/src/H5FDros3.c b/src/H5FDros3.c
index c0361f9..a32d65e 100644
--- a/src/H5FDros3.c
+++ b/src/H5FDros3.c
@@ -269,6 +269,7 @@ static const H5FD_class_t H5FD_ros3_g = {
     NULL,                     /* lock                 */
     NULL,                     /* unlock               */
     NULL,                     /* del                  */
+    NULL,                     /* ctl                  */
     H5FD_FLMAP_DICHOTOMY      /* fl_map               */
 };
 
diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c
index d823e3c..15103da 100644
--- a/src/H5FDsec2.c
+++ b/src/H5FDsec2.c
@@ -139,6 +139,8 @@ static herr_t  H5FD__sec2_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing
 static herr_t  H5FD__sec2_lock(H5FD_t *_file, hbool_t rw);
 static herr_t  H5FD__sec2_unlock(H5FD_t *_file);
 static herr_t  H5FD__sec2_delete(const char *filename, hid_t fapl_id);
+static herr_t  H5FD__sec2_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input,
+                              void **output);
 
 static const H5FD_class_t H5FD_sec2_g = {
     "sec2",                /* name                 */
@@ -173,6 +175,7 @@ static const H5FD_class_t H5FD_sec2_g = {
     H5FD__sec2_lock,       /* lock                 */
     H5FD__sec2_unlock,     /* unlock               */
     H5FD__sec2_delete,     /* del                  */
+    H5FD__sec2_ctl,        /* ctl                  */
     H5FD_FLMAP_DICHOTOMY   /* fl_map               */
 };
 
@@ -1068,3 +1071,48 @@ H5FD__sec2_delete(const char *filename, hid_t H5_ATTR_UNUSED fapl_id)
 done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5FD__sec2_delete() */
+
+/*-------------------------------------------------------------------------
+ * Function:    H5FD__sec2_ctl
+ *
+ * Purpose:     Sec2 VFD version of the ctl callback.
+ *
+ *              The desired operation is specified by the op_code
+ *              parameter.
+ *
+ *              The flags parameter controls management of op_codes that
+ *              are unknown to the callback
+ *
+ *              The input and output parameters allow op_code specific
+ *              input and output
+ *
+ *              At present, no op codes are supported by this VFD.
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD__sec2_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_ATTR_UNUSED *input,
+               void H5_ATTR_UNUSED **output)
+{
+    H5FD_sec2_t *file      = (H5FD_sec2_t *)_file;
+    herr_t       ret_value = SUCCEED;
+
+    FUNC_ENTER_STATIC
+
+    /* Sanity checks */
+    HDassert(file);
+    HDassert(H5FD_SEC2 == file->pub.driver_id);
+
+    switch (op_code) {
+        /* Unknown op code */
+        default:
+            if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG)
+                HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "unknown op_code and fail if unknown flag is set")
+            break;
+    }
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD__sec2_ctl() */
diff --git a/src/H5FDsplitter.c b/src/H5FDsplitter.c
index 73c898a..3113e8b 100644
--- a/src/H5FDsplitter.c
+++ b/src/H5FDsplitter.c
@@ -129,6 +129,8 @@ static herr_t  H5FD__splitter_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closin
 static herr_t  H5FD__splitter_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing);
 static herr_t  H5FD__splitter_lock(H5FD_t *_file, hbool_t rw);
 static herr_t  H5FD__splitter_unlock(H5FD_t *_file);
+static herr_t  H5FD__splitter_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input,
+                                  void **output);
 
 static const H5FD_class_t H5FD_splitter_g = {
     "splitter",                   /* name                 */
@@ -163,6 +165,7 @@ static const H5FD_class_t H5FD_splitter_g = {
     H5FD__splitter_lock,          /* lock                 */
     H5FD__splitter_unlock,        /* unlock               */
     NULL,                         /* del                  */
+    H5FD__splitter_ctl,           /* ctl                  */
     H5FD_FLMAP_DICHOTOMY          /* fl_map               */
 };
 
@@ -1145,6 +1148,63 @@ done:
 } /* end H5FD__splitter_unlock */
 
 /*-------------------------------------------------------------------------
+ * Function:    H5FD__splitter_ctl
+ *
+ * Purpose:     Splitter VFD version of the ctl callback.
+ *
+ *              The desired operation is specified by the op_code
+ *              parameter.
+ *
+ *              The flags parameter controls management of op_codes that
+ *              are unknown to the callback
+ *
+ *              The input and output parameters allow op_code specific
+ *              input and output
+ *
+ *              At present, this VFD supports no op codes of its own and
+ *              simply passes ctl calls on to the R/W channel VFD.
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD__splitter_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input, void **output)
+{
+    H5FD_splitter_t *file      = (H5FD_splitter_t *)_file;
+    herr_t           ret_value = SUCCEED;
+
+    FUNC_ENTER_STATIC
+
+    /* Sanity checks */
+    HDassert(file);
+    HDassert(H5FD_SPLITTER == file->pub.driver_id);
+
+    switch (op_code) {
+        /* Unknown op code */
+        default:
+            if (flags & H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG) {
+                /* Pass ctl call down to R/W channel VFD */
+                if (H5FDctl(file->rw_file, op_code, flags, input, output) < 0)
+                    HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "VFD ctl request failed")
+            }
+            else {
+                /* If no valid VFD routing flag is specified, fail for unknown op code
+                 * if H5FD_CTL__FAIL_IF_UNKNOWN_FLAG flag is set.
+                 */
+                if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG)
+                    HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL,
+                                "VFD ctl request failed (unknown op code and fail if unknown flag is set)")
+            }
+
+            break;
+    }
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD__splitter_ctl() */
+
+/*-------------------------------------------------------------------------
  * Function:    H5FD__splitter_query
  *
  * Purpose:     Set the flags that this VFL driver is capable of supporting.
diff --git a/src/H5FDstdio.c b/src/H5FDstdio.c
index 6631325..312263c 100644
--- a/src/H5FDstdio.c
+++ b/src/H5FDstdio.c
@@ -215,6 +215,7 @@ static const H5FD_class_t H5FD_stdio_g = {
     H5FD_stdio_lock,       /* lock         */
     H5FD_stdio_unlock,     /* unlock       */
     H5FD_stdio_delete,     /* del          */
+    NULL,                  /* ctl          */
     H5FD_FLMAP_DICHOTOMY   /* fl_map       */
 };