VFD ctl feature (#981)

* Added "ctl" callback to the VFD interface, and the associated
H5FDctl() and H5FD_ctl() calls.

Modified the MPIO VFD accordingly -- specifically:

Added ctl() call with op-code support to expose rank, size,
and communicator.

Modified H5FD_mpi_get_rank(), H5FD_mpi_get_size(),
and H5FD_mpi_get_comm() to use the new ctl() callback.  In passing
removed the const qualifier from the file parameter of these
functions, as the file parameter of the ctl callback is not
const.

Deleted the old  H5FD__mpio_mpi_rank(), H5FD__mpio_mpi_size(), and
H5FD__mpio_communicator() calls from the MPIO VFD.

Deleted H5FD_class_mpi_t from H5FDprivate.h, and modified the
MPIO VFD accordingly.  Note that all VFDs now use H5FD_class_t,
with no special class for VFDs that that support MPI.

Some minor touch ups to the Neil's selection I/O mods in passing.

Tested serial and parallel, debug and production on charis and
jelly.

* Reserve a range of VFD "ctl" opcodes for library and experimental usage

* Add "ctl" callbacks to passthrough VFDs

* Add RELEASE.txt entry for "ctl" callback

* Use H5FDopen with H5F_ACC_RDWR flag instead of H5F_ACC_TRUNC in vfd test

* Remove handling of passthrough "ctl" flag from multi VFD

* Move logic for testing H5FD_CTL__TEST_OPCODE into a testing VFD

Revise description of "ctl" callback in RELEASE.txt

Remove unused H5FD_CTL__NUM_OPCODES definition

Fix some warnings in multi VFD

Co-authored-by: mainzer <mainzer#hdfgroup.org>

1 files changed, 106 insertions, 117 deletions

diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c
index dd40399..7c85897 100644
--- a/src/H5FDmpio.c
+++ b/src/H5FDmpio.c
@@ -72,66 +72,60 @@ typedef struct H5FD_mpio_t {
 /* Private Prototypes */
 
 /* Callbacks */
-static herr_t   H5FD__mpio_term(void);
-static H5FD_t * H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr);
-static herr_t   H5FD__mpio_close(H5FD_t *_file);
-static herr_t   H5FD__mpio_query(const H5FD_t *_f1, unsigned long *flags);
-static haddr_t  H5FD__mpio_get_eoa(const H5FD_t *_file, H5FD_mem_t type);
-static herr_t   H5FD__mpio_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr);
-static haddr_t  H5FD__mpio_get_eof(const H5FD_t *_file, H5FD_mem_t type);
-static herr_t   H5FD__mpio_get_handle(H5FD_t *_file, hid_t fapl, void **file_handle);
-static herr_t   H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size,
-                                void *buf);
-static herr_t   H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size,
-                                 const void *buf);
-static herr_t   H5FD__mpio_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closing);
-static herr_t   H5FD__mpio_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing);
-static herr_t   H5FD__mpio_delete(const char *filename, hid_t fapl_id);
-static int      H5FD__mpio_mpi_rank(const H5FD_t *_file);
-static int      H5FD__mpio_mpi_size(const H5FD_t *_file);
-static MPI_Comm H5FD__mpio_communicator(const H5FD_t *_file);
+static herr_t  H5FD__mpio_term(void);
+static H5FD_t *H5FD__mpio_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr);
+static herr_t  H5FD__mpio_close(H5FD_t *_file);
+static herr_t  H5FD__mpio_query(const H5FD_t *_f1, unsigned long *flags);
+static haddr_t H5FD__mpio_get_eoa(const H5FD_t *_file, H5FD_mem_t type);
+static herr_t  H5FD__mpio_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr);
+static haddr_t H5FD__mpio_get_eof(const H5FD_t *_file, H5FD_mem_t type);
+static herr_t  H5FD__mpio_get_handle(H5FD_t *_file, hid_t fapl, void **file_handle);
+static herr_t  H5FD__mpio_read(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size,
+                               void *buf);
+static herr_t  H5FD__mpio_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size,
+                                const void *buf);
+static herr_t  H5FD__mpio_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closing);
+static herr_t  H5FD__mpio_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing);
+static herr_t  H5FD__mpio_delete(const char *filename, hid_t fapl_id);
+static herr_t  H5FD__mpio_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input,
+                              void **output);
 
 /* The MPIO file driver information */
-static const H5FD_class_mpi_t H5FD_mpio_g = {
-    {
-        /* Start of superclass information */
-        "mpio",                /* name                  */
-        HADDR_MAX,             /* maxaddr               */
-        H5F_CLOSE_SEMI,        /* fc_degree             */
-        H5FD__mpio_term,       /* terminate             */
-        NULL,                  /* sb_size               */
-        NULL,                  /* sb_encode             */
-        NULL,                  /* sb_decode             */
-        0,                     /* fapl_size             */
-        NULL,                  /* fapl_get              */
-        NULL,                  /* fapl_copy             */
-        NULL,                  /* fapl_free             */
-        0,                     /* dxpl_size             */
-        NULL,                  /* dxpl_copy             */
-        NULL,                  /* dxpl_free             */
-        H5FD__mpio_open,       /* open                  */
-        H5FD__mpio_close,      /* close                 */
-        NULL,                  /* cmp                   */
-        H5FD__mpio_query,      /* query                 */
-        NULL,                  /* get_type_map          */
-        NULL,                  /* alloc                 */
-        NULL,                  /* free                  */
-        H5FD__mpio_get_eoa,    /* get_eoa               */
-        H5FD__mpio_set_eoa,    /* set_eoa               */
-        H5FD__mpio_get_eof,    /* get_eof               */
-        H5FD__mpio_get_handle, /* get_handle            */
-        H5FD__mpio_read,       /* read                  */
-        H5FD__mpio_write,      /* write                 */
-        H5FD__mpio_flush,      /* flush                 */
-        H5FD__mpio_truncate,   /* truncate              */
-        NULL,                  /* lock                  */
-        NULL,                  /* unlock                */
-        H5FD__mpio_delete,     /* del                   */
-        H5FD_FLMAP_DICHOTOMY   /* fl_map                */
-    },                         /* End of superclass information */
-    H5FD__mpio_mpi_rank,       /* get_rank              */
-    H5FD__mpio_mpi_size,       /* get_size              */
-    H5FD__mpio_communicator    /* get_comm              */
+static const H5FD_class_t H5FD_mpio_g = {
+    "mpio",                /* name                  */
+    HADDR_MAX,             /* maxaddr               */
+    H5F_CLOSE_SEMI,        /* fc_degree             */
+    H5FD__mpio_term,       /* terminate             */
+    NULL,                  /* sb_size               */
+    NULL,                  /* sb_encode             */
+    NULL,                  /* sb_decode             */
+    0,                     /* fapl_size             */
+    NULL,                  /* fapl_get              */
+    NULL,                  /* fapl_copy             */
+    NULL,                  /* fapl_free             */
+    0,                     /* dxpl_size             */
+    NULL,                  /* dxpl_copy             */
+    NULL,                  /* dxpl_free             */
+    H5FD__mpio_open,       /* open                  */
+    H5FD__mpio_close,      /* close                 */
+    NULL,                  /* cmp                   */
+    H5FD__mpio_query,      /* query                 */
+    NULL,                  /* get_type_map          */
+    NULL,                  /* alloc                 */
+    NULL,                  /* free                  */
+    H5FD__mpio_get_eoa,    /* get_eoa               */
+    H5FD__mpio_set_eoa,    /* set_eoa               */
+    H5FD__mpio_get_eof,    /* get_eof               */
+    H5FD__mpio_get_handle, /* get_handle            */
+    H5FD__mpio_read,       /* read                  */
+    H5FD__mpio_write,      /* write                 */
+    H5FD__mpio_flush,      /* flush                 */
+    H5FD__mpio_truncate,   /* truncate              */
+    NULL,                  /* lock                  */
+    NULL,                  /* unlock                */
+    H5FD__mpio_delete,     /* del                   */
+    H5FD__mpio_ctl,        /* ctl                   */
+    H5FD_FLMAP_DICHOTOMY   /* fl_map                */
 };
 
 #ifdef H5FDmpio_DEBUG
@@ -245,7 +239,7 @@ H5FD_mpio_init(void)
 
     /* Register the MPI-IO VFD, if it isn't already */
     if (H5I_VFL != H5I_get_type(H5FD_MPIO_g))
-        H5FD_MPIO_g = H5FD_register((const H5FD_class_t *)&H5FD_mpio_g, sizeof(H5FD_class_mpi_t), FALSE);
+        H5FD_MPIO_g = H5FD_register((const H5FD_class_t *)&H5FD_mpio_g, sizeof(H5FD_class_t), FALSE);
 
     if (!H5FD_mpio_Debug_inited) {
         const char *s; /* String for environment variables */
@@ -1802,83 +1796,78 @@ done:
 } /* end H5FD__mpio_delete() */
 
 /*-------------------------------------------------------------------------
- * Function:    H5FD__mpio_mpi_rank
+ * Function:    H5FD__mpio_ctl
  *
- * Purpose:     Returns the MPI rank for a process
+ * Purpose:     MPIO version of the ctl callback.
  *
- * Return:      Success: non-negative
- *              Failure: negative
+ *              The desired operation is specified by the op_code
+ *              parameter.
  *
- * Programmer:  Quincey Koziol
- *              Thursday, May 16, 2002
+ *              The flags parameter controls management of op_codes that
+ *              are unknown to the callback
  *
- *-------------------------------------------------------------------------
- */
-static int
-H5FD__mpio_mpi_rank(const H5FD_t *_file)
-{
-    const H5FD_mpio_t *file = (const H5FD_mpio_t *)_file;
-
-    FUNC_ENTER_STATIC_NOERR
-
-    /* Sanity checks */
-    HDassert(file);
-    HDassert(H5FD_MPIO == file->pub.driver_id);
-
-    FUNC_LEAVE_NOAPI(file->mpi_rank)
-} /* end H5FD__mpio_mpi_rank() */
-
-/*-------------------------------------------------------------------------
- * Function:    H5FD__mpio_mpi_size
+ *              The input and output parameters allow op_code specific
+ *              input and output
  *
- * Purpose:     Returns the number of MPI processes
+ *              At present, the supported op codes are:
  *
- * Return:      Success: non-negative
- *              Failure: negative
+ *                  H5FD_CTL__GET_MPI_COMMUNICATOR_OPCODE
+ *                  H5FD_CTL__GET_MPI_RANK_OPCODE
+ *                  H5FD_CTL__GET_MPI_SIZE_OPCODE
  *
- * Programmer:  Quincey Koziol
- *              Thursday, May 16, 2002
+ *              Note that these opcodes must be supported by all VFDs that
+ *              support MPI.
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Programmer:  JRM -- 8/3/21
  *
  *-------------------------------------------------------------------------
  */
-static int
-H5FD__mpio_mpi_size(const H5FD_t *_file)
+static herr_t
+H5FD__mpio_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_ATTR_UNUSED *input,
+               void **output)
 {
-    const H5FD_mpio_t *file = (const H5FD_mpio_t *)_file;
+    H5FD_mpio_t *file      = (H5FD_mpio_t *)_file;
+    herr_t       ret_value = SUCCEED; /* Return value */
 
-    FUNC_ENTER_STATIC_NOERR
+    FUNC_ENTER_NOAPI(FAIL)
 
     /* Sanity checks */
     HDassert(file);
     HDassert(H5FD_MPIO == file->pub.driver_id);
 
-    FUNC_LEAVE_NOAPI(file->mpi_size)
-} /* end H5FD__mpio_mpi_size() */
+    switch (op_code) {
 
-/*-------------------------------------------------------------------------
- * Function:    H5FD__mpio_communicator
- *
- * Purpose:     Returns the MPI communicator for the file.
- *
- * Return:      Success:    The communicator
- *              Failure:    Can't fail
- *
- * Programmer:  Robb Matzke
- *              Monday, August  9, 1999
- *
- *-------------------------------------------------------------------------
- */
-static MPI_Comm
-H5FD__mpio_communicator(const H5FD_t *_file)
-{
-    const H5FD_mpio_t *file = (const H5FD_mpio_t *)_file;
+        case H5FD_CTL__GET_MPI_COMMUNICATOR_OPCODE:
+            HDassert(output);
+            HDassert(*output);
+            **((MPI_Comm **)output) = file->comm;
+            break;
 
-    FUNC_ENTER_STATIC_NOERR
+        case H5FD_CTL__GET_MPI_RANK_OPCODE:
+            HDassert(output);
+            HDassert(*output);
+            **((int **)output) = file->mpi_rank;
+            break;
 
-    /* Sanity checks */
-    HDassert(file);
-    HDassert(H5FD_MPIO == file->pub.driver_id);
+        case H5FD_CTL__GET_MPI_SIZE_OPCODE:
+            HDassert(output);
+            HDassert(*output);
+            **((int **)output) = file->mpi_size;
+            break;
+
+        default: /* unknown op code */
+            if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG) {
+
+                HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "unknown op_code and fail if unknown")
+            }
+            break;
+    }
+
+done:
+
+    FUNC_LEAVE_NOAPI(ret_value)
 
-    FUNC_LEAVE_NOAPI(file->comm)
-} /* end H5FD__mpio_communicator() */
+} /* end H5FD__mpio_ctl() */
 #endif /* H5_HAVE_PARALLEL */