Merge in VFD SWMR changes by John Mainzer.

Commit log message from John:
    Returned VFD SWMR to using the VFD SWMR reader VFD only in the reader case.
    In passing, added a private VFD SWMR reader VFD fapl entry that is pushed
    and popped off the FAPL during file open, and removed the code that set
    the VFD SWMR reader VFD as the driver in the FAPL when VFD SWMR is
    configured.  This was necessary, as there is no mechanism to prevent the
    user from overwriting this entry on the FAPL before file open.  While
    we don't use it now, it also gives us a mechanism for allowing the
    user to specify an underlying VFD for VFD SWMR.

    Modified code to compare file opens to compare the terminal VFDs, not
    the top level VFDs.  Failure to do this allowed multiple opens of
    the same file with the same VFD but with different pass through
    VFDs to appear to be treated as different files -- with the obvious
    file corruption issues.  To support this, added a new VFD ctl op code
    to return a pointer to the instance of H5FD_t associated with the
    terminal VFD.

    Note that this change does not address the case of the same file being
    opened twice with different terminal VFDs -- that will have to be addressed
    another day.

Overview of major changes from John:
1) Reworked file open so that the VFD SWMR reader VFD is only
   used when a file is opened VFD SWMR reader.  This required
   the following changes:
   a) Removed code to set the driver in H5Pset_vfd_swmr_config()
   b) Added a private fapl entry for the VFD SWMR reader VFD
   c) Modified H5F_open to test for VFD SWMR reader opens, and push the
      vfd swmr reader vfd FAPL entry on the VFD stack if so.  In this case
      the entry is popped off the VFD stack on exit so as to avoid any net
      modification from the supplied FAPL.
2) Removed dedup code, and augmented H5FD_cmp() to provide the
   necessary functionality.  This required the following changes:
   a) Added the get terminal VFD op code to the H5FD ctl call.
      This allows duplicate file opens with the same VFD but different overlying
      pass through VFDs to be recognized.  Updated ctl callback in VFDs as required
      to support the new op code.
   b) Modified H5FD_cmp to use the above ctl op code to allow it to recognize
      duplicate file opens with the same VFD but with different overlying
      passthoguh VFDs.  This is necessary to recognize duplicate VFD SWMR reader
      and regular opens.  Note that this does not allow us to recognize duplicate
      opens with different terminal VFDs.

19 files changed, 1156 insertions, 491 deletions

diff --git a/src/H5.c b/src/H5.c
index 8fbd874..29c21e6 100644
--- a/src/H5.c
+++ b/src/H5.c
@@ -463,6 +463,7 @@ H5_term_library(void)
     do {
         pending = 0;
         for (i = 0; i < NELMTS(terminator); i++) {
+
             if (terminator[i].completed)
                 continue;
             if (pending != 0 && terminator[i].await_prior)
diff --git a/src/H5FD.c b/src/H5FD.c
index 0f38848..4488c5c 100644
--- a/src/H5FD.c
+++ b/src/H5FD.c
@@ -51,12 +51,6 @@
 /* Package Typedefs */
 /********************/
 
-/* H5FD wrapper for VFD SWMR. Allows use as a BSD TAILQ element. */
-typedef struct H5FD_wrap_t {
-    TAILQ_ENTRY(H5FD_wrap_t) link; /* Linkage for list of all VFDs. */
-    H5FD_t *file;                  /* Pointer to wrapped VFD struct */
-} H5FD_wrap_t;
-
 /********************/
 /* Local Prototypes */
 /********************/
@@ -89,7 +83,6 @@ static herr_t H5FD__query(const H5FD_t *f, unsigned long *flags /*out*/);
  */
 static unsigned long H5FD_file_serial_no_g;
 
-static TAILQ_HEAD(_all_vfds, H5FD_wrap_t) all_vfds = TAILQ_HEAD_INITIALIZER(all_vfds);
 
 /* File driver ID class */
 static const H5I_class_t H5I_VFL_CLS[1] = {{
@@ -713,103 +706,6 @@ done:
     FUNC_LEAVE_API(ret_value)
 }
 
-/*-------------------------------------------------------------------------
- * Function:    H5FD__dedup
- *
- * Purpose:     Helper routine for H5FD_deduplicate
- *
- *              Compares `self` and `other` using the dedup callback of
- *              `self` (if it has one); otherwise compares using `H5FDcmp()`.
- *
- *      No `dedup' callback:
- *
- *              If `self` has no de-duplication method, compare `self` and
- *              `other` using `H5FDcmp()` and return `self` if they're equal
- *              and `other` if unequal.
- *
- *      `dedup' callback present:
- *
- *              If `self` does have a de-duplication callback, invoke it and
- *              return the method's result: `other` if it duplicates `self`,
- *              `self` if `other` does NOT duplicate it, NULL if `other`
- *              conflicts with `self` or if there is an error.
- *
- * Return:      Success:    `self' or `other', as described above
- *
- *              Failure:    NULL
- *
- * Note:        Unlike H5FD_deduplicate(), this routine does not free `self`
- *              under any circumstances.
- *
- *-------------------------------------------------------------------------
- *
- */
-
-/*-------------------------------------------------------------------------
- * Function:    H5FD_deduplicate
- *
- * Purpose:     Search the already-opened VFD instances for an instance
- *              similar to the instance `file` newly-opened using file access
- *              properties given by `fapl_id`.
- *
- * Return:      It's complicated...
- *
- *              If there is an already-open instance that is functionally
- *              identical to `file`, close `file` and return the already
- *              open instance.
- *
- *              If there is an already open instance that conflicts with
- *              `file` because, for example, its file-access properties are
- *              incompatible with `fapl_id`'s or, for another example, it is
- *              under exclusive control by a third VFD instance, then close
- *              `file` and return `NULL`.
- *
- *              Otherwise, return `file` to indicate that there are no
- *              identical or conflicting VFD instances already open.
- *-------------------------------------------------------------------------
- */
-H5FD_t *
-H5FD_deduplicate(H5FD_t *file, hid_t fapl_id)
-{
-    H5FD_wrap_t *item;
-    H5FD_t *     ret_value = file;
-
-    FUNC_ENTER_NOAPI(NULL)
-
-    TAILQ_FOREACH(item, &all_vfds, link)
-    {
-        /* Skip "self" */
-        if (item->file == file)
-            continue;
-
-        /* Skip files with exclusive owners, for now */
-        if (item->file->exc_owner != NULL)
-            continue;
-
-        if ((ret_value = H5FD_vfd_swmr_dedup(item->file, file, fapl_id)) != file)
-            goto done;
-    }
-
-    /* If we reach this stage, then we identified neither a conflict nor a
-     * duplicate.  If any lower VFD with an exclusive owner matches `file`,
-     * return NULL to indicate the conflict.
-     */
-    TAILQ_FOREACH(item, &all_vfds, link)
-    {
-        if (item->file == file || item->file->exc_owner == NULL)
-            continue;
-
-        if (H5FD_cmp(file, item->file) == 0)
-            HGOTO_ERROR(H5E_VFL, H5E_FILEOPEN, NULL,
-                        "found a conflicting open file when searching for duplicates")
-    }
-
-done:
-    if (ret_value != file && H5FD_close(file) < 0)
-        HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "could not close file")
-
-    FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5FD_deduplicate() */
 
 /*-------------------------------------------------------------------------
  * Function:    H5FD_open
@@ -831,7 +727,6 @@ H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
     H5P_genplist_t *       plist;               /* Property list pointer */
     unsigned long          driver_flags = 0;    /* File-inspecific driver feature flags */
     H5FD_file_image_info_t file_image_info;     /* Initial file image */
-    H5FD_wrap_t *          swmr_wrapper = NULL; /* H5FD wrapper for SWMR queue */
     H5FD_t *               ret_value    = NULL; /* Return value */
 
     FUNC_ENTER_NOAPI(NULL)
@@ -874,8 +769,6 @@ H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
     if (NULL == (file = (driver->open)(name, flags, fapl_id, maxaddr)))
         HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "open failed")
 
-    file->exc_owner = NULL;
-
     /* Set the file access flags */
     file->access_flags = flags;
 
@@ -907,12 +800,6 @@ H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
     /* (This will be changed later, when the superblock is located) */
     file->base_addr = 0;
 
-    /* Create and insert a SWMR wrapper for the file */
-    if (NULL == (swmr_wrapper = H5MM_calloc(sizeof(H5FD_wrap_t))))
-        HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, NULL, "unable to allocate file wrap struct")
-    swmr_wrapper->file = file;
-    TAILQ_INSERT_TAIL(&all_vfds, swmr_wrapper, link);
-
     /* Set return value */
     ret_value = file;
 
@@ -969,8 +856,6 @@ herr_t
 H5FD_close(H5FD_t *file)
 {
     const H5FD_class_t *driver;
-    H5FD_wrap_t *       item;
-    H5FD_wrap_t *       temp      = NULL;
     herr_t              ret_value = SUCCEED;
 
     FUNC_ENTER_NOAPI(FAIL)
@@ -984,18 +869,6 @@ H5FD_close(H5FD_t *file)
     if (H5I_dec_ref(file->driver_id) < 0)
         HGOTO_ERROR(H5E_VFL, H5E_CANTDEC, FAIL, "can't close driver ID")
 
-    TAILQ_FOREACH(item, &all_vfds, link)
-    {
-        if (item->file->exc_owner == file)
-            item->file->exc_owner = NULL;
-        if (item->file == file)
-            temp = item;
-    }
-
-    HDassert(temp);
-    TAILQ_REMOVE(&all_vfds, temp, link);
-    H5MM_xfree(temp);
-
     /* Dispatch to the driver for actual close. If the driver fails to
      * close the file then the file will be in an unusable state.
      */
@@ -1050,42 +923,105 @@ done:
  *
  *              Failure:    Must never fail.
  *
+ * Changes:     Re-worked function to use H5FD_ctl() to obtain the terminal
+ *              VFDs for f1 and f2.  Typically, these are the same thing, 
+ *              however, if there is an intervening pass through VFD
+ *              (i.e. splitter of vfd swrmr reader vfd), using the terminal
+ *              VFD for the comparison will avoid some false negatives.
+ *
+ *              Note, however, that we will still fail to detect the 
+ *              case in which a give file is opened twice with different
+ *              terminal VFDs.  
+ *
+ *                                           JRM -- 5/5/22
+ *
  *-------------------------------------------------------------------------
  */
+
 int
 H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2)
 {
-    int ret_value = -1; /* Return value */
+    const H5FD_t * term_f1 = f1;
+    const H5FD_t * term_f2 = f2;
+    herr_t         ctl_result;
+    int            ret_value = -1; /* Return value */
 
     FUNC_ENTER_NOAPI_NOERR; /* return value is arbitrary */
 
-    if ((!f1 || !f1->cls) && (!f2 || !f2->cls))
+
+    /* For each of f1 and f2, check to see if the ctl call is defined.  If it is,
+     * use the ctl call to try to obtain the terminal VFD.  Since this function
+     * is not allowed to fail, discard the error stack if either of the ctl call 
+     * fail.
+     */
+    if ( ( f1 ) && ( f1->cls ) && ( f1->cls->ctl ) ) {
+
+        H5E_BEGIN_TRY
+        {
+           ctl_result = H5FD_ctl(f1, H5FD_CTL__GET_TERMINAL_VFD, H5FD_CTL__FAIL_IF_UNKNOWN_FLAG | 
+                                 H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG, NULL, (void **)(&term_f1));
+        }
+        H5E_END_TRY;
+
+        /* if the ctl call failed, set term_f1 equal to f1.  This will probably be 
+         * wrong -- but it will be no worse than using the top level VFD unconditionally.
+         */
+        if ( ctl_result != SUCCEED ) {
+
+            H5E_clear_stack(NULL);
+
+            term_f1 = f1;
+        }
+    }
+
+    if ( ( f2 ) && ( f2->cls ) && ( f2->cls->ctl ) ) {
+
+        H5E_BEGIN_TRY
+        {
+           ctl_result = H5FD_ctl(f2, H5FD_CTL__GET_TERMINAL_VFD, H5FD_CTL__FAIL_IF_UNKNOWN_FLAG | 
+                                 H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG, NULL, (void **)(&term_f2));
+        }
+        H5E_END_TRY;
+
+        /* if the ctl call failed, set term_f1 equal to f1.  This will probably be 
+         * wrong -- but it will be no worse than using the top level VFD unconditionally.
+         */
+        if ( ctl_result != SUCCEED ) {
+
+            H5E_clear_stack(NULL);
+
+            term_f2 = f2;
+        }
+    }
+
+    if ((!term_f1 || !term_f1->cls) && (!term_f2 || !term_f2->cls))
         HGOTO_DONE(0)
-    if (!f1 || !f1->cls)
+    if (!term_f1 || !term_f1->cls)
         HGOTO_DONE(-1)
-    if (!f2 || !f2->cls)
+    if (!term_f2 || !term_f2->cls)
         HGOTO_DONE(1)
-    if (f1->cls < f2->cls)
+    if (term_f1->cls < term_f2->cls)
         HGOTO_DONE(-1)
-    if (f1->cls > f2->cls)
+    if (term_f1->cls > term_f2->cls)
         HGOTO_DONE(1)
 
     /* Files are same driver; no cmp callback */
-    if (!f1->cls->cmp) {
-        if (f1 < f2)
+    if (!term_f1->cls->cmp) {
+        if (term_f1 < term_f2)
             HGOTO_DONE(-1)
-        if (f1 > f2)
+        if (term_f1 > term_f2)
             HGOTO_DONE(1)
         HGOTO_DONE(0)
     }
 
     /* Dispatch to driver */
-    ret_value = (f1->cls->cmp)(f1, f2);
+    ret_value = (term_f1->cls->cmp)(term_f1, term_f2);
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5FD_cmp() */
 
+
 /*-------------------------------------------------------------------------
  * Function:    H5FDquery
  *
diff --git a/src/H5FDcore.c b/src/H5FDcore.c
index a207e9e..64088fe 100644
--- a/src/H5FDcore.c
+++ b/src/H5FDcore.c
@@ -149,6 +149,8 @@ static herr_t  H5FD__core_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing
 static herr_t  H5FD__core_lock(H5FD_t *_file, hbool_t rw);
 static herr_t  H5FD__core_unlock(H5FD_t *_file);
 static herr_t  H5FD__core_delete(const char *filename, hid_t fapl_id);
+static herr_t  H5FD__core_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void  *input,
+                              void **output);
 static inline const H5FD_core_fapl_t *H5FD__core_get_default_config(void);
 
 static const H5FD_class_t H5FD_core_g = {
@@ -185,7 +187,7 @@ static const H5FD_class_t H5FD_core_g = {
     H5FD__core_lock,          /* lock                 */
     H5FD__core_unlock,        /* unlock               */
     H5FD__core_delete,        /* del                  */
-    NULL,                     /* ctl                  */
+    H5FD__core_ctl,           /* ctl                  */
     H5FD_FLMAP_DICHOTOMY      /* fl_map               */
 };
 
@@ -1755,3 +1757,60 @@ H5FD__core_delete(const char *filename, hid_t fapl_id)
 done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5FD__core_delete() */
+
+/*-------------------------------------------------------------------------
+ * Function:    H5FD__core_ctl
+ *
+ * Purpose:     Core VFD version of the ctl callback.
+ *
+ *              The desired operation is specified by the op_code
+ *              parameter.
+ *
+ *              The flags parameter controls management of op_codes that
+ *              are unknown to the callback
+ *
+ *              The input and output parameters allow op_code specific
+ *              input and output
+ *
+ *              At present, the only op code supported is
+ *              H5FD_CTL__GET_TERMINAL_VFD, which is used to obtain a 
+ *              pointer to the instance of H5FD_t associated with the 
+ *              terminal VFD.  This allows comparison of files whose 
+ *              terminal VFD may have overlying pass through VFDs.
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Changes:     None.
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD__core_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_ATTR_UNUSED *input,
+               void **output)
+{
+    H5FD_core_t *file      = (H5FD_core_t *)_file;
+    herr_t       ret_value = SUCCEED;
+
+    FUNC_ENTER_PACKAGE
+
+    /* Sanity checks */
+    HDassert(file);
+
+    switch (op_code) {
+
+        case H5FD_CTL__GET_TERMINAL_VFD:
+            HDassert(output);
+            *output = (void *)(file);
+            break;
+
+        /* Unknown op code */
+        default:
+            if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG)
+                HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "unknown op_code and fail if unknown flag is set")
+            break;
+    }
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD__core_ctl() */
+
diff --git a/src/H5FDdevelop.h b/src/H5FDdevelop.h
index 32cd36f..be3b545 100644
--- a/src/H5FDdevelop.h
+++ b/src/H5FDdevelop.h
@@ -217,10 +217,6 @@ struct H5FD_t {
     haddr_t             maxaddr;       /* For this file, overrides class */
     haddr_t             base_addr;     /* Base address for HDF5 data w/in file */
 
-    H5FD_t *exc_owner; /* Pointer to an exclusive owner
-                        * or NULL if none.
-                        */
-
     /* Space allocation management fields */
     hsize_t threshold;  /* Threshold for alignment  */
     hsize_t alignment;  /* Allocation alignment     */
diff --git a/src/H5FDdirect.c b/src/H5FDdirect.c
index a6139da..66843e9 100644
--- a/src/H5FDdirect.c
+++ b/src/H5FDdirect.c
@@ -140,6 +140,8 @@ static herr_t  H5FD__direct_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closi
 static herr_t  H5FD__direct_lock(H5FD_t *_file, hbool_t rw);
 static herr_t  H5FD__direct_unlock(H5FD_t *_file);
 static herr_t  H5FD__direct_delete(const char *filename, hid_t fapl_id);
+static herr_t  H5FD__direct_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input,
+                                void **output);
 
 static const H5FD_class_t H5FD_direct_g = {
     H5FD_DIRECT_VALUE,          /* value                */
@@ -175,7 +177,7 @@ static const H5FD_class_t H5FD_direct_g = {
     H5FD__direct_lock,          /* lock                 */
     H5FD__direct_unlock,        /* unlock               */
     H5FD__direct_delete,        /* del                  */
-    NULL,                       /* ctl                  */
+    H5FD__direct_ctl,           /* ctl                  */
     H5FD_FLMAP_DICHOTOMY        /* fl_map               */
 };
 
@@ -1421,4 +1423,60 @@ done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5FD__direct_delete() */
 
+/*-------------------------------------------------------------------------
+ * Function:    H5FD__direct_ctl
+ *
+ * Purpose:     Direct VFD version of the ctl callback.
+ *
+ *              The desired operation is specified by the op_code
+ *              parameter.
+ *
+ *              The flags parameter controls management of op_codes that
+ *              are unknown to the callback
+ *
+ *              The input and output parameters allow op_code specific
+ *              input and output
+ *
+ *              At present, the only op code supported is
+ *              H5FD_CTL__GET_TERMINAL_VFD, which is used to obtain the
+ *              instance of H5FD_t associated with the terminal
+ *              VFD.  This allows comparison of files whose terminal
+ *              VFD may have overlying pass through VFDs.
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Changes:     None.
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD__direct_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_ATTR_UNUSED *input,
+                 void **output)
+{
+    H5FD_direct_t *file      = (H5FD_direct_t *)_file;
+    herr_t         ret_value = SUCCEED;
+
+    FUNC_ENTER_PACKAGE
+
+    /* Sanity checks */
+    HDassert(file);
+
+    switch (op_code) {
+
+        case H5FD_CTL__GET_TERMINAL_VFD:
+            HDassert(output);
+            *output = (void *)(file);
+            break;
+
+        /* Unknown op code */
+        default:
+            if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG)
+                HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "unknown op_code and fail if unknown flag is set")
+            break;
+    }
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD__direct_ctl() */
+
 #endif /* H5_HAVE_DIRECT */
diff --git a/src/H5FDfamily.c b/src/H5FDfamily.c
index b4cf58e..4619259 100644
--- a/src/H5FDfamily.c
+++ b/src/H5FDfamily.c
@@ -109,6 +109,8 @@ static herr_t  H5FD__family_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closi
 static herr_t  H5FD__family_lock(H5FD_t *_file, hbool_t rw);
 static herr_t  H5FD__family_unlock(H5FD_t *_file);
 static herr_t  H5FD__family_delete(const char *filename, hid_t fapl_id);
+static herr_t  H5FD__family_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input,
+                                void **output);
 
 /* The class struct */
 static const H5FD_class_t H5FD_family_g = {
@@ -145,7 +147,7 @@ static const H5FD_class_t H5FD_family_g = {
     H5FD__family_lock,          /* lock                 */
     H5FD__family_unlock,        /* unlock               */
     H5FD__family_delete,        /* del                  */
-    NULL,                       /* ctl                  */
+    H5FD__family_ctl,           /* ctl                  */
     H5FD_FLMAP_DICHOTOMY        /* fl_map               */
 };
 
@@ -1580,3 +1582,65 @@ done:
 
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5FD__family_delete() */
+
+/*-------------------------------------------------------------------------
+ * Function:    H5FD__family_ctl
+ *
+ * Purpose:     Family VFD version of the ctl callback.
+ *
+ *              The desired operation is specified by the op_code
+ *              parameter.
+ *
+ *              The flags parameter controls management of op_codes that
+ *              are unknown to the callback
+ *
+ *              The input and output parameters allow op_code specific
+ *              input and output
+ *
+ *              At present, the only op code supported is
+ *              H5FD_CTL__GET_TERMINAL_VFD, which is used to obtain the
+ *              instance of H5FD_t associated with the terminal
+ *              VFD.  This allows comparison of files whose terminal
+ *              VFD may have overlying pass through VFDs.
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD__family_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_ATTR_UNUSED *input,
+               void **output)
+{
+    H5FD_family_t *file      = (H5FD_family_t *)_file;
+    herr_t         ret_value = SUCCEED;
+
+    FUNC_ENTER_PACKAGE
+
+    /* Sanity checks */
+    HDassert(file);
+
+    switch (op_code) {
+
+        case H5FD_CTL__GET_TERMINAL_VFD:
+            /* On can argue as to whether the family VFD should be regarded as terminal.
+             * It is treated as such here, as it is the lowest VFD through which all I/O
+             * request pass.  
+             *
+             * For now at least, this works as this is the level at which files are compared.
+             */
+            HDassert(output);
+            *output = (void *)(file);
+            break;
+
+        /* Unknown op code */
+        default:
+            if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG)
+                HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "unknown op_code and fail if unknown flag is set")
+            break;
+    }
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD__sec2_ctl() */
+
+
diff --git a/src/H5FDlog.c b/src/H5FDlog.c
index 1c8ff7d..315e6f8 100644
--- a/src/H5FDlog.c
+++ b/src/H5FDlog.c
@@ -178,6 +178,8 @@ static herr_t  H5FD__log_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing)
 static herr_t  H5FD__log_lock(H5FD_t *_file, hbool_t rw);
 static herr_t  H5FD__log_unlock(H5FD_t *_file);
 static herr_t  H5FD__log_delete(const char *filename, hid_t fapl_id);
+static herr_t  H5FD__log_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input, 
+                             void **output);
 
 static const H5FD_class_t H5FD_log_g = {
     H5FD_LOG_VALUE,          /* value                */
@@ -213,7 +215,7 @@ static const H5FD_class_t H5FD_log_g = {
     H5FD__log_lock,          /* lock                 */
     H5FD__log_unlock,        /* unlock               */
     H5FD__log_delete,        /* del                  */
-    NULL,                    /* ctl                  */
+    H5FD__log_ctl,           /* ctl                  */
     H5FD_FLMAP_DICHOTOMY     /* fl_map               */
 };
 
@@ -1789,3 +1791,61 @@ H5FD__log_delete(const char *filename, hid_t H5_ATTR_UNUSED fapl_id)
 done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5FD__log_delete() */
+
+/*-------------------------------------------------------------------------
+ * Function:    H5FD__log_ctl
+ *
+ * Purpose:     log VFD version of the ctl callback.
+ *
+ *              The desired operation is specified by the op_code
+ *              parameter.
+ *
+ *              The flags parameter controls management of op_codes that
+ *              are unknown to the callback
+ *
+ *              The input and output parameters allow op_code specific
+ *              input and output
+ *
+ *              At present, the only op code supported is
+ *              H5FD_CTL__GET_TERMINAL_VFD, which is used to obtain the
+ *              instance of H5FD_t associated with the terminal
+ *              VFD.  This allows comparison of files whose terminal
+ *              VFD may have overlying pass through VFDs.
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Changes:     Added support for H5FD_CTL__GET_TERMINAL_VFD.
+ *                                            JRM -- 5/4/22
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD__log_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_ATTR_UNUSED *input,
+              void **output)
+{
+    H5FD_log_t *file      = (H5FD_log_t *)_file;
+    herr_t      ret_value = SUCCEED;
+
+    FUNC_ENTER_PACKAGE
+
+    /* Sanity checks */
+    HDassert(file);
+
+    switch (op_code) {
+
+        case H5FD_CTL__GET_TERMINAL_VFD:
+            HDassert(output);
+            *output = (void *)(file);
+            break;
+
+        /* Unknown op code */
+        default:
+            if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG)
+                HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "unknown op_code and fail if unknown flag is set")
+            break;
+    }
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD__log_ctl() */
+
diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c
index 6a168c7..cd59285 100644
--- a/src/H5FDmpio.c
+++ b/src/H5FDmpio.c
@@ -1918,6 +1918,9 @@ done:
  *
  * Programmer:  JRM -- 8/3/21
  *
+ * Changes:     Added support for H5FD_CTL__GET_TERMINAL_VFD.
+ *                                            JRM -- 5/4/22
+ *
  *-------------------------------------------------------------------------
  */
 static herr_t
@@ -1953,6 +1956,11 @@ H5FD__mpio_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_AT
             **((int **)output) = file->mpi_size;
             break;
 
+        case H5FD_CTL__GET_TERMINAL_VFD:
+            HDassert(output);
+            *output = (void *)(file);
+            break;
+
         default: /* unknown op code */
             if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG) {
 
diff --git a/src/H5FDmulti.c b/src/H5FDmulti.c
index d9a6ce9..faf4b87 100644
--- a/src/H5FDmulti.c
+++ b/src/H5FDmulti.c
@@ -2243,6 +2243,9 @@ H5_MULTI_GCC_DIAG_ON("format-nonliteral")
  *
  * Return:      Non-negative on success/Negative on failure
  *
+ * Changes:     Added support for H5FD_CTL__GET_TERMINAL_VFD.
+ *                                            JRM -- 5/4/22
+ *
  *-------------------------------------------------------------------------
  */
 static herr_t
@@ -2261,6 +2264,18 @@ H5FD_multi_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *inpu
     H5Eclear2(H5E_DEFAULT);
 
     switch (op_code) {
+
+        case H5FD_CTL__GET_TERMINAL_VFD:
+            /* One can argue as to whether the multi VFD should be regarded as terminal.
+             * It is treated as such here, as it is the lowest VFD through which all I/O
+             * request pass.
+             *
+             * For now at least, this works as this is the level at which files are compared.
+             */
+            assert(output);
+            *output = (void *)(file);
+            break;
+
         /* Unknown op code */
         default:
             if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG)
diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h
index 9411874..94ab92b 100644
--- a/src/H5FDpublic.h
+++ b/src/H5FDpublic.h
@@ -203,6 +203,7 @@
 #define H5FD_CTL__MEM_ALLOC                   5
 #define H5FD_CTL__MEM_FREE                    6
 #define H5FD_CTL__MEM_COPY                    7
+#define H5FD_CTL__GET_TERMINAL_VFD            8
 
 /* ctl function flags: */
 
diff --git a/src/H5FDros3.c b/src/H5FDros3.c
index 4b15616..3e01e27 100644
--- a/src/H5FDros3.c
+++ b/src/H5FDros3.c
@@ -233,6 +233,8 @@ static herr_t  H5FD__ros3_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, ha
 static herr_t  H5FD__ros3_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, size_t size,
                                 const void *buf);
 static herr_t  H5FD__ros3_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing);
+static herr_t  H5FD__ros3_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void  *input,
+                              void **output);
 
 static herr_t H5FD__ros3_validate_config(const H5FD_ros3_fapl_t *fa);
 
@@ -270,7 +272,7 @@ static const H5FD_class_t H5FD_ros3_g = {
     NULL,                     /* lock                 */
     NULL,                     /* unlock               */
     NULL,                     /* del                  */
-    NULL,                     /* ctl                  */
+    H5FD__ros3_ctl,           /* ctl                  */
     H5FD_FLMAP_DICHOTOMY      /* fl_map               */
 };
 
@@ -1563,4 +1565,60 @@ done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5FD__ros3_truncate() */
 
+/*-------------------------------------------------------------------------
+ * Function:    H5FD__ros3_ctl
+ *
+ * Purpose:     ROS3 VFD version of the ctl callback.
+ *
+ *              The desired operation is specified by the op_code
+ *              parameter.
+ *
+ *              The flags parameter controls management of op_codes that
+ *              are unknown to the callback
+ *
+ *              The input and output parameters allow op_code specific
+ *              input and output
+ *
+ *              At present, the only op code supported is
+ *              H5FD_CTL__GET_TERMINAL_VFD, which is used to obtain the
+ *              instance of H5FD_t associated with the terminal
+ *              VFD.  This allows comparison of files whose terminal
+ *              VFD may have overlying pass through VFDs.
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Changes:     None,
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD__ros3_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_ATTR_UNUSED *input,
+               void **output)
+{
+    H5FD_ros3_t *file      = (H5FD_ros3_t *)_file;
+    herr_t       ret_value = SUCCEED;
+
+    FUNC_ENTER_PACKAGE
+
+    /* Sanity checks */
+    HDassert(file);
+
+    switch (op_code) {
+
+        case H5FD_CTL__GET_TERMINAL_VFD:
+            HDassert(output);
+            *output = (void *)(file);
+            break;
+
+        /* Unknown op code */
+        default:
+            if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG)
+                HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "unknown op_code and fail if unknown flag is set")
+            break;
+    }
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD__ros3_ctl() */
+
 #endif /* H5_HAVE_ROS3_VFD */
diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c
index accacb6..2611506 100644
--- a/src/H5FDsec2.c
+++ b/src/H5FDsec2.c
@@ -1064,15 +1064,20 @@ done:
  *              The input and output parameters allow op_code specific
  *              input and output
  *
- *              At present, no op codes are supported by this VFD.
+ *              At present, the only op code supported is 
+ *              H5FD_CTL__GET_TERMINAL_VFD, which is used in the 
+ *              comparison of files under layers of pass through VFDs.
  *
  * Return:      Non-negative on success/Negative on failure
  *
+ * Changes:     Added support for H5FD_CTL__GET_TERMINAL_VFD.
+ *                                            JRM -- 5/4/22
+ *
  *-------------------------------------------------------------------------
  */
 static herr_t
 H5FD__sec2_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_ATTR_UNUSED *input,
-               void H5_ATTR_UNUSED **output)
+               void **output)
 {
     H5FD_sec2_t *file      = (H5FD_sec2_t *)_file;
     herr_t       ret_value = SUCCEED;
@@ -1083,6 +1088,12 @@ H5FD__sec2_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_AT
     HDassert(file);
 
     switch (op_code) {
+
+        case H5FD_CTL__GET_TERMINAL_VFD:
+            HDassert(output);
+            *output = (void *)(file);
+            break;
+
         /* Unknown op code */
         default:
             if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG)
diff --git a/src/H5FDstdio.c b/src/H5FDstdio.c
index b59aae3..5a1bb4c 100644
--- a/src/H5FDstdio.c
+++ b/src/H5FDstdio.c
@@ -181,6 +181,8 @@ static herr_t  H5FD_stdio_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing
 static herr_t  H5FD_stdio_lock(H5FD_t *_file, hbool_t rw);
 static herr_t  H5FD_stdio_unlock(H5FD_t *_file);
 static herr_t  H5FD_stdio_delete(const char *filename, hid_t fapl_id);
+static herr_t  H5FD__stdio_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input,
+                               void **output);
 
 static const H5FD_class_t H5FD_stdio_g = {
     H5_VFD_STDIO,          /* value        */
@@ -216,7 +218,7 @@ static const H5FD_class_t H5FD_stdio_g = {
     H5FD_stdio_lock,       /* lock         */
     H5FD_stdio_unlock,     /* unlock       */
     H5FD_stdio_delete,     /* del          */
-    NULL,                  /* ctl          */
+    H5FD__stdio_ctl,       /* ctl          */
     H5FD_FLMAP_DICHOTOMY   /* fl_map       */
 };
 
@@ -1233,6 +1235,63 @@ H5FD_stdio_delete(const char *filename, hid_t /*UNUSED*/ fapl_id)
     return 0;
 } /* end H5FD_stdio_delete() */
 
+/*-------------------------------------------------------------------------
+ * Function:    H5FD__stdio_ctl
+ *
+ * Purpose:     Sec2 VFD version of the ctl callback.
+ *
+ *              The desired operation is specified by the op_code
+ *              parameter.
+ *
+ *              The flags parameter controls management of op_codes that
+ *              are unknown to the callback
+ *
+ *              The input and output parameters allow op_code specific
+ *              input and output
+ *
+ *              At present, the only op code supported is
+ *              H5FD_CTL__GET_TERMINAL_VFD, which is used to obtain the
+ *              instance of H5FD_t associated with the terminal
+ *              VFD.  This allows comparison of files whose terminal
+ *              VFD may have overlying pass through VFDs.
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ * Changes:     None.
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD__stdio_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void /* UNUSED */ *input, 
+                void **output)
+{
+    static const char *func = "H5FD__stdio_ctl"; /* Function Name for error reporting    */
+     H5FD_stdio_t *file      = (H5FD_stdio_t *)_file;
+
+    /* Clear the error stack */
+    H5Eclear2(H5E_DEFAULT);
+   
+    /* Quiet compiler */
+    (void)input;
+
+    switch (op_code) {
+
+        case H5FD_CTL__GET_TERMINAL_VFD:
+            assert(output);
+            *output = (void *)(file);
+            break;
+
+        /* Unknown op code */
+        default:
+            if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG)
+                H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_FCNTL, "unknown op_code and fail if unknown flag is set", -1);
+            break;
+    }
+
+    return(0); /* SUCCEED */
+
+} /* end H5FD__stdio_ctl() */
+
 #ifdef H5private_H
 /*
  * This is not related to the functionality of the driver code.
diff --git a/src/H5FDvfd_swmr.c b/src/H5FDvfd_swmr.c
index 892d084..4e0a978 100644
--- a/src/H5FDvfd_swmr.c
+++ b/src/H5FDvfd_swmr.c
@@ -34,6 +34,8 @@ static hid_t H5FD_VFD_SWMR_g = 0;
 typedef struct H5FD_vfd_swmr_t {
     H5FD_t pub; /* public stuff, must be first      */
 
+    H5FD_vfd_swmr_reader_fapl_t fa;      /* driver-specific file access properties */
+
     /* HDF5 file */
     char    hdf5_filename[H5FD_MAX_FILENAME_LEN]; /* Name of the HDF5 file from open */
     H5FD_t *hdf5_file_lf;                         /* Driver info for the HDF5 file */
@@ -56,11 +58,6 @@ typedef struct H5FD_vfd_swmr_t {
     hbool_t               pb_configured; /* Sanity-checking flag set when page buffer is configured */
     H5F_vfd_swmr_config_t config;        /* VFD SWMR configuration */
 
-    /* Flag set if the file is a SWMR writer.
-     * All methods on a write-mode SWMR VFD instance are passed
-     * to the lower VFD instance.
-     */
-    hbool_t writer;
     /*
      * Indicate whether we are in make_believe state or not
      */
@@ -73,6 +70,9 @@ typedef struct H5FD_vfd_swmr_t {
 
 /* Prototypes */
 static herr_t  H5FD__vfd_swmr_term(void);
+static void *  H5FD__vfd_swmr_fapl_get(H5FD_t *_file);
+static void *  H5FD__vfd_swmr_fapl_copy(const void *_old_fa);
+static herr_t  H5FD__vfd_swmr_fapl_free(void *_fapl);
 static H5FD_t *H5FD__vfd_swmr_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr);
 static herr_t  H5FD__vfd_swmr_close(H5FD_t *_file);
 static int     H5FD__vfd_swmr_cmp(const H5FD_t *_f1, const H5FD_t *_f2);
@@ -88,6 +88,8 @@ static herr_t  H5FD__vfd_swmr_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_i
 static herr_t  H5FD__vfd_swmr_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing);
 static herr_t  H5FD__vfd_swmr_lock(H5FD_t *_file, hbool_t rw);
 static herr_t  H5FD__vfd_swmr_unlock(H5FD_t *_file);
+static herr_t  H5FD__vfd_swmr_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, 
+                                  const void *input, void **output);
 
 /* VFD SWMR */
 static htri_t H5FD__vfd_swmr_header_deserialize(H5FD_vfd_swmr_t *, H5FD_vfd_swmr_md_header *);
@@ -96,46 +98,49 @@ static htri_t H5FD__vfd_swmr_index_deserialize(const H5FD_vfd_swmr_t *file, H5FD
 static herr_t H5FD__vfd_swmr_load_hdr_and_idx(H5FD_vfd_swmr_t *, hbool_t);
 
 static const H5FD_class_t H5FD_vfd_swmr_g = {
-    H5FD_VFD_SWMR_VALUE,       /* value                */
-    "vfd_swmr",                /* name                 */
-    MAXADDR,                   /* maxaddr              */
-    H5F_CLOSE_WEAK,            /* fc_degree            */
-    H5FD__vfd_swmr_term,       /* terminate            */
-    NULL,                      /* sb_size              */
-    NULL,                      /* sb_encode            */
-    NULL,                      /* sb_decode            */
-    0,                         /* fapl_size            */
-    NULL,                      /* fapl_get             */
-    NULL,                      /* fapl_copy            */
-    NULL,                      /* fapl_free            */
-    0,                         /* dxpl_size            */
-    NULL,                      /* dxpl_copy            */
-    NULL,                      /* dxpl_free            */
-    H5FD__vfd_swmr_open,       /* open                 */
-    H5FD__vfd_swmr_close,      /* close                */
-    H5FD__vfd_swmr_cmp,        /* cmp                  */
-    H5FD__vfd_swmr_query,      /* query                */
-    NULL,                      /* get_type_map         */
-    NULL,                      /* alloc                */
-    NULL,                      /* free                 */
-    H5FD__vfd_swmr_get_eoa,    /* get_eoa              */
-    H5FD__vfd_swmr_set_eoa,    /* set_eoa              */
-    H5FD__vfd_swmr_get_eof,    /* get_eof              */
-    H5FD__vfd_swmr_get_handle, /* get_handle           */
-    H5FD__vfd_swmr_read,       /* read                 */
-    H5FD__vfd_swmr_write,      /* write                */
-    NULL,                      /* flush                */
-    H5FD__vfd_swmr_truncate,   /* truncate             */
-    H5FD__vfd_swmr_lock,       /* lock                 */
-    H5FD__vfd_swmr_unlock,     /* unlock               */
-    NULL,                      /* del                  */
-    NULL,                      /* ctl                  */
-    H5FD_FLMAP_DICHOTOMY       /* fl_map               */
+    H5FD_VFD_SWMR_VALUE,                  /* value                */
+    "vfd_swmr",                           /* name                 */
+    MAXADDR,                              /* maxaddr              */
+    H5F_CLOSE_WEAK,                       /* fc_degree            */
+    H5FD__vfd_swmr_term,                  /* terminate            */
+    NULL,                                 /* sb_size              */
+    NULL,                                 /* sb_encode            */
+    NULL,                                 /* sb_decode            */
+    sizeof(H5FD_vfd_swmr_reader_fapl_t),  /* fapl_size            */
+    H5FD__vfd_swmr_fapl_get,              /* fapl_get             */
+    H5FD__vfd_swmr_fapl_copy,             /* fapl_copy            */
+    H5FD__vfd_swmr_fapl_free,             /* fapl_free            */
+    0,                                    /* dxpl_size            */
+    NULL,                                 /* dxpl_copy            */
+    NULL,                                 /* dxpl_free            */
+    H5FD__vfd_swmr_open,                  /* open                 */
+    H5FD__vfd_swmr_close,                 /* close                */
+    H5FD__vfd_swmr_cmp,                   /* cmp                  */
+    H5FD__vfd_swmr_query,                 /* query                */
+    NULL,                                 /* get_type_map         */
+    NULL,                                 /* alloc                */
+    NULL,                                 /* free                 */
+    H5FD__vfd_swmr_get_eoa,               /* get_eoa              */
+    H5FD__vfd_swmr_set_eoa,               /* set_eoa              */
+    H5FD__vfd_swmr_get_eof,               /* get_eof              */
+    H5FD__vfd_swmr_get_handle,            /* get_handle           */
+    H5FD__vfd_swmr_read,                  /* read                 */
+    H5FD__vfd_swmr_write,                 /* write                */
+    NULL,                                 /* flush                */
+    H5FD__vfd_swmr_truncate,              /* truncate             */
+    H5FD__vfd_swmr_lock,                  /* lock                 */
+    H5FD__vfd_swmr_unlock,                /* unlock               */
+    NULL,                                 /* del                  */
+    H5FD__vfd_swmr_ctl,                   /* ctl                  */
+    H5FD_FLMAP_DICHOTOMY                  /* fl_map               */
 };
 
 /* Declare a free list to manage the H5FD_vfd_swmr_t struct */
 H5FL_DEFINE_STATIC(H5FD_vfd_swmr_t);
 
+/* Declare a free list to manage the H5FD_vfd_swmr_reader_config_t struct */
+H5FL_DEFINE_STATIC(H5FD_vfd_swmr_reader_fapl_t);
+
 /* Declare a free list to manage the H5FD_vfd_swmr_idx_entry_t sequence information */
 H5FL_SEQ_DEFINE(H5FD_vfd_swmr_idx_entry_t);
 
@@ -186,33 +191,316 @@ H5FD__vfd_swmr_term(void)
     FUNC_LEAVE_NOAPI(SUCCEED)
 } /* end H5FD__vfd_swmr_term() */
 
+
+/*-------------------------------------------------------------------------
+ * Function:    H5FD__vfd_swmr_fapl_get
+ *
+ * Purpose:     Returns a file access property list which indicates how the
+ *              specified file is being accessed. The return list could be
+ *              used to access another file the same way.
+ *
+ * Return:      Success:    Ptr to new file access property list with all
+ *                          members copied from the file struct.
+ *              Failure:    NULL
+ *-------------------------------------------------------------------------
+ */
+static void *
+H5FD__vfd_swmr_fapl_get(H5FD_t *_file)
+{
+    H5FD_vfd_swmr_t *file      = (H5FD_vfd_swmr_t *)_file;
+    void *           ret_value = NULL;
+
+    FUNC_ENTER_PACKAGE_NOERR
+
+    ret_value = H5FD__vfd_swmr_fapl_copy(&(file->fa));
+
+    FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD__vfd_swmr_fapl_get() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:    H5FD__vfd_swmr_fapl_copy
+ *
+ * Purpose:     Copies the file access properties.
+ *
+ * Return:      Success:    Pointer to a new property list info structure.
+ *              Failure:    NULL
+ *-------------------------------------------------------------------------
+ */
+static void *
+H5FD__vfd_swmr_fapl_copy(const void *_old_fa)
+{
+    const H5FD_vfd_swmr_reader_fapl_t *old_fa_ptr = (const H5FD_vfd_swmr_reader_fapl_t *)_old_fa;
+    H5FD_vfd_swmr_reader_fapl_t       *new_fa_ptr = NULL;
+    void *                             ret_value  = NULL;
+
+    FUNC_ENTER_PACKAGE
+
+    HDassert(old_fa_ptr);
+    HDassert(old_fa_ptr->magic == H5FD_VFD_SWMR_READER_MAGIC);
+
+    new_fa_ptr = H5FL_CALLOC(H5FD_vfd_swmr_reader_fapl_t);
+    if (NULL == new_fa_ptr)
+        HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate vfd swmr FAPL")
+
+    H5MM_memcpy(new_fa_ptr, old_fa_ptr, sizeof(H5FD_vfd_swmr_reader_fapl_t));
+
+    ret_value = (void *)new_fa_ptr;
+
+done:
+    if (NULL == ret_value)
+    {
+        if (new_fa_ptr)
+        {
+            new_fa_ptr->magic = 0;
+            new_fa_ptr = H5FL_FREE(H5FD_vfd_swmr_reader_fapl_t, new_fa_ptr);
+        }
+    }
+
+    FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD__vfd_swmr_fapl_copy() */
+
+
+/*--------------------------------------------------------------------------
+ * Function:    H5FD__vfd_swmr_fapl_free
+ *
+ * Purpose:     Releases the file access lists
+ *
+ * Return:      SUCCEED/FAIL
+ *--------------------------------------------------------------------------
+ */
+static herr_t
+H5FD__vfd_swmr_fapl_free(void *_fapl)
+{
+    H5FD_vfd_swmr_reader_fapl_t *fapl      = (H5FD_vfd_swmr_reader_fapl_t *)_fapl;
+    herr_t                       ret_value = SUCCEED;
+
+    FUNC_ENTER_PACKAGE
+
+    /* Check arguments */
+    if ( ( NULL == fapl ) || ( fapl->magic != H5FD_VFD_SWMR_READER_MAGIC ) )
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "NULL or invalid H5FD_vfd_swmr_reader_fapl_t *")
+
+    /* Free the property list */
+    fapl->magic = 0;
+    fapl = H5FL_FREE(H5FD_vfd_swmr_reader_fapl_t, fapl);
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD__vfd_swmr_fapl_free() */
+
+
 /*-------------------------------------------------------------------------
- * Function:    H5Pset_fapl_vfd_swmr (Not yet)
+ * Function:    H5P_pop_vfd_swmr_reader_vfd_off_fapl
+ *
+ * Purpose:     After a file has been opened in VFD SWMR reader mode, we 
+ *              must pop the vfd swmr reader driver entry off the supplied 
+ *              fapl.  If we don't, and the fapl is used to open a second 
+ *              file (i.e. via virtual data sets), we would have multiple
+ *              vfd swmr reader driver entries pushed on the vfd stack.
  *
- * Purpose:     Modify the file access property list to use the H5FD_SWMR
- *              driver
+ *              Do this as follows:
+ *
+ *              1) Read the file driver entry from the supplied fapl.  Verify
+ *                 that it specifies the vfd swmr reader VFD.
+ *
+ *              2) Read the file driver entry from the sub fapl specified 
+ *                 in the vfd swmr reader vfd fapl entry.  Set the file 
+ *                 driver entry on the supplied fapl equal to that on 
+ *                 the sub-fapl.
+ *
+ *              3) Discard the sub-fapl?  Not sure if this is necessary.
  *
  * Return:      SUCCEED/FAIL
  *
+ * Programmer   JRM -- 4/28/22
+ *
  *-------------------------------------------------------------------------
  */
 herr_t
-H5Pset_fapl_vfd_swmr(hid_t fapl_id)
+H5P_pop_vfd_swmr_reader_vfd_off_fapl(hid_t fapl_id)
 {
-    H5P_genplist_t *plist; /* Property list pointer */
-    herr_t          ret_value;
+    H5FD_driver_prop_t             driver_prop;         /* Property for driver ID & info */
+    H5P_genplist_t *               plist_ptr = NULL;
+    hid_t                          sub_fapl_id;
+    H5P_genplist_t *               sub_plist_ptr = NULL;
+    herr_t                         ret_value = SUCCEED;
 
-    FUNC_ENTER_API(FAIL)
-    H5TRACE1("e", "i", fapl_id);
+    FUNC_ENTER_NOAPI(FAIL)
 
-    if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+    /* sanity checks -- get ptr to plist in passing */
+    if (NULL == (plist_ptr = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list")
 
-    ret_value = H5P_set_driver(plist, H5FD_VFD_SWMR, NULL, NULL);
+
+    /* 1) Read the file driver entry from the supplied fapl.  Verify
+     *    that it specifies the vfd swmr reader VFD.
+     */
+
+    /* get the driver property from the supplied fapl */
+    if (H5P_peek(plist_ptr, H5F_ACS_FILE_DRV_NAME, &driver_prop) < 0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file driver ID & info")
+
+    /* verify that it specifies the vfd swrm reader vfd */
+    if ( driver_prop.driver_id != H5FD_VFD_SWMR )
+        HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "fapl driver prop not vfd swmr reader")
+
+    if ( ( ((const H5FD_vfd_swmr_reader_fapl_t *)(driver_prop.driver_info)) == NULL ) ||
+         ( ((const H5FD_vfd_swmr_reader_fapl_t *)(driver_prop.driver_info))->magic != 
+           H5FD_VFD_SWMR_READER_MAGIC ) )
+        HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "fapl driver info invalid")
+
+    sub_fapl_id = ((const H5FD_vfd_swmr_reader_fapl_t *)(driver_prop.driver_info))->fapl_id;
+
+
+    /* 2) Read the file driver entry from the sub fapl specified
+     *    in the vfd swmr reader vfd fapl entry.  Set the file
+     *    driver entry on the supplied fapl equal to that on
+     *    the sub-fapl.
+     */
+    /* get a pointer to the sub-fapl */
+    if (NULL == (sub_plist_ptr = (H5P_genplist_t *)H5I_object(sub_fapl_id)))
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get sub-fapl pointer")
+
+    /* get the driver property from the sub-fapl */
+    if (H5P_peek(sub_plist_ptr, H5F_ACS_FILE_DRV_NAME, &driver_prop) < 0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get sub-fapl file driver ID & info")
+
+    /* insert the driver info from the sub-fapl into the supplied fapl.  There is 
+     * some question in my mind as to whether I should be making a copy of the 
+     * info and string obtained above.  While I don't think it is necessary,
+     * if we get occult failures, this is a good place to look.
+     *
+     * Note that for now, the driver info on the sub-fapl should only specify the 
+     * sec2 VFD -- which has NULL info and config string.  Thus, if it is an 
+     * issue, it may not appear immediately.
+     */
+    if ( H5P_set_driver(plist_ptr, driver_prop.driver_id, 
+                        driver_prop.driver_info, driver_prop.driver_config_str) < 0 ) 
+        HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't set driver on supplied fapl")
+
+
+    /* 3) Discard the sub-fapl?  Not sure if this is necessary.  Will wait on this for now. */
 
 done:
-    FUNC_LEAVE_API(ret_value)
-} /* end H5Pset_fapl_vfd_swmr() */
+
+    FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5P_pop_vfd_swmr_reader_vfd_off_fapl() */
+
+
+/*-------------------------------------------------------------------------
+ * Function:    H5P_push_vfd_swmr_reader_vfd_on_fapl
+ *
+ * Purpose:     When a file is opened in VFD SWMR reader mode, we must 
+ *              adjust the fapl so as to push the VFD SWMR reader vfd on 
+ *              the VFD stack specified in the fapl.  
+ *
+ *              Do this as follows:
+ *
+ *              1) Copy the file driver from the supplied fapl.  Note
+ *                 that due to potential VFD stacking, we can't verify 
+ *                 that this VFD supports vfd swmr.  This will have to 
+ *                 wait until after the file is opened.
+ *
+ *              2) Create a new FAPL, and set the file driver optained 
+ *                 in 1) in the new FAPL.
+ *
+ *              3) Allocate a new instance of H5FD_vfd_swmr_reader_config_t,
+ *                 load it with the ID of the FAPL created in 2, and use
+ *                 it to overwrite the file driver entry in the supplied
+ *                 FAPL.
+ *
+ * Return:      SUCCEED/FAIL
+ *
+ * Programmer   JRM -- 4/28/22
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5P_push_vfd_swmr_reader_vfd_on_fapl(hid_t fapl_id)
+{
+    H5FD_driver_prop_t             driver_prop;         /* Property for driver ID & info */
+    H5FD_vfd_swmr_reader_fapl_t *  info      = NULL;
+    H5P_genplist_t *               plist_ptr = NULL;
+    hid_t                          sub_fapl_id;
+    H5P_genplist_t *               sub_plist_ptr = NULL;
+    H5P_genclass_t *               pclass = NULL; 
+    herr_t                         ret_value = SUCCEED;
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    /* sanity checks -- get ptr to plist in passing */
+    if (NULL == (plist_ptr = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+        HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list")
+
+
+    /* 1) Copy the file driver from the supplied fapl.  Note
+     *    that due to potential VFD stacking, we can't verify
+     *    that this VFD supports vfd swmr.  This will have to
+     *    wait until after the file is opened.
+     */
+
+    /* get the driver property from the supplied fapl */
+    if (H5P_peek(plist_ptr, H5F_ACS_FILE_DRV_NAME, &driver_prop) < 0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get init driver ID & info")
+
+
+    /* 2) Create a new FAPL, and set the file driver optained in 1) in the new FAPL. */
+
+    /* create a new FAPL */
+    if (NULL == (pclass = (H5P_genclass_t *)H5I_object_verify(H5P_FILE_ACCESS, H5I_GENPROP_CLS)))
+        HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, H5I_INVALID_HID, "not a property list class");
+
+    if ((sub_fapl_id = H5P_create_id(pclass, TRUE)) < 0)
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTCREATE, H5I_INVALID_HID, "unable to create fapl");
+
+    /* get a pointer to it */
+    if (NULL == (sub_plist_ptr = (H5P_genplist_t *)H5I_object(sub_fapl_id)))
+        HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get sub-fapl pointer")
+
+    /* insert the driver info from the base fapl into the sub-fapl.  There is 
+     * some question in my mind as to whether I should be making a copy of the 
+     * info and string obtained above.  While I don't think it is necessary,
+     * if we get occult failures, this is a good place to look.
+     *
+     * Note that for now, the driver info being inserted in sub-fapl should only specify 
+     * the sec2 VFD -- which has NULL info and config string.  Thus, if it is an 
+     * issue, it may not appear immediately.
+     */
+    if ( H5P_set_driver(sub_plist_ptr, driver_prop.driver_id, 
+                        driver_prop.driver_info, driver_prop.driver_config_str) < 0 ) 
+        HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "can't set driver on sub-fapl")
+
+
+    /* 3) Allocate a new instance of H5FD_vfd_swmr_reader_config_t,
+     *    load it with the ID of the FAPL created in 2, and use
+     *    it to overwrite the file driver entry in the supplied
+     *    FAPL.
+     */
+
+    info = H5FL_CALLOC(H5FD_vfd_swmr_reader_fapl_t);
+    if (NULL == info)
+        HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate vfd swmr fapl struct")
+
+    /* initialize the vfd swmr reader vfd info */
+    info->magic   = H5FD_VFD_SWMR_READER_MAGIC;
+    info->fapl_id = sub_fapl_id;
+
+    /* set the driver on the main fapl */
+    ret_value = H5P_set_driver(plist_ptr, H5FD_VFD_SWMR, info, NULL);
+
+done:
+    if (info)
+        info = H5FL_FREE(H5FD_vfd_swmr_reader_fapl_t, info);
+
+    FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5P_push_vfd_swmr_reader_vfd_on_fapl() */
+
 
 /*-------------------------------------------------------------------------
  * Function:    H5FD__swmr_reader_open
@@ -271,60 +559,6 @@ done:
 } /* end H5FD__swmr_reader_open() */
 
 /*-------------------------------------------------------------------------
- *
- * Function:    H5FD__vfd_swmr_build_md_path_name
- *
- * Purpose:     To construct the metadata file's full name based on config's
- *              md_file_path and md_file_name.  See RFC for details.
- *
- *
- * Return:      Success:        SUCCEED
- *              Failure:        FAIL
- *
- * Programmer:  Vailin Choi -- 1/13/2022
- *
- *-------------------------------------------------------------------------
- */
-static herr_t
-H5FD__vfd_swmr_build_md_path_name(H5F_vfd_swmr_config_t *config, const char *hdf5_filename,
-                                  char *name /*out*/)
-{
-    size_t tot_len   = 0;
-    size_t tmp_len   = 0;
-    herr_t ret_value = SUCCEED; /* Return value */
-
-    FUNC_ENTER_PACKAGE
-
-    if ((tot_len = HDstrlen(config->md_file_path)) != 0) {
-
-        /* md_file_path + '/' */
-        if (++tot_len > H5F__MAX_VFD_SWMR_FILE_NAME_LEN)
-            HGOTO_ERROR(H5E_FILE, H5E_CANTCOPY, FAIL, "md_file_path and md_file_name exceeds maximum");
-        HDstrcat(name, config->md_file_path);
-        HDstrcat(name, "/");
-    }
-
-    if ((tmp_len = HDstrlen(config->md_file_name)) != 0) {
-        if ((tot_len += tmp_len) > H5F__MAX_VFD_SWMR_FILE_NAME_LEN)
-            HGOTO_ERROR(H5E_FILE, H5E_CANTCOPY, FAIL, "md_file_path and md_file_name exceeds maximum");
-        HDstrcat(name, config->md_file_name);
-    }
-    else {
-        /* Automatic generation of metadata file name based on hdf5_filename + '.md' */
-        if ((tot_len += (HDstrlen(hdf5_filename) + 3)) > H5F__MAX_VFD_SWMR_FILE_NAME_LEN)
-            HGOTO_ERROR(H5E_FILE, H5E_CANTCOPY, FAIL, "md_file_path and md_file_name maximum");
-
-        HDstrcat(name, hdf5_filename);
-        HDstrcat(name, VFD_SWMR_MD_SUFFIX);
-    }
-
-done:
-
-    FUNC_LEAVE_NOAPI(ret_value)
-
-} /* H5FD__vfd_swmr_build_md_path_name() */
-
-/*-------------------------------------------------------------------------
  * Function:    H5FD__vfd_swmr_create_make_believe_data
  *
  * Purpose:     Set up pretend data when make_believe is true
@@ -377,15 +611,19 @@ H5FD__vfd_swmr_create_make_believe_data(H5FD_vfd_swmr_t *_file)
 static H5FD_t *
 H5FD__vfd_swmr_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
 {
-    H5FD_vfd_swmr_t *      file = NULL;
-    size_t                 page_buf_size;
-    H5P_genplist_t *       plist;
-    H5F_vfd_swmr_config_t *vfd_swmr_config;
-    H5FD_t *               ret_value = NULL; /* Return value     */
-    htri_t                 is_hdf5;
+    H5FD_vfd_swmr_t *                   file = NULL;
+    size_t                              page_buf_size;
+    H5P_genplist_t *                    plist;
+    H5F_vfd_swmr_config_t *             vfd_swmr_config;
+    const H5FD_vfd_swmr_reader_fapl_t * fa_ptr    = NULL;
+    H5FD_t *                            ret_value = NULL; /* Return value     */
+    htri_t                              is_hdf5;
 
     FUNC_ENTER_PACKAGE
 
+    /* VFD SWMR reader VFD should only be called to open a file read only */
+    HDassert((H5F_ACC_RDWR & flags) == 0);
+
     /* Get file access property list */
     if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list")
@@ -404,12 +642,35 @@ H5FD__vfd_swmr_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t max
     if (NULL == (file = H5FL_CALLOC(H5FD_vfd_swmr_t)))
         HGOTO_ERROR(H5E_VFL, H5E_NOSPACE, NULL, "unable to allocate file struct")
 
+
+    /* get the vfd swrm reader fapl entry. */
+    if (H5P_peek_driver(plist) != H5FD_VFD_SWMR)
+        HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "incorrect VFL driver");
+
+    fa_ptr = (const H5FD_vfd_swmr_reader_fapl_t *)H5P_peek_driver_info(plist);
+    if (NULL == fa_ptr)
+        HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "bad VFL driver info");
+
+    HDassert(fa_ptr->magic == H5FD_VFD_SWMR_READER_MAGIC); /* sanity check */
+  
+    /* the fapl id stored in fa_ptr->fapl_id should contain a driver entry that 
+     * specifies a VFD that supports VFD SWMR.  Since there may be a stack of 
+     * VFDs, we can't check this until after file open.  Further, the vfd swmr
+     * reader vfd is currenty hard coded to use the sec2 vfd as its underlying 
+     * vfd.  Thus we just save a copy of the H5FD_vfd_swmr_reader_fapl_t for 
+     * now.
+     */
+    H5MM_memcpy(&(file->fa), fa_ptr, sizeof(H5FD_vfd_swmr_reader_fapl_t));
+    
+
     vfd_swmr_config = &file->config;
 
     /* Get VFD SWMR configuration */
     if (H5P_get(plist, H5F_ACS_VFD_SWMR_CONFIG_NAME, vfd_swmr_config) < 0)
         HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get VFD SWMR config info")
 
+    HDassert(!vfd_swmr_config->writer);
+
     file->md_fd             = -1;
     file->hdf5_file_lf      = NULL;
     file->md_pages_reserved = vfd_swmr_config->md_pages_reserved;
@@ -419,13 +680,11 @@ H5FD__vfd_swmr_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t max
     file->hdf5_filename[sizeof(file->hdf5_filename) - 1] = '\0';
 
     /* Retain a copy of the metadata file name */
-    if (H5FD__vfd_swmr_build_md_path_name(vfd_swmr_config, name, file->md_file_path_name) < 0)
+    if (H5F_vfd_swmr_build_md_path_name(vfd_swmr_config, name, file->md_file_path_name) < 0)
         HGOTO_ERROR(H5E_VFL, H5E_OPENERROR, NULL, "building md_file_path and md_file_name failed")
 
     file->md_file_path_name[sizeof(file->md_file_path_name) - 1] = '\0';
 
-    file->writer = vfd_swmr_config->writer;
-
     /* Make sure the hdf5 file exists and is valid */
     is_hdf5 = H5F__is_hdf5(name, H5P_FILE_ACCESS_DEFAULT);
 
@@ -448,7 +707,6 @@ H5FD__vfd_swmr_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t max
     if ((file->hdf5_file_lf = H5FD_open(name, flags, H5P_FILE_ACCESS_DEFAULT, maxaddr)) == NULL)
         HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "can't set driver info");
 
-    file->hdf5_file_lf->exc_owner = &file->pub;
 
     /* set pb_configured to FALSE.  This field should not exist, but
      * until we modify the file open procedure to create the page buffer
@@ -473,39 +731,6 @@ done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5FD__vfd_swmr_open() */
 
-/*-------------------------------------------------------------------------
- * Function:    H5FD__swmr_reader_close
- *
- * Purpose:     Perform the reader-only aspects of closing in VFD SWMR mode:
- *              optionally log and always release the histogram of ticks spent
- *              in API calls, close the shadow file, release the shadow index.
- *
- * Return:      void
- *
- *-------------------------------------------------------------------------
- */
-static herr_t
-H5FD__swmr_reader_close(H5FD_vfd_swmr_t *file)
-{
-    herr_t ret_value = SUCCEED; /* Return value */
-
-    FUNC_ENTER_PACKAGE
-
-    if (file->api_elapsed_ticks != NULL)
-        H5MM_xfree(file->api_elapsed_ticks);
-
-    /* Close the metadata file */
-    if (file->md_fd >= 0 && HDclose(file->md_fd) < 0)
-        /* Push error, but keep going */
-        HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close the metadata file");
-
-    /* Free the index entries */
-    if (file->md_index.num_entries && file->md_index.entries)
-        file->md_index.entries = H5FL_SEQ_FREE(H5FD_vfd_swmr_idx_entry_t, file->md_index.entries);
-
-    FUNC_LEAVE_NOAPI(ret_value)
-
-} /* end H5FD__swmr_reader_close() */
 
 /*-------------------------------------------------------------------------
  * Function:    H5FD__vfd_swmr_close
@@ -529,10 +754,6 @@ H5FD__vfd_swmr_close(H5FD_t *_file)
     FUNC_ENTER_PACKAGE
 
     if (file->hdf5_file_lf != NULL) {
-        if (file->hdf5_file_lf->exc_owner != NULL) {
-            HDassert(file->hdf5_file_lf->exc_owner == &file->pub);
-            file->hdf5_file_lf->exc_owner = NULL;
-        }
 
         /* Close the underlying file */
         if (H5FD_close(file->hdf5_file_lf) < 0)
@@ -540,8 +761,17 @@ H5FD__vfd_swmr_close(H5FD_t *_file)
             HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close the HDF5 file")
     }
 
-    if (!file->writer)
-        H5FD__swmr_reader_close(file);
+    if (file->api_elapsed_ticks != NULL)
+        H5MM_xfree(file->api_elapsed_ticks);
+
+    /* Close the metadata file */
+    if (file->md_fd >= 0 && HDclose(file->md_fd) < 0)
+        /* Push error, but keep going */
+        HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close the metadata file");
+
+    /* Free the index entries */
+    if (file->md_index.num_entries && file->md_index.entries)
+        file->md_index.entries = H5FL_SEQ_FREE(H5FD_vfd_swmr_idx_entry_t, file->md_index.entries);
 
     /* Release the driver info */
     file = H5FL_FREE(H5FD_vfd_swmr_t, file);
@@ -576,140 +806,6 @@ H5FD__vfd_swmr_cmp(const H5FD_t *_f1, const H5FD_t *_f2)
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5FD__vfd__swmr_cmp() */
 
-/*-------------------------------------------------------------------------
- * Function:    H5FD_vfd_swmr_dedup (original description with H5P_FILE_ACCESS_ANY_VFD)
- *
- * Purpose:     Compare the already-opened VFD instance `_self` with the
- *              VFD instance `_other` newly-opened with file-access properties
- *              `fapl_id` and indicate whether the instances duplicate each
- *              other, if they conflict with each other, or if they are
- *              dissimilar.
- *
- *              If `_self` duplicates `_other`, return `_self`.
- *
- *              Return NULL on error, or if `_other` and `_self` refer to the
- *              same file but the file-access properties, `fapl_id`, conflict
- *              with the properties of `_self`.
- *
- *              If `_other` neither duplicates nor conflicts with `_self`,
- *              then return `_other`.
- *
- * NOTE:        Judging duplicate/conflicting/dissimilar VFD instances
- *
- *              `_self` DUPLICATES `_other` if `_other` is also an instance
- *              of SWMR class, the instances' lower files are equal under
- *              `H5FD_cmp()`, and the file-access properties of `_self` match
- *              `fapl_id`. The wildcard `fapl_id` value,
- *              `H5P_FILE_ACCESS_ANY_VFD`, matches all.
- *
- *              `_self` also DUPLICATES `_other` if `_other` is not a SWMR
- *              instance, but it equals the lower file of `_self` under
- *              `H5FD_cmp()`, and `fapl_id` is `H5P_FILE_ACCESS_ANY_VFD`.
- *
- *              `_self` and `_other` CONFLICT if both are SWMR instances
- *              referring to the same lower file, and their file-access
- *              properties differ.
- *
- *              `_self` and `_other` CONFLICT if `_other` is not a SWMR
- *              instance, it equals the lower file of `_self`, and `fapl_id`
- *              is not equal to `H5P_FILE_ACCESS_ANY_VFD`.
- *
- * Return:      Success:    `_self' or `_other', as described above
- *              Failure:    NULL
- *-------------------------------------------------------------------------
- */
-/*-------------------------------------------------------------------------
- * Function:    H5FD_vfd_swmr_dedup (modified version without H5P_FILE_ACCESS_ANY_VFD)
- *
- * Purpose:     Compare the already-opened VFD instance `_self` with the
- *              VFD instance `_other` newly-opened with file-access properties
- *              `fapl_id` and indicate whether the instances duplicate each
- *              other, if they conflict with each other, or if they are
- *              dissimilar.
- *
- *              If `_self` duplicates `_other`, return `_self`.
- *
- *              Return NULL on error, or if `_other` and `_self` refer to the
- *              same file but the file-access properties, `fapl_id`, conflict
- *              with the properties of `_self`.
- *
- *              If `_other` neither duplicates nor conflicts with `_self`,
- *              then return `_other`.
- *
- * NOTE:        Judging duplicate/conflicting/dissimilar VFD instances
- *
- *              `_self` DUPLICATES `_other` if `_other` is also an instance
- *              of SWMR class, the instances' lower files are equal under
- *              `H5FD_cmp()`, and the file-access properties of `_self` match
- *              `fapl_id`.
- *
- *              `_self` and `_other` CONFLICT if both are SWMR instances
- *              referring to the same lower file, and their file-access
- *              properties differ.
- *
- *              `_self` and `_other` CONFLICT if `_other` is not a SWMR
- *              instance, it equals the lower file of `_self`.
- *
- * Return:      Success:    `_self' or `_other', as described above
- *              Failure:    NULL
- *-------------------------------------------------------------------------
- */
-H5FD_t *
-H5FD_vfd_swmr_dedup(H5FD_t *_self, H5FD_t *_other, hid_t fapl_id)
-{
-    H5FD_vfd_swmr_t *self      = (H5FD_vfd_swmr_t *)_self;
-    H5FD_t *         ret_value = NULL;
-
-    FUNC_ENTER_NOAPI(NULL);
-
-    /* Not VFD SWMR */
-    if (_self->driver_id != H5FD_VFD_SWMR_g) {
-        if (H5FD_cmp(_self, _other) == 0)
-            HGOTO_DONE(_self)
-        else
-            HGOTO_DONE(_other)
-    }
-
-    /* VFD SWMR */
-    if (_self->cls == _other->cls) {
-        H5FD_vfd_swmr_t *      other = (H5FD_vfd_swmr_t *)_other;
-        H5P_genplist_t *       plist;
-        H5F_vfd_swmr_config_t *config;
-        hbool_t                equal_configs;
-
-        if (H5FD_cmp(self->hdf5_file_lf, other->hdf5_file_lf) != 0)
-            HGOTO_DONE(_other)
-
-        /* If fapl_id != _ANY_VFD, then we have either a duplicate or
-         * a conflict.  If the VFD SWMR parameters match, then
-         * return `self` to indicate a duplicate.  Otherwise, return
-         * NULL to indicate a mismatch.
-         */
-        if (NULL == (plist = H5I_object(fapl_id)))
-            HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "could not get fapl")
-
-        if ((config = H5MM_malloc(sizeof(*config))) == NULL)
-            HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, NULL, "could not allocate config")
-        if (H5P_get(plist, H5F_ACS_VFD_SWMR_CONFIG_NAME, config) < 0)
-            HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "cannot get VFD SWMR config")
-
-        equal_configs = HDmemcmp(&self->config, config, sizeof(*config)) == 0;
-
-        H5MM_xfree(config);
-
-        if (equal_configs)
-            HGOTO_DONE(_self)
-
-        HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "inconsistent VFD SWMR config")
-    }
-    else if (H5FD_cmp(self->hdf5_file_lf, _other) == 0)
-        ret_value = NULL;
-    else
-        ret_value = _other;
-
-done:
-    FUNC_LEAVE_NOAPI(ret_value);
-} /* end H5FD_vfd_swmr_dedup() */
 
 /*-------------------------------------------------------------------------
  * Function:    H5FD__vfd_swmr_query
@@ -924,9 +1020,6 @@ H5FD__vfd_swmr_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id
     herr_t                     ret_value = SUCCEED;
     char *                     p         = buf;
 
-    if (file->writer)
-        return H5FD_read(file->hdf5_file_lf, type, addr, size, buf);
-
     FUNC_ENTER_PACKAGE
 
     HDassert(file && file->pub.cls);
@@ -1024,55 +1117,52 @@ done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5FD__vfd_swmr_read() */
 
+
 /*-------------------------------------------------------------------------
  * Function:    H5FD__vfd_swmr_write
  *
- * Purpose:     Writes SIZE bytes of data to FILE beginning at address ADDR
- *              from buffer BUF according to data transfer properties in
- *              DXPL_ID.
+ * Purpose:     As the VFD SWMR reader VFD is only use on files that are
+ *              opened read only, this function should be unreachable.
  *
- * Return:      SUCCEED/FAIL
+ * Return:      FAIL
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5FD__vfd_swmr_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, size_t size,
-                     const void *buf)
+H5FD__vfd_swmr_write(H5FD_t H5_ATTR_UNUSED * _file, H5FD_mem_t H5_ATTR_UNUSED type, 
+                     hid_t H5_ATTR_UNUSED dxpl_id, haddr_t H5_ATTR_UNUSED addr, size_t H5_ATTR_UNUSED size,
+                     const void H5_ATTR_UNUSED * buf)
 {
-    H5FD_vfd_swmr_t *file = (H5FD_vfd_swmr_t *)_file;
+    FUNC_ENTER_PACKAGE_NOERR /* Yes, even though this pushes an error on the stack */
 
-    /* This routine should only be called if the VFD instance is opened
-     * for writing.
-     */
-    HDassert(file->writer);
+        HERROR(H5E_CACHE, H5E_SYSTEM, "called unreachable fcn.");
+
+    FUNC_LEAVE_NOAPI(FAIL)
 
-    return H5FD_write(file->hdf5_file_lf, type, addr, size, buf);
 } /* end H5FD__vfd_swmr_write() */
 
+
 /*-------------------------------------------------------------------------
  * Function:    H5FD_vfd_swmr_truncate
  *
- * Purpose:     Makes sure that the true file size is the same (or larger)
- *              than the end-of-address for the underlying HDF5 file
+ * Purpose:     As the VFD SWMR reader VFD is only use on files that are
+ *              opened read only, this function should be unreachable.
  *
- * Return:      SUCCEED/FAIL
+ * Return:      FAIL
  *
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5FD__vfd_swmr_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t closing)
+H5FD__vfd_swmr_truncate(H5FD_t H5_ATTR_UNUSED *_file, hid_t H5_ATTR_UNUSED dxpl_id, 
+                        hbool_t H5_ATTR_UNUSED closing)
 {
-    H5FD_vfd_swmr_t *file = (H5FD_vfd_swmr_t *)_file; /* VFD SWMR file struct */
+    FUNC_ENTER_PACKAGE_NOERR /* Yes, even though this pushes an error on the stack */
 
-    FUNC_ENTER_PACKAGE_NOERR
+        HERROR(H5E_CACHE, H5E_SYSTEM, "called unreachable fcn.");
 
-    /* This routine should only be called if the VFD instance is opened
-     * for writing.
-     */
-    HDassert(file->writer);
-
-    FUNC_LEAVE_NOAPI(H5FD_truncate(file->hdf5_file_lf, closing))
+    FUNC_LEAVE_NOAPI(FAIL)
 }
 
+
 /*-------------------------------------------------------------------------
  * Function:    H5FD__vfd_swmr_lock
  *
@@ -1127,6 +1217,67 @@ done:
 
 } /* end H5FD__vfd_swmr_unlock() */
 
+
+/*-------------------------------------------------------------------------
+ * Function:    H5FD__vfd_swmr_ctl
+ *
+ * Purpose:     VFD SWMR reader VFD version of the ctl callback.
+ *
+ *              The desired operation is specified by the op_code
+ *              parameter.
+ *
+ *              The flags parameter controls management of op_codes that
+ *              are unknown to the callback
+ *
+ *              The input and output parameters allow op_code specific
+ *              input and output
+ *
+ *              At present, this VFD supports no op codes of its own and
+ *              simply passes ctl calls on to the underlying VFD.
+ *
+ * Return:      Non-negative on success/Negative on failure
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD__vfd_swmr_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void *input, void **output)
+{
+    H5FD_vfd_swmr_t *file      = (H5FD_vfd_swmr_t *)_file;
+    herr_t           ret_value = SUCCEED;
+
+    FUNC_ENTER_PACKAGE
+
+    /* Sanity checks */
+    HDassert(file);
+
+    switch (op_code) {
+
+        /* Unknown op code */
+        default:
+            if (flags & H5FD_CTL__ROUTE_TO_TERMINAL_VFD_FLAG) {
+                /* Pass ctl call down to R/W channel VFD */
+                if (H5FDctl(file->hdf5_file_lf, op_code, flags, input, output) < 0)
+                    HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL, "VFD ctl request failed")
+            }
+            else {
+                /* If no valid VFD routing flag is specified, fail for unknown op code
+                 * if H5FD_CTL__FAIL_IF_UNKNOWN_FLAG flag is set.
+                 */
+                if (flags & H5FD_CTL__FAIL_IF_UNKNOWN_FLAG)
+                    HGOTO_ERROR(H5E_VFL, H5E_FCNTL, FAIL,
+                                "VFD ctl request failed (unknown op code and fail if unknown flag is set)")
+            }
+
+            break;
+    }
+
+done:
+
+    FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD__vfd_swmr_ctl() */
+
+
 /*-------------------------------------------------------------------------
  * Function:    H5FD__vfd_swmr_load_hdr_and_idx()
  *
diff --git a/src/H5FDvfd_swmr.h b/src/H5FDvfd_swmr.h
index dfd8d19..326f825 100644
--- a/src/H5FDvfd_swmr.h
+++ b/src/H5FDvfd_swmr.h
@@ -19,12 +19,52 @@
 #define H5FD_VFD_SWMR       (H5FDperform_init(H5FD_vfd_swmr_init))
 #define H5FD_VFD_SWMR_VALUE H5_VFD_SWMR
 
+/* Semi-unique constant used to help identify structure pointers */
+#define H5FD_VFD_SWMR_READER_MAGIC 0xABC123
+
+
+/* ----------------------------------------------------------------------------
+ * Structure:   H5FD_vfd_swmr_reader_fapl_t
+ *
+ * Structure for configuring the VFD SWMR reader VFD.  This structure and the
+ * associated FAPL entry and get / set routines are for HDF5 library use only, 
+ * and should never be seen by the user.  
+ *
+ * When a file is opened in VFD SWMR mode, the VFD SWMR reader VFD is inserted 
+ * at the top of the user supplied (or default) VFD stack.
+ *
+ * The fields of this structure are discussed indvidually below.  Note that 
+ * there is no version field, since this structure should not be accessible 
+ * to the user.  The set of fields is quite limited, as most of the necessary
+ * configuration data is taken from the VFD SWMR configuration FAPL entry
+ *
+ * magic (int32_t)
+ *      Semi-unique number, used to sanity-check that a given pointer is
+ *      likely (or not) to be this structure type. MUST be first.
+ *      If magic is not H5FD_VFD_SWMR_READER_MAGIC, the structure (and/or 
+ *      pointer to) must be considered invalid.
+ *
+ * fapl_id (hid_t)
+ *      Library-given identification number of the FAPL containing the user 
+ *      supplied VFD stack.  Must be set to H5P_DEFAULT or contain a file 
+ *      driver entry specifying a VFD that supports VFD SWMR
+ *
+ * ----------------------------------------------------------------------------
+ */
+typedef struct H5FD_vfd_swmr_reader_fapl_t {
+    int32_t      magic;
+    hid_t        fapl_id;
+} H5FD_vfd_swmr_reader_fapl_t;
+
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 H5_DLL hid_t  H5FD_vfd_swmr_init(void);
-H5_DLL herr_t H5Pset_fapl_vfd_swmr(hid_t fapl_id);
+H5_DLL herr_t H5P_pop_vfd_swmr_reader_vfd_off_fapl(hid_t fapl_id);
+H5_DLL herr_t H5P_push_vfd_swmr_reader_vfd_on_fapl(hid_t fapl_id);
+
 
 #ifdef __cplusplus
 }
diff --git a/src/H5Fint.c b/src/H5Fint.c
index 9880126..c700ac6 100644
--- a/src/H5Fint.c
+++ b/src/H5Fint.c
@@ -38,6 +38,10 @@
 #include "H5Tprivate.h"  /* Datatypes                                */
 #include "H5VLprivate.h" /* Virtual Object Layer                     */
 
+#if 1 /* JRM */ /* probably want to re-work this */
+#include "H5FDvfd_swmr.h"
+#endif /* JRM */ 
+
 #include "H5VLnative_private.h" /* Native VOL connector                     */
 
 /****************/
@@ -1813,6 +1817,12 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
     size_t                  page_buf_size;
     unsigned                page_buf_min_meta_perc = 0;
     unsigned                page_buf_min_raw_perc  = 0;
+    hbool_t                 vfd_swmr                = FALSE; /* TRUE iff opening file with VFD SWMR */
+    hbool_t                 vfd_swmr_writer         = FALSE; /* TRUE iff opening file as VFD SWMR   */
+                                                             /* writer.                             */
+    hbool_t                 pop_vfd_swmr_reader_vfd = FALSE; /* Flag set when the VFD SWMR reader VFD */
+                                                             /* has been pushed on the supplied fapl  */
+                                                             /* and must be poped before return.      */
     hbool_t                 set_flag               = FALSE; /* Set the status_flags in the superblock */
     hbool_t                 clear                  = FALSE; /* Clear the status_flags */
     hbool_t                 evict_on_close;                 /* Evict on close value from plist */
@@ -1828,24 +1838,18 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
 
     FUNC_ENTER_NOAPI(NULL)
 
-    /*
-     * If the driver has a 'cmp' method then the driver is capable of
-     * determining when two file handles refer to the same file and the
-     * library can insure that when the application opens a file twice
-     * that the two handles coordinate their operations appropriately.
-     * Otherwise it is the application's responsibility to never open the
-     * same file more than once at a time.
-     */
-    if (NULL == (drvr = H5FD_get_class(fapl_id)))
-        HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to retrieve VFL class")
-
     /* Get the file access property list, for future queries */
     if (NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id)))
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list")
 
-    /* Check if we are using file locking */
-    if (H5F__check_if_using_file_locks(a_plist, &use_file_locking) < 0)
-        HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to get file locking flag")
+
+    /* start by testing to see if we are opening the file VFD SWMR reader.  If
+     * we are, we must "push" the vfd swrm reader vfd on the vfd "stack" supplied
+     * by the user in the fapl.  Since the user may use the fapl elsewhere, we 
+     * must "pop" the vfd swmr reader vfd off the vfd "stack" before we return.
+     *
+     * In passing, collect the VFD SWMR configuration info for later use.
+     */
 
     /* Allocate space for VFD SWMR configuration info */
     if (NULL == (vfd_swmr_config_ptr = H5MM_calloc(sizeof(H5F_vfd_swmr_config_t))))
@@ -1857,20 +1861,83 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
 
     /* When configured with VFD SWMR */
     if (vfd_swmr_config_ptr->version) {
+
+        /* get the page buffer size and verify that it is greater tha zero.  Note
+         * that this get of the page buffer size is redundant -- we do it again 
+         * below.
+         */
+        if (H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &page_buf_size) < 0)
+            HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get page buffer size");
+
+        if (page_buf_size == 0)
+            HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "page buffering must be enabled")
+
+        /* Paged allocation must also be enabled, but the page buffer
+         * initialization (H5PB_create) will detect a conflicting configuration
+         * and return an error.
+         */
+
+        /* Legacy SWMR and VFD SWMR are incompatible.  Fail if the legacy SWMR flags are set */
+        if ( (flags & H5F_ACC_SWMR_WRITE) || ( flags & H5F_ACC_SWMR_READ) )
+            HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "Legacy and VFD SWMR are incompatible")
+
         /* Verify that file access flags are consistent with VFD SWMR configuration */
         if ((flags & H5F_ACC_RDWR) && !vfd_swmr_config_ptr->writer)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "file access is writer but VFD SWMR config is reader")
         if ((flags & H5F_ACC_RDWR) == 0 && vfd_swmr_config_ptr->writer)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "file access is reader but VFD SWMR config is writer")
 
+
+        if ( ((flags & H5F_ACC_RDWR) == 0) && (! vfd_swmr_config_ptr->writer) ) {
+
+            vfd_swmr = TRUE;
+
+            /* We are opening a file as a VFD SWMR reader.  Push the vfd swrm reader vfd on the 
+             * vfd stack specified in the fapl.  Set the pop_vfd_swmr_reader flag to trigger a 
+             * pop of the vfd swmr reader vfd on exit from this function.
+             */
+            if ( H5P_push_vfd_swmr_reader_vfd_on_fapl(fapl_id) < 0 )
+                 HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, NULL, "can't push VFD SWMR reader VFD on FAPL");
+
+            pop_vfd_swmr_reader_vfd = TRUE;
+        
+        } else if ( ( flags & H5F_ACC_RDWR ) && ( vfd_swmr_config_ptr->writer ) ) {
+
+            vfd_swmr = TRUE;
+            vfd_swmr_writer = TRUE;
+        } 
+
+        /* if we get to this point, vfd_swmr must be TRUE. */
+        HDassert(vfd_swmr);
+
+
         /* Retrieve the private property for VFD SWMR testing */
         if (H5P_get(a_plist, H5F_ACS_GENERATE_MD_CK_CB_NAME, &cb_info) < 0)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get generate_md_ck_cb info")
+    }
 
-        if (!vfd_swmr_config_ptr->writer)
-            use_file_locking = FALSE;
+    /*
+     * If the driver has a 'cmp' method then the driver is capable of
+     * determining when two file handles refer to the same file and the
+     * library can insure that when the application opens a file twice
+     * that the two handles coordinate their operations appropriately.
+     * Otherwise it is the application's responsibility to never open the
+     * same file more than once at a time.
+     */
+    if (NULL == (drvr = H5FD_get_class(fapl_id)))
+        HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to retrieve VFL class")
+
+    /* Check if we are using file locking */
+    if (H5F__check_if_using_file_locks(a_plist, &use_file_locking) < 0)
+        HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to get file locking flag")
+
+    /* turn off file locking unconditionally if the file is being opened VFD SWMR reader */
+    if ( ( vfd_swmr ) && ( ! vfd_swmr_writer ) ) {
+
+         use_file_locking = FALSE;
     }
 
+
     /*
      * Opening a file is a two step process. First we try to open the
      * file in a way which doesn't affect its state (like not truncating
@@ -1898,11 +1965,10 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
                         name, tent_flags)
     } /* end if */
 
+
     /* Avoid reusing a virtual file opened exclusively by a second virtual
      * file, or opening the same file twice with different parameters.
      */
-    if ((lf = H5FD_deduplicate(lf, fapl_id)) == NULL)
-        HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "an already-open file conflicts with '%s'", name)
 
     /* Is the file already open? */
     if ((shared = H5F__sfile_search(lf)) != NULL) {
@@ -1916,6 +1982,8 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
          * readers don't expect the file to change under them), or if the
          * SWMR write/read access flags don't agree.
          */
+        if (H5FD_close(lf) < 0)
+            HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to close low-level file info")
         if (flags & H5F_ACC_TRUNC)
             HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to truncate a file which is already open")
         if (flags & H5F_ACC_EXCL)
@@ -1932,6 +2000,17 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id)
             HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL,
                         "SWMR read access flag not the same for file that is already open")
 
+        /* fail if VFD SWMR configurations disagree */
+        if ( HDmemcmp(&(shared->vfd_swmr_config), vfd_swmr_config_ptr, sizeof(H5F_vfd_swmr_config_t)) )
+            HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL,
+                        "VFD SWMR configuration not the same for file that is already open")
+
+        /* Arguably, we should fail if there is a page size mismatch.  However, if I read 
+         * the code correctly, the page size and page buffer configuration from the open file
+         * will domininate.  Thus, there probably isn't a functional issue.  That said, 
+         * this should be thought about.
+         */
+
         /* Allocate new "high-level" file struct */
         if ((file = H5F__new(shared, flags, fcpl_id, fapl_id, NULL)) == NULL)
             HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to create new file object")
@@ -2220,6 +2299,9 @@ done:
     if (vfd_swmr_config_ptr)
         H5MM_free(vfd_swmr_config_ptr);
 
+    if ( ( pop_vfd_swmr_reader_vfd ) && ( H5P_pop_vfd_swmr_reader_vfd_off_fapl(fapl_id) < 0 ) )
+        HDONE_ERROR(H5E_PLIST, H5E_CANTSET, NULL, "can't pop vfd swrm reader vfd off vfd stack")
+
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5F_open() */
 
diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
index c594ed8..adc2b40 100644
--- a/src/H5Fprivate.h
+++ b/src/H5Fprivate.h
@@ -1218,6 +1218,8 @@ H5_DLL herr_t  H5F_get_vfd_handle(const H5F_t *file, hid_t fapl, void **file_han
 /* VFD SWMR functions */
 H5_DLL hbool_t H5F_get_use_vfd_swmr(const H5F_t *f);
 H5_DLL herr_t  H5F_vfd_swmr_init(H5F_t *f, hbool_t file_create);
+H5_DLL herr_t H5F_vfd_swmr_build_md_path_name(H5F_vfd_swmr_config_t *config, const char *hdf5_filename,
+                                              char *name /*out*/);
 H5_DLL herr_t  H5F_vfd_swmr_close_or_flush(H5F_t *f, hbool_t closing);
 H5_DLL herr_t  H5F_update_vfd_swmr_metadata_file(H5F_t *f, uint32_t index_len,
                                                  H5FD_vfd_swmr_idx_entry_t *index);
diff --git a/src/H5Fvfd_swmr.c b/src/H5Fvfd_swmr.c
index 9c4360e..0e24804 100644
--- a/src/H5Fvfd_swmr.c
+++ b/src/H5Fvfd_swmr.c
@@ -49,8 +49,9 @@
 /* Local Macros */
 /****************/
 
-#define NANOSECS_PER_SECOND    1000000000 /* nanoseconds per second */
-#define NANOSECS_PER_TENTH_SEC 100000000  /* nanoseconds per 0.1 second */
+#define VFD_SWMR_MD_FILE_SUFFIX ".md"
+#define NANOSECS_PER_SECOND     1000000000 /* nanoseconds per second */
+#define NANOSECS_PER_TENTH_SEC  100000000  /* nanoseconds per 0.1 second */
 
 /* Declare an array of string to identify the VFD SMWR Log tags.
  * Note this array is used to generate the entry tag by the log reporting macro
@@ -186,8 +187,13 @@ H5F_vfd_swmr_init(H5F_t *f, hbool_t file_create)
         shared->vfd_swmr_writer = TRUE;
         shared->tick_num        = 0;
 
-        /* Retrieve the metadata filename built with md_file_path and md_file_name */
-        H5FD_vfd_swmr_get_md_path_name(f->shared->lf, &shared->md_file_path_name);
+        /* Allocate space for the (possibly constructed) metadata file name */
+        if (NULL == (shared->md_file_path_name = H5MM_calloc((H5FD_MAX_FILENAME_LEN + 1) * sizeof(char))))
+            HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate memory for mdc log file name")
+
+        if ( H5F_vfd_swmr_build_md_path_name(&(shared->vfd_swmr_config), f->open_name,
+                                             shared->md_file_path_name) < 0 )
+            HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to build metadata file name")
 
         if (((shared->vfd_swmr_md_fd = HDopen(shared->md_file_path_name, O_CREAT | O_RDWR | O_TRUNC,
                                               H5_POSIX_CREATE_MODE_RW))) < 0)
@@ -281,6 +287,67 @@ done:
 } /* H5F_vfd_swmr_init() */
 
 /*-------------------------------------------------------------------------
+ *
+ * Function:    H5F_vfd_swmr_build_md_path_name
+ *
+ * Purpose:     To construct the metadata file's full name based on config's
+ *              md_file_path and md_file_name.  See RFC for details.
+ *
+ *
+ * Return:      Success:        SUCCEED
+ *              Failure:        FAIL
+ *
+ * Programmer:  Vailin Choi -- 1/13/2022
+ *
+ * Changes:     Moved to H5Fvfd_swmr.c from H5FDvfd_swmr.c, and renamed
+ *              accordingly.  Changed FUNC_ENTER_PACKAGE to 
+ *              FUNC_ENTER_NOAPI.  Converted to a private function so 
+ *              that it can be called in H5FDvfd_swmr.c
+ *
+ *                                               JRM -- 5/17/22
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5F_vfd_swmr_build_md_path_name(H5F_vfd_swmr_config_t *config, const char *hdf5_filename,
+                                  char *name /*out*/)
+{
+    size_t tot_len   = 0;
+    size_t tmp_len   = 0;
+    herr_t ret_value = SUCCEED; /* Return value */
+
+    FUNC_ENTER_NOAPI(FAIL)
+
+    if ((tot_len = HDstrlen(config->md_file_path)) != 0) {
+
+        /* md_file_path + '/' */
+        if (++tot_len > H5F__MAX_VFD_SWMR_FILE_NAME_LEN)
+            HGOTO_ERROR(H5E_FILE, H5E_CANTCOPY, FAIL, "md_file_path and md_file_name exceeds maximum");
+        HDstrcat(name, config->md_file_path);
+        HDstrcat(name, "/");
+    }
+
+    if ((tmp_len = HDstrlen(config->md_file_name)) != 0) {
+        if ((tot_len += tmp_len) > H5F__MAX_VFD_SWMR_FILE_NAME_LEN)
+            HGOTO_ERROR(H5E_FILE, H5E_CANTCOPY, FAIL, "md_file_path and md_file_name exceeds maximum");
+        HDstrcat(name, config->md_file_name);
+    }
+    else {
+        /* Automatic generation of metadata file name based on hdf5_filename + '.md' */
+        if ((tot_len += (HDstrlen(hdf5_filename) + 3)) > H5F__MAX_VFD_SWMR_FILE_NAME_LEN)
+            HGOTO_ERROR(H5E_FILE, H5E_CANTCOPY, FAIL, "md_file_path and md_file_name maximum");
+
+        HDstrcat(name, hdf5_filename);
+        HDstrcat(name, VFD_SWMR_MD_FILE_SUFFIX);
+    }
+
+done:
+
+    FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5F_vfd_swmr_build_md_path_name() */
+
+/*-------------------------------------------------------------------------
  * Function:    H5F_vfd_swmr_close_or_flush
  *
  * Purpose:     Used by the VFD SWMR writer when the HDF5 file is closed
diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c
index f900965..d6beece 100644
--- a/src/H5Pfapl.c
+++ b/src/H5Pfapl.c
@@ -6283,9 +6283,6 @@ H5Pset_vfd_swmr_config(hid_t plist_id, H5F_vfd_swmr_config_t *config_ptr)
     if (H5P_set(plist, H5F_ACS_VFD_SWMR_CONFIG_NAME, config_ptr) < 0)
         HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set metadata cache initial config")
 
-    if (H5P_set_driver(plist, H5FD_VFD_SWMR, NULL, NULL) < 0)
-        HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set VFD SWMR driver info");
-
 done:
     FUNC_LEAVE_API(ret_value)
 } /* H5Pset_vfd_swmr_config() */