Subfiling updates for release (#1963)

* Remove generated file h5fuse.sh

* Link pthreads library when Subfiling VFD is built

* Switch to MPI I/O driver for Subfiling HDF5 stub file

* Rough first implementation for Subfiling file deletion

* Subfiling VFD - get file dirname for file deletion

* Subfiling VFD - set lock callback to NULL for now to avoid performance
issues

* Committing clang-format changes

* Minor tidying up of Subfiling testing

* Fixups for Subfiling VFD support in tools

* Tidy up Subfiling public interface and add Doxygen

* Respect Subfiling configuration settings from application

* Add release note for Subfiling VFD

* Committing clang-format changes

* Committing clang-format changes

* Shorten some Subfiling environment variable names

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

1 files changed, 200 insertions, 77 deletions

diff --git a/src/H5FDsubfiling/H5FDioc.c b/src/H5FDsubfiling/H5FDioc.c
index 2eb7970..f779c46 100644
--- a/src/H5FDsubfiling/H5FDioc.c
+++ b/src/H5FDsubfiling/H5FDioc.c
@@ -16,6 +16,8 @@
  *              another underlying VFD. Maintains two files simultaneously.
  */
 
+#include <libgen.h>
+
 /* This source code file is part of the H5FD driver module */
 #include "H5FDdrvr_module.h"
 
@@ -25,7 +27,7 @@
 #include "H5FDprivate.h"  /* File drivers             */
 #include "H5FDioc.h"      /* IOC file driver          */
 #include "H5FDioc_priv.h" /* IOC file driver          */
-#include "H5FDsec2.h"     /* Sec2 VFD                 */
+#include "H5FDmpio.h"     /* MPI I/O VFD              */
 #include "H5FLprivate.h"  /* Free Lists               */
 #include "H5Fprivate.h"   /* File access              */
 #include "H5Iprivate.h"   /* IDs                      */
@@ -53,7 +55,7 @@ typedef struct H5FD_ioc_t {
     int      mpi_rank;
     int      mpi_size;
 
-    H5FD_t *ioc_file; /* native HDF5 file pointer (sec2) */
+    H5FD_t *ioc_file; /* native HDF5 file pointer */
 
     int64_t context_id; /* The value used to lookup a subfiling context for the file */
 
@@ -68,7 +70,6 @@ typedef struct H5FD_ioc_t {
      * Windows code further below.
      */
     dev_t device; /* file device number   */
-    ino_t inode;  /* file i-node number   */
 #else
     /* Files in windows are uniquely identified by the volume serial
      * number and the file index (both low and high parts).
@@ -161,7 +162,7 @@ static herr_t H5FD__ioc_ctl(H5FD_t *file, uint64_t op_code, uint64_t flags,
                             const void *input, void **result);
 */
 
-static herr_t H5FD__ioc_get_default_config(H5FD_ioc_config_t *config_out);
+static herr_t H5FD__ioc_get_default_config(hid_t fapl_id, H5FD_ioc_config_t *config_out);
 static herr_t H5FD__ioc_validate_config(const H5FD_ioc_config_t *fa);
 static int    H5FD__copy_plist(hid_t fapl_id, hid_t *id_out_ptr);
 
@@ -358,13 +359,13 @@ H5Pset_fapl_ioc(hid_t fapl_id, H5FD_ioc_config_t *vfd_config)
         H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
 
     if (vfd_config == NULL) {
-        if (NULL == (ioc_conf = HDcalloc(1, sizeof(*ioc_conf))))
+        if (NULL == (ioc_conf = H5FL_CALLOC(H5FD_ioc_config_t)))
             H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL,
                                     "can't allocate IOC VFD configuration");
-        ioc_conf->ioc_fapl_id = H5I_INVALID_HID;
+        ioc_conf->under_fapl_id = H5I_INVALID_HID;
 
         /* Get IOC VFD defaults */
-        if (H5FD__ioc_get_default_config(ioc_conf) < 0)
+        if (H5FD__ioc_get_default_config(fapl_id, ioc_conf) < 0)
             H5_SUBFILING_GOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't get default IOC VFD configuration");
 
         vfd_config = ioc_conf;
@@ -377,9 +378,9 @@ H5Pset_fapl_ioc(hid_t fapl_id, H5FD_ioc_config_t *vfd_config)
 
 done:
     if (ioc_conf) {
-        if (ioc_conf->ioc_fapl_id >= 0 && H5I_dec_ref(ioc_conf->ioc_fapl_id) < 0)
-            H5_SUBFILING_DONE_ERROR(H5E_PLIST, H5E_CANTDEC, FAIL, "can't close IOC FAPL");
-        HDfree(ioc_conf);
+        if (ioc_conf->under_fapl_id >= 0 && H5I_dec_ref(ioc_conf->under_fapl_id) < 0)
+            H5_SUBFILING_DONE_ERROR(H5E_PLIST, H5E_CANTDEC, FAIL, "can't close IOC under FAPL");
+        H5FL_FREE(H5FD_ioc_config_t, ioc_conf);
     }
 
     H5_SUBFILING_FUNC_LEAVE;
@@ -423,7 +424,7 @@ H5Pget_fapl_ioc(hid_t fapl_id, H5FD_ioc_config_t *config_out)
     }
 
     if (use_default_config) {
-        if (H5FD__ioc_get_default_config(config_out) < 0)
+        if (H5FD__ioc_get_default_config(fapl_id, config_out) < 0)
             H5_SUBFILING_GOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get default IOC VFD configuration");
     }
     else {
@@ -431,8 +432,8 @@ H5Pget_fapl_ioc(hid_t fapl_id, H5FD_ioc_config_t *config_out)
         HDmemcpy(config_out, config_ptr, sizeof(H5FD_ioc_config_t));
 
         /* Copy the driver info value */
-        if (H5FD__copy_plist(config_ptr->ioc_fapl_id, &(config_out->ioc_fapl_id)) < 0)
-            H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't copy IOC FAPL");
+        if (H5FD__copy_plist(config_ptr->under_fapl_id, &(config_out->under_fapl_id)) < 0)
+            H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't copy IOC under FAPL");
     }
 
 done:
@@ -451,35 +452,53 @@ done:
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5FD__ioc_get_default_config(H5FD_ioc_config_t *config_out)
+H5FD__ioc_get_default_config(hid_t fapl_id, H5FD_ioc_config_t *config_out)
 {
-    herr_t ret_value = SUCCEED;
+    MPI_Comm comm      = MPI_COMM_NULL;
+    MPI_Info info      = MPI_INFO_NULL;
+    herr_t   ret_value = SUCCEED;
 
     HDassert(config_out);
 
     HDmemset(config_out, 0, sizeof(*config_out));
 
     config_out->magic         = H5FD_IOC_FAPL_MAGIC;
-    config_out->version       = H5FD_CURR_IOC_FAPL_VERSION;
-    config_out->ioc_fapl_id   = H5I_INVALID_HID;
-    config_out->stripe_count  = 0;
-    config_out->stripe_depth  = H5FD_DEFAULT_STRIPE_DEPTH;
-    config_out->ioc_selection = SELECT_IOC_ONE_PER_NODE;
+    config_out->version       = H5FD_IOC_CURR_FAPL_VERSION;
+    config_out->under_fapl_id = H5I_INVALID_HID;
+
+    /*
+     * Use default subfiling configuration. Do NOT call
+     * H5Pget_fapl_subfiling here as that can cause issues
+     */
+    config_out->subf_config.ioc_selection = SELECT_IOC_ONE_PER_NODE;
+    config_out->subf_config.stripe_size   = H5FD_SUBFILING_DEFAULT_STRIPE_SIZE;
+    config_out->subf_config.stripe_count  = 0;
 
     /* Create a default FAPL and choose an appropriate underlying driver */
-    if ((config_out->ioc_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
+    if ((config_out->under_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
         H5_SUBFILING_GOTO_ERROR(H5E_PLIST, H5E_CANTCREATE, FAIL, "can't create default FAPL");
 
-    /* Currently, only sec2 vfd supported */
-    if (H5Pset_fapl_sec2(config_out->ioc_fapl_id) < 0)
-        H5_SUBFILING_GOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set Sec2 VFD on IOC FAPL");
+    /* Check if any MPI parameters were set on the FAPL */
+    if (H5Pget_mpi_params(fapl_id, &comm, &info) < 0)
+        H5_SUBFILING_GOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get MPI Comm/Info");
+    if (comm == MPI_COMM_NULL)
+        comm = MPI_COMM_WORLD;
+
+    /* Hardwire MPI I/O VFD for now */
+    if (H5Pset_fapl_mpio(config_out->under_fapl_id, comm, info) < 0)
+        H5_SUBFILING_GOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set MPI I/O VFD on IOC under FAPL");
 
     /* Specific to this I/O Concentrator */
-    config_out->thread_pool_count = H5FD_IOC_THREAD_POOL_SIZE;
+    config_out->thread_pool_count = H5FD_IOC_DEFAULT_THREAD_POOL_SIZE;
 
 done:
+    if (H5_mpi_comm_free(&comm) < 0)
+        H5_SUBFILING_DONE_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "can't free MPI Communicator");
+    if (H5_mpi_info_free(&info) < 0)
+        H5_SUBFILING_DONE_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "can't free MPI Info object");
+
     if (ret_value < 0) {
-        if (config_out->ioc_fapl_id >= 0 && H5Pclose(config_out->ioc_fapl_id) < 0)
+        if (config_out->under_fapl_id >= 0 && H5Pclose(config_out->under_fapl_id) < 0)
             H5_SUBFILING_DONE_ERROR(H5E_PLIST, H5E_CANTCLOSEOBJ, FAIL, "can't close FAPL");
     }
 
@@ -510,7 +529,7 @@ H5FD__ioc_validate_config(const H5FD_ioc_config_t *fa)
 
     HDassert(fa != NULL);
 
-    if (fa->version != H5FD_CURR_IOC_FAPL_VERSION)
+    if (fa->version != H5FD_IOC_CURR_FAPL_VERSION)
         H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Unknown H5FD_ioc_config_t version");
 
     if (fa->magic != H5FD_IOC_FAPL_MAGIC)
@@ -696,8 +715,8 @@ H5FD__ioc_fapl_copy(const void *_old_fa)
     HDmemcpy(new_fa_ptr, old_fa_ptr, sizeof(H5FD_ioc_config_t));
 
     /* Copy the FAPL */
-    if (H5FD__copy_plist(old_fa_ptr->ioc_fapl_id, &(new_fa_ptr->ioc_fapl_id)) < 0)
-        H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't copy the IOC FAPL");
+    if (H5FD__copy_plist(old_fa_ptr->under_fapl_id, &(new_fa_ptr->under_fapl_id)) < 0)
+        H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't copy the IOC under FAPL");
 
     ret_value = (void *)new_fa_ptr;
 
@@ -728,8 +747,8 @@ H5FD__ioc_fapl_free(void *_fapl)
     /* Check arguments */
     HDassert(fapl);
 
-    if (fapl->ioc_fapl_id >= 0 && H5I_dec_ref(fapl->ioc_fapl_id) < 0)
-        H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTDEC, FAIL, "can't close FAPL ID");
+    if (fapl->under_fapl_id >= 0 && H5I_dec_ref(fapl->under_fapl_id) < 0)
+        H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTDEC, FAIL, "can't close IOC under FAPL ID");
 
     /* Free the property list */
     fapl = H5FL_FREE(H5FD_ioc_config_t, fapl);
@@ -774,10 +793,10 @@ H5FD__ioc_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
 
     if (NULL == (file_ptr = (H5FD_ioc_t *)H5FL_CALLOC(H5FD_ioc_t)))
         H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate file struct");
-    file_ptr->comm           = MPI_COMM_NULL;
-    file_ptr->info           = MPI_INFO_NULL;
-    file_ptr->context_id     = -1;
-    file_ptr->fa.ioc_fapl_id = H5I_INVALID_HID;
+    file_ptr->comm             = MPI_COMM_NULL;
+    file_ptr->info             = MPI_INFO_NULL;
+    file_ptr->context_id       = -1;
+    file_ptr->fa.under_fapl_id = H5I_INVALID_HID;
 
     /* Get the driver-specific file access properties */
     if (NULL == (plist_ptr = (H5P_genplist_t *)H5I_object(fapl_id)))
@@ -814,7 +833,7 @@ H5FD__ioc_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
 
     config_ptr = H5P_peek_driver_info(plist_ptr);
     if (!config_ptr || (H5P_FILE_ACCESS_DEFAULT == fapl_id)) {
-        if (H5FD__ioc_get_default_config(&default_config) < 0)
+        if (H5FD__ioc_get_default_config(fapl_id, &default_config) < 0)
             H5_SUBFILING_GOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get default IOC VFD configuration");
         config_ptr = &default_config;
     }
@@ -848,11 +867,11 @@ H5FD__ioc_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
     }
 
     /* Copy the ioc FAPL. */
-    if (H5FD__copy_plist(config_ptr->ioc_fapl_id, &(file_ptr->fa.ioc_fapl_id)) < 0)
-        H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't copy FAPL");
+    if (H5FD__copy_plist(config_ptr->under_fapl_id, &(file_ptr->fa.under_fapl_id)) < 0)
+        H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't copy IOC under FAPL");
 
     /* Check the underlying driver (sec2/mpio/etc.) */
-    if (NULL == (plist_ptr = (H5P_genplist_t *)H5I_object(config_ptr->ioc_fapl_id)))
+    if (NULL == (plist_ptr = (H5P_genplist_t *)H5I_object(config_ptr->under_fapl_id)))
         H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list");
 
     if (H5P_peek(plist_ptr, H5F_ACS_FILE_DRV_NAME, &driver_prop) < 0)
@@ -861,13 +880,13 @@ H5FD__ioc_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
         H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL,
                                 "invalid driver ID in file access property list");
 
-    if (driver->value != H5_VFD_SEC2) {
+    if (driver->value != H5_VFD_MPIO) {
         H5_SUBFILING_GOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL,
-                                "unable to open file '%s' - only Sec2 VFD is currently supported", name);
+                                "unable to open file '%s' - only MPI I/O VFD is currently supported", name);
     }
     else {
-        subfiling_context_t *sf_context = NULL;
-        uint64_t             inode_id   = UINT64_MAX;
+        subfiling_context_t *sf_context  = NULL;
+        void                *file_handle = NULL;
         int                  ioc_flags;
         int                  l_error = 0;
         int                  g_error = 0;
@@ -881,34 +900,12 @@ H5FD__ioc_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
         if (H5F_ACC_EXCL & flags)
             ioc_flags |= O_EXCL;
 
-        file_ptr->ioc_file = H5FD_open(file_ptr->file_path, flags, config_ptr->ioc_fapl_id, HADDR_UNDEF);
+        file_ptr->ioc_file = H5FD_open(file_ptr->file_path, flags, config_ptr->under_fapl_id, HADDR_UNDEF);
         if (file_ptr->ioc_file) {
-            h5_stat_t sb;
-            void     *file_handle = NULL;
-
-            if (file_ptr->mpi_rank == 0) {
-                if (H5FDget_vfd_handle(file_ptr->ioc_file, config_ptr->ioc_fapl_id, &file_handle) < 0)
-                    H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get file handle");
-
-                if (HDfstat(*(int *)file_handle, &sb) < 0)
-                    H5_SUBFILING_SYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, NULL, "unable to fstat file");
-
-                HDcompile_assert(sizeof(uint64_t) >= sizeof(ino_t));
-                file_ptr->inode = sb.st_ino;
-                inode_id        = (uint64_t)sb.st_ino;
-            }
-
-            if (MPI_SUCCESS != (mpi_code = MPI_Bcast(&inode_id, 1, MPI_UINT64_T, 0, file_ptr->comm)))
-                H5_SUBFILING_MPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mpi_code);
-
-            if (file_ptr->mpi_rank != 0)
-                file_ptr->inode = (ino_t)inode_id;
+            if (H5FDget_vfd_handle(file_ptr->ioc_file, config_ptr->under_fapl_id, &file_handle) < 0)
+                H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get file handle");
         }
         else {
-            /* The two-step file opening approach may be
-             * the root cause for the sec2 open to return a NULL.
-             * It is prudent then, to collectively fail (early) in this case.
-             */
             l_error = 1;
         }
 
@@ -925,7 +922,7 @@ H5FD__ioc_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
          * context ID will be returned, which is used for
          * further interactions with this file's subfiles.
          */
-        if (H5_open_subfiles(file_ptr->file_path, inode_id, file_ptr->fa.ioc_selection, ioc_flags,
+        if (H5_open_subfiles(file_ptr->file_path, file_handle, &file_ptr->fa.subf_config, ioc_flags,
                              file_ptr->comm, &file_ptr->context_id) < 0)
             H5_SUBFILING_GOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open subfiles for file '%s'",
                                     name);
@@ -991,9 +988,9 @@ H5FD__ioc_close_int(H5FD_ioc_t *file_ptr)
     }
 #endif
 
-    if (file_ptr->fa.ioc_fapl_id >= 0 && H5I_dec_ref(file_ptr->fa.ioc_fapl_id) < 0)
-        H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_ARGS, FAIL, "can't close FAPL");
-    file_ptr->fa.ioc_fapl_id = H5I_INVALID_HID;
+    if (file_ptr->fa.under_fapl_id >= 0 && H5I_dec_ref(file_ptr->fa.under_fapl_id) < 0)
+        H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_ARGS, FAIL, "can't close IOC under FAPL");
+    file_ptr->fa.under_fapl_id = H5I_INVALID_HID;
 
     /* Close underlying file */
     if (file_ptr->ioc_file) {
@@ -1331,7 +1328,7 @@ H5FD__ioc_get_handle(H5FD_t *_file, hid_t H5_ATTR_UNUSED fapl, void **file_handl
     HDassert(file->ioc_file);
     HDassert(file_handle);
 
-    if (H5FD_get_vfd_handle(file->ioc_file, file->fa.ioc_fapl_id, file_handle) < 0)
+    if (H5FD_get_vfd_handle(file->ioc_file, file->fa.under_fapl_id, file_handle) < 0)
         H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "unable to get handle of R/W file");
 
 done:
@@ -1597,12 +1594,138 @@ done:
 static herr_t
 H5FD__ioc_del(const char *name, hid_t fapl)
 {
-    herr_t ret_value = SUCCEED;
+    H5P_genplist_t *plist;
+    h5_stat_t       st;
+    MPI_Comm        comm          = MPI_COMM_NULL;
+    MPI_Info        info          = MPI_INFO_NULL;
+    FILE           *config_file   = NULL;
+    char           *name_copy     = NULL;
+    char           *name_copy2    = NULL;
+    char           *tmp_filename  = NULL;
+    char           *base_filename = NULL;
+    char           *file_dirname  = NULL;
+    int             mpi_rank      = INT_MAX;
+    int             mpi_code;
+    herr_t          ret_value = SUCCEED;
+
+    /* TODO: Eventually this routine should share common code
+     * with H5_subfiling_common's routines so it doesn't get
+     * out of sync
+     */
+
+    if (NULL == (plist = H5P_object_verify(fapl, H5P_FILE_ACCESS)))
+        H5_SUBFILING_GOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list");
+    HDassert(H5FD_IOC == H5P_peek_driver(plist));
+
+    if (H5FD_mpi_self_initialized) {
+        comm = MPI_COMM_WORLD;
+    }
+    else {
+        /* Get the MPI communicator and info from the fapl */
+        if (H5P_get(plist, H5F_ACS_MPI_PARAMS_INFO_NAME, &info) < 0)
+            H5_SUBFILING_GOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get MPI info object");
+        if (H5P_get(plist, H5F_ACS_MPI_PARAMS_COMM_NAME, &comm) < 0)
+            H5_SUBFILING_GOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get MPI communicator");
+    }
+
+    /* Get the MPI rank of this process */
+    if (MPI_SUCCESS != (mpi_code = MPI_Comm_rank(comm, &mpi_rank)))
+        H5_SUBFILING_MPI_GOTO_ERROR(FAIL, "MPI_Comm_rank failed", mpi_code);
+
+    if (mpi_rank == 0) {
+        int n_io_concentrators = 0;
+        int num_digits         = 0;
+
+        if (HDstat(name, &st) < 0)
+            H5_SUBFILING_SYS_GOTO_ERROR(H5E_FILE, H5E_SYSERRSTR, FAIL, "HDstat failed");
 
-    (void)name;
-    (void)fapl;
+        if (NULL == (name_copy = HDstrdup(name)))
+            H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't copy filename");
+        if (NULL == (name_copy2 = HDstrdup(name)))
+            H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't copy filename");
 
-    /* TODO: implement later */
+        base_filename = basename(name_copy);
+        file_dirname  = dirname(name_copy2);
+
+        /* Try to open the subfiling configuration file and get the number of IOCs */
+        if (NULL == (tmp_filename = HDmalloc(PATH_MAX)))
+            H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL,
+                                    "can't allocate config file name buffer");
+
+        /* TODO: No support for subfile directory prefix currently */
+        HDsnprintf(tmp_filename, PATH_MAX, "%s/%s" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, file_dirname,
+                   base_filename, (uint64_t)st.st_ino);
+
+        if (NULL == (config_file = HDfopen(tmp_filename, "r"))) {
+            if (ENOENT == errno) {
+#ifdef H5FD_IOC_DEBUG
+                HDprintf("** WARNING: couldn't delete Subfiling configuration file '%s'\n", tmp_filename);
+#endif
+
+                H5_SUBFILING_GOTO_DONE(SUCCEED);
+            }
+            else
+                H5_SUBFILING_SYS_GOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL,
+                                            "can't open subfiling config file");
+        }
+
+        if (H5_get_num_iocs_from_config_file(config_file, &n_io_concentrators) < 0)
+            H5_SUBFILING_GOTO_ERROR(H5E_FILE, H5E_READERROR, FAIL, "can't read subfiling config file");
+
+        /* Delete the Subfiling configuration file */
+        if (EOF == HDfclose(config_file)) {
+            config_file = NULL;
+            H5_SUBFILING_SYS_GOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL,
+                                        "can't close subfiling config file");
+        }
+
+        config_file = NULL;
+
+        if (HDremove(tmp_filename) < 0)
+            H5_SUBFILING_SYS_GOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL,
+                                        "can't delete subfiling config file");
+
+        /* Try to delete each of the subfiles */
+        num_digits = (int)(HDlog10(n_io_concentrators) + 1);
+
+        for (int i = 0; i < n_io_concentrators; i++) {
+            /* TODO: No support for subfile directory prefix currently */
+            HDsnprintf(tmp_filename, PATH_MAX, "%s/%s" H5FD_SUBFILING_FILENAME_TEMPLATE, file_dirname,
+                       base_filename, (uint64_t)st.st_ino, num_digits, i + 1, n_io_concentrators);
+
+            if (HDremove(tmp_filename) < 0) {
+#ifdef H5FD_IOC_DEBUG
+                HDprintf("** WARNING: couldn't delete subfile '%s'\n", tmp_filename);
+#endif
+
+                if (ENOENT != errno)
+                    H5_SUBFILING_SYS_GOTO_ERROR(H5E_FILE, H5E_CANTDELETEFILE, FAIL, "can't delete subfile");
+            }
+        }
+
+        /* Delete the HDF5 stub file */
+        if (HDremove(name) < 0)
+            H5_SUBFILING_SYS_GOTO_ERROR(H5E_FILE, H5E_CANTDELETEFILE, FAIL, "can't delete HDF5 file");
+    }
+
+done:
+    if (config_file)
+        if (EOF == HDfclose(config_file))
+            H5_SUBFILING_DONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "can't close subfiling config file");
+
+    /* Set up a barrier (don't want processes to run ahead of the delete) */
+    if (MPI_SUCCESS != (mpi_code = MPI_Barrier(comm)))
+        H5_SUBFILING_MPI_DONE_ERROR(FAIL, "MPI_Barrier failed", mpi_code);
+
+    /* Free duplicated MPI Communicator and Info objects */
+    if (H5_mpi_comm_free(&comm) < 0)
+        H5_SUBFILING_DONE_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to free MPI communicator");
+    if (H5_mpi_info_free(&info) < 0)
+        H5_SUBFILING_DONE_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to free MPI info object");
+
+    HDfree(tmp_filename);
+    HDfree(name_copy);
+    HDfree(name_copy2);
 
     H5_SUBFILING_FUNC_LEAVE;
 }