Subfiling VFD - add option to specify config file prefix (#2495) (#2570)

5 files changed, 165 insertions, 18 deletions

diff --git a/src/H5FDsubfiling/H5FDioc.c b/src/H5FDsubfiling/H5FDioc.c
index 152c045..2fd8b64 100644
--- a/src/H5FDsubfiling/H5FDioc.c
+++ b/src/H5FDsubfiling/H5FDioc.c
@@ -1455,6 +1455,7 @@ H5FD__ioc_del(const char *name, hid_t fapl)
     if (mpi_rank == 0) {
         int64_t read_n_subfiles = 0;
         int32_t n_subfiles      = 0;
+        char   *prefix_env      = NULL;
         int     num_digits      = 0;
 
         if (HDstat(name, &st) < 0)
@@ -1470,9 +1471,13 @@ H5FD__ioc_del(const char *name, hid_t fapl)
             H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL,
                                     "can't allocate config file name buffer");
 
+        /* Check if a prefix has been set for the configuration file name */
+        prefix_env = HDgetenv(H5FD_SUBFILING_CONFIG_FILE_PREFIX);
+
         /* TODO: No support for subfile directory prefix currently */
-        HDsnprintf(tmp_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, file_dirname,
-                   base_filename, (uint64_t)st.st_ino);
+        /* TODO: Possibly try loading config file prefix from file before deleting */
+        HDsnprintf(tmp_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE,
+                   prefix_env ? prefix_env : file_dirname, base_filename, (uint64_t)st.st_ino);
 
         if (NULL == (config_file = HDfopen(tmp_filename, "r"))) {
             if (ENOENT == errno) {
diff --git a/src/H5FDsubfiling/H5FDsubfiling.c b/src/H5FDsubfiling/H5FDsubfiling.c
index 5fcaddb..afdf073 100644
--- a/src/H5FDsubfiling/H5FDsubfiling.c
+++ b/src/H5FDsubfiling/H5FDsubfiling.c
@@ -106,6 +106,8 @@ typedef struct H5FD_subfiling_t {
     uint64_t file_id;
     int64_t  context_id; /* The value used to lookup a subfiling context for the file */
 
+    hbool_t fail_to_encode; /* Used to check for failures from sb_get_size routine */
+
     char *file_dir;  /* Directory where we find files */
     char *file_path; /* The user defined filename */
 
@@ -145,6 +147,12 @@ typedef struct H5FD_subfiling_t {
 #define REGION_OVERFLOW(A, Z)                                                                                \
     (ADDR_OVERFLOW(A) || SIZE_OVERFLOW(Z) || HADDR_UNDEF == (A) + (Z) || (HDoff_t)((A) + (Z)) < (HDoff_t)(A))
 
+/*
+ * NOTE: Must be kept in sync with the private
+ * H5F_MAX_DRVINFOBLOCK_SIZE macro value for now
+ */
+#define H5FD_SUBFILING_MAX_DRV_INFO_SIZE 1024
+
 /* Prototypes */
 static herr_t  H5FD__subfiling_term(void);
 static hsize_t H5FD__subfiling_sb_size(H5FD_t *_file);
@@ -679,8 +687,9 @@ done:
 static hsize_t
 H5FD__subfiling_sb_size(H5FD_t *_file)
 {
-    H5FD_subfiling_t *file      = (H5FD_subfiling_t *)_file;
-    hsize_t           ret_value = 0;
+    subfiling_context_t *sf_context = NULL;
+    H5FD_subfiling_t    *file       = (H5FD_subfiling_t *)_file;
+    hsize_t              ret_value  = 0;
 
     HDassert(file);
 
@@ -699,6 +708,24 @@ H5FD__subfiling_sb_size(H5FD_t *_file)
     /* Subfiling stripe count (encoded as int64_t for future) */
     ret_value += sizeof(int64_t);
 
+    /* Subfiling config file prefix string length */
+    ret_value += sizeof(uint64_t);
+
+    /*
+     * Since this callback currently can't return any errors, we
+     * will set the "fail to encode" flag on the file if we fail
+     * to retrieve the context object here so we can check for
+     * errors later.
+     */
+    if (NULL == (sf_context = H5_get_subfiling_object(file->context_id))) {
+        file->fail_to_encode = TRUE;
+    }
+    else {
+        if (sf_context->config_file_prefix) {
+            ret_value += HDstrlen(sf_context->config_file_prefix) + 1;
+        }
+    }
+
     /* Add superblock information from IOC file if necessary */
     if (file->sf_file) {
         /* Encode the IOC's name into the subfiling information */
@@ -707,6 +734,16 @@ H5FD__subfiling_sb_size(H5FD_t *_file)
         ret_value += H5FD_sb_size(file->sf_file);
     }
 
+    /*
+     * Since the library doesn't currently properly check this,
+     * set the "fail to encode" flag if the message size is
+     * larger than the library's currently accepted max message
+     * size so that we don't try to encode the message and overrun
+     * a buffer.
+     */
+    if (ret_value > H5FD_SUBFILING_MAX_DRV_INFO_SIZE)
+        file->fail_to_encode = TRUE;
+
     H5_SUBFILING_FUNC_LEAVE;
 } /* end H5FD__subfiling_sb_size() */
 
@@ -725,9 +762,17 @@ H5FD__subfiling_sb_encode(H5FD_t *_file, char *name, unsigned char *buf)
     subfiling_context_t *sf_context = NULL;
     H5FD_subfiling_t    *file       = (H5FD_subfiling_t *)_file;
     uint8_t             *p          = (uint8_t *)buf;
+    uint64_t             tmpu64;
     int64_t              tmp64;
     int32_t              tmp32;
-    herr_t               ret_value = SUCCEED;
+    size_t               prefix_len = 0;
+    herr_t               ret_value  = SUCCEED;
+
+    /* Check if the "fail to encode flag" is set */
+    if (file->fail_to_encode)
+        H5_SUBFILING_GOTO_ERROR(
+            H5E_VFL, H5E_CANTENCODE, FAIL,
+            "can't encode subfiling driver info message - message was too large or internal error occurred");
 
     if (NULL == (sf_context = H5_get_subfiling_object(file->context_id)))
         H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get subfiling context object");
@@ -753,6 +798,21 @@ H5FD__subfiling_sb_encode(H5FD_t *_file, char *name, unsigned char *buf)
     tmp64 = sf_context->sf_num_subfiles;
     INT64ENCODE(p, tmp64);
 
+    /* Encode config file prefix string length */
+    if (sf_context->config_file_prefix) {
+        prefix_len = HDstrlen(sf_context->config_file_prefix) + 1;
+        H5_CHECKED_ASSIGN(tmpu64, uint64_t, prefix_len, size_t);
+    }
+    else
+        tmpu64 = 0;
+    UINT64ENCODE(p, tmpu64);
+
+    /* Encode config file prefix string */
+    if (sf_context->config_file_prefix) {
+        HDmemcpy(p, sf_context->config_file_prefix, prefix_len);
+        p += prefix_len;
+    }
+
     /* Encode IOC VFD configuration information if necessary */
     if (file->sf_file) {
         char ioc_name[9];
@@ -786,10 +846,17 @@ H5FD__subfiling_sb_decode(H5FD_t *_file, const char *name, const unsigned char *
     subfiling_context_t *sf_context = NULL;
     H5FD_subfiling_t    *file       = (H5FD_subfiling_t *)_file;
     const uint8_t       *p          = (const uint8_t *)buf;
+    uint64_t             tmpu64;
     int64_t              tmp64;
     int32_t              tmp32;
     herr_t               ret_value = SUCCEED;
 
+    /* Check if we previously failed to encode the info */
+    if (file->fail_to_encode)
+        H5_SUBFILING_GOTO_ERROR(
+            H5E_VFL, H5E_CANTDECODE, FAIL,
+            "can't decode subfiling driver info message - message wasn't encoded (or encoded improperly)");
+
     if (NULL == (sf_context = H5_get_subfiling_object(file->context_id)))
         H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "can't get subfiling context object");
 
@@ -814,6 +881,25 @@ H5FD__subfiling_sb_decode(H5FD_t *_file, const char *name, const unsigned char *
     H5_CHECK_OVERFLOW(tmp64, int64_t, int32_t);
     file->fa.shared_cfg.stripe_count = (int32_t)tmp64;
 
+    /* Decode config file prefix string length */
+    UINT64DECODE(p, tmpu64);
+
+    /* Decode config file prefix string */
+    if (tmpu64 > 0) {
+        if (!sf_context->config_file_prefix) {
+            if (NULL == (sf_context->config_file_prefix = HDmalloc(tmpu64)))
+                H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL,
+                                        "can't allocate space for config file prefix string");
+
+            HDmemcpy(sf_context->config_file_prefix, p, tmpu64);
+
+            /* Just in case.. */
+            sf_context->config_file_prefix[tmpu64 - 1] = '\0';
+        }
+
+        p += tmpu64;
+    }
+
     if (file->sf_file) {
         char ioc_name[9];
 
@@ -1051,6 +1137,7 @@ H5FD__subfiling_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t ma
     file_ptr->context_id     = -1;
     file_ptr->fa.ioc_fapl_id = H5I_INVALID_HID;
     file_ptr->ext_comm       = MPI_COMM_NULL;
+    file_ptr->fail_to_encode = FALSE;
 
     /* Get the driver-specific file access properties */
     if (NULL == (plist_ptr = (H5P_genplist_t *)H5I_object(fapl_id)))
@@ -1240,6 +1327,8 @@ H5FD__subfiling_close_int(H5FD_subfiling_t *file_ptr)
     if (H5_mpi_comm_free(&file_ptr->ext_comm) < 0)
         H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "can't free MPI communicator");
 
+    file_ptr->fail_to_encode = FALSE;
+
 done:
     HDfree(file_ptr->file_path);
     file_ptr->file_path = NULL;
diff --git a/src/H5FDsubfiling/H5FDsubfiling.h b/src/H5FDsubfiling/H5FDsubfiling.h
index 22dd906..b1cf53a 100644
--- a/src/H5FDsubfiling/H5FDsubfiling.h
+++ b/src/H5FDsubfiling/H5FDsubfiling.h
@@ -148,6 +148,27 @@
  * pathname.
  */
 #define H5FD_SUBFILING_SUBFILE_PREFIX "H5FD_SUBFILING_SUBFILE_PREFIX"
+/**
+ * \def H5FD_SUBFILING_CONFIG_FILE_PREFIX
+ * Macro for name of the environment variable that specifies a prefix
+ * to apply to the subfiling configuration filename. Useful for cases
+ * where the application wants to place the configuration file in a
+ * different directory than the default of putting it alongside the
+ * generated subfiles. For example, when writing to node-local storage
+ * one may wish to place the configuration file on a scratch file
+ * system readable by all nodes, while the subfiles are initially
+ * written to the node-local storage.
+ *
+ * The value set for this environment variable is interpreted as a
+ * pathname that must already exist.
+ *
+ * NOTE: As this prefix string will be encoded in the driver info
+ *       message that gets written to the file, there is an upper
+ *       limit of about ~900 single-byte characters for this string,
+ *       though possibly less due to other information the driver
+ *       may encode. Avoid long prefix names where possible.
+ */
+#define H5FD_SUBFILING_CONFIG_FILE_PREFIX "H5FD_SUBFILING_CONFIG_FILE_PREFIX"
 
 /**
  * \enum H5FD_subfiling_ioc_select_t
diff --git a/src/H5FDsubfiling/H5subfiling_common.c b/src/H5FDsubfiling/H5subfiling_common.c
index 3e83cf5..b58c4d3 100644
--- a/src/H5FDsubfiling/H5subfiling_common.c
+++ b/src/H5FDsubfiling/H5subfiling_common.c
@@ -73,8 +73,8 @@ static herr_t record_fid_to_subfile(uint64_t file_id, int64_t subfile_context_id
 static void   clear_fid_map_entry(uint64_t file_id, int64_t sf_context_id);
 static herr_t ioc_open_files(int64_t file_context_id, int file_acc_flags);
 static herr_t create_config_file(subfiling_context_t *sf_context, const char *base_filename,
-                                 const char *subfile_dir, hbool_t truncate_if_exists);
-static herr_t open_config_file(const char *base_filename, const char *subfile_dir, uint64_t file_id,
+                                 const char *config_dir, const char *subfile_dir, hbool_t truncate_if_exists);
+static herr_t open_config_file(const char *base_filename, const char *config_dir, uint64_t file_id,
                                const char *mode, FILE **config_file_out);
 
 /*-------------------------------------------------------------------------
@@ -383,6 +383,9 @@ H5_free_subfiling_object_int(subfiling_context_t *sf_context)
     HDfree(sf_context->subfile_prefix);
     sf_context->subfile_prefix = NULL;
 
+    HDfree(sf_context->config_file_prefix);
+    sf_context->config_file_prefix = NULL;
+
     HDfree(sf_context->h5_filename);
     sf_context->h5_filename = NULL;
 
@@ -721,6 +724,7 @@ init_subfiling(const char *base_filename, uint64_t file_id, H5FD_subfiling_param
     FILE                *config_file   = NULL;
     char                *file_basename = NULL;
     char                *subfile_dir   = NULL;
+    char                *prefix_env    = NULL;
     int                  mpi_rank;
     int                  mpi_size;
     int                  mpi_code;
@@ -748,6 +752,13 @@ init_subfiling(const char *base_filename, uint64_t file_id, H5FD_subfiling_param
     new_context->sf_node_comm  = MPI_COMM_NULL;
     new_context->sf_group_comm = MPI_COMM_NULL;
 
+    /* Check if a prefix has been set for the configuration file name */
+    prefix_env = HDgetenv(H5FD_SUBFILING_CONFIG_FILE_PREFIX);
+    if (prefix_env) {
+        if (NULL == (new_context->config_file_prefix = HDstrdup(prefix_env)))
+            H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTCOPY, FAIL, "couldn't copy config file prefix string");
+    }
+
     /*
      * If there's an existing subfiling configuration file for
      * this file, read the stripe size and number of subfiles
@@ -767,7 +778,12 @@ init_subfiling(const char *base_filename, uint64_t file_id, H5FD_subfiling_param
             }
 
             if (config[0] >= 0) {
-                if (open_config_file(file_basename, subfile_dir, file_id, "r", &config_file) < 0)
+                /*
+                 * If a prefix has been specified, try to read the config file
+                 * from there, otherwise look for it next to the generated subfiles.
+                 */
+                if (open_config_file(file_basename, prefix_env ? prefix_env : subfile_dir, file_id, "r",
+                                     &config_file) < 0)
                     config[0] = -1;
             }
 
@@ -2133,7 +2149,19 @@ ioc_open_files(int64_t file_context_id, int file_acc_flags)
      * check if we also need to create a config file.
      */
     if ((file_acc_flags & O_CREAT) && (sf_context->topology->ioc_idx == 0)) {
-        if (create_config_file(sf_context, base, subfile_dir, (file_acc_flags & O_TRUNC)) < 0)
+        char *config_dir = NULL;
+
+        /*
+         * If a config file prefix has been specified, place the
+         * config file there, otherwise place it next to the
+         * generated subfiles.
+         */
+        if (sf_context->config_file_prefix)
+            config_dir = sf_context->config_file_prefix;
+        else
+            config_dir = subfile_dir;
+
+        if (create_config_file(sf_context, base, config_dir, subfile_dir, (file_acc_flags & O_TRUNC)) < 0)
             H5_SUBFILING_GOTO_ERROR(H5E_FILE, H5E_CANTCREATE, FAIL,
                                     "couldn't create subfiling configuration file");
     }
@@ -2174,8 +2202,8 @@ done:
  *-------------------------------------------------------------------------
  */
 static herr_t
-create_config_file(subfiling_context_t *sf_context, const char *base_filename, const char *subfile_dir,
-                   hbool_t truncate_if_exists)
+create_config_file(subfiling_context_t *sf_context, const char *base_filename, const char *config_dir,
+                   const char *subfile_dir, hbool_t truncate_if_exists)
 {
     hbool_t config_file_exists = FALSE;
     FILE   *config_file        = NULL;
@@ -2186,6 +2214,7 @@ create_config_file(subfiling_context_t *sf_context, const char *base_filename, c
 
     HDassert(sf_context);
     HDassert(base_filename);
+    HDassert(config_dir);
     HDassert(subfile_dir);
 
     if (sf_context->h5_file_id == UINT64_MAX)
@@ -2194,6 +2223,8 @@ create_config_file(subfiling_context_t *sf_context, const char *base_filename, c
     if (*base_filename == '\0')
         H5_SUBFILING_GOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "invalid base HDF5 filename '%s'",
                                 base_filename);
+    if (*config_dir == '\0')
+        config_dir = ".";
     if (*subfile_dir == '\0')
         subfile_dir = ".";
 
@@ -2201,7 +2232,7 @@ create_config_file(subfiling_context_t *sf_context, const char *base_filename, c
         H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL,
                                 "couldn't allocate space for subfiling configuration filename");
 
-    HDsnprintf(config_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, subfile_dir,
+    HDsnprintf(config_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, config_dir,
                base_filename, sf_context->h5_file_id);
 
     /* Determine whether a subfiling configuration file exists */
@@ -2226,7 +2257,7 @@ create_config_file(subfiling_context_t *sf_context, const char *base_filename, c
 
         if (NULL == (config_file = HDfopen(config_filename, "w+")))
             H5_SUBFILING_SYS_GOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL,
-                                        "couldn't open subfiling configuration file");
+                                        "couldn't create/truncate subfiling configuration file");
 
         if (NULL == (line_buf = HDmalloc(PATH_MAX)))
             H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL,
@@ -2302,7 +2333,7 @@ done:
  *-------------------------------------------------------------------------
  */
 static herr_t
-open_config_file(const char *base_filename, const char *subfile_dir, uint64_t file_id, const char *mode,
+open_config_file(const char *base_filename, const char *config_dir, uint64_t file_id, const char *mode,
                  FILE **config_file_out)
 {
     hbool_t config_file_exists = FALSE;
@@ -2312,7 +2343,7 @@ open_config_file(const char *base_filename, const char *subfile_dir, uint64_t fi
     herr_t  ret_value          = SUCCEED;
 
     HDassert(base_filename);
-    HDassert(subfile_dir);
+    HDassert(config_dir);
     HDassert(file_id != UINT64_MAX);
     HDassert(mode);
     HDassert(config_file_out);
@@ -2322,14 +2353,14 @@ open_config_file(const char *base_filename, const char *subfile_dir, uint64_t fi
     if (*base_filename == '\0')
         H5_SUBFILING_GOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "invalid base HDF5 filename '%s'",
                                 base_filename);
-    if (*subfile_dir == '\0')
-        subfile_dir = ".";
+    if (*config_dir == '\0')
+        config_dir = ".";
 
     if (NULL == (config_filename = HDmalloc(PATH_MAX)))
         H5_SUBFILING_GOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL,
                                 "couldn't allocate space for subfiling configuration filename");
 
-    HDsnprintf(config_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, subfile_dir,
+    HDsnprintf(config_filename, PATH_MAX, "%s/" H5FD_SUBFILING_CONFIG_FILENAME_TEMPLATE, config_dir,
                base_filename, file_id);
 
     /* Determine whether a subfiling configuration file exists */
diff --git a/src/H5FDsubfiling/H5subfiling_common.h b/src/H5FDsubfiling/H5subfiling_common.h
index a5d6f61..51d8b22 100644
--- a/src/H5FDsubfiling/H5subfiling_common.h
+++ b/src/H5FDsubfiling/H5subfiling_common.h
@@ -208,6 +208,7 @@ typedef struct {
     int            sf_group_size;           /* IOC count (in sf_group_comm)                      */
     int            sf_group_rank;           /* IOC rank  (in sf_group_comm)                      */
     char          *subfile_prefix;          /* If subfiles are node-local                        */
+    char          *config_file_prefix;      /* Prefix added to config file name                  */
     char          *h5_filename;             /* The user supplied file name                       */
     void          *ioc_data;                /* Private data for underlying IOC                   */
     sf_topology_t *topology;                /* Pointer to our topology                           */