Add my work in progress on variable-length string test for VFD SWMR.

3 files changed, 554 insertions, 1 deletions

diff --git a/test/Makefile.am b/test/Makefile.am
index db8e83d..ad63093 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -46,7 +46,8 @@ SCRIPT_DEPEND = error_test$(EXEEXT) err_compat$(EXEEXT) links_env$(EXEEXT) \
     vfd_swmr_generator$(EXEEXT) vfd_swmr_reader$(EXEEXT) vfd_swmr_writer$(EXEEXT)  \
     vfd_swmr_remove_reader$(EXEEXT) vfd_swmr_remove_writer$(EXEEXT)  \
     vfd_swmr_addrem_writer$(EXEEXT) vfd_swmr_sparse_reader$(EXEEXT) \
-    vfd_swmr_sparse_writer$(EXEEXT) \
+    vfd_swmr_sparse_writer$(EXEEXT) vfd_swmr_vlstr$(EXEEXT) \
+    vfd_swmr_vlstr_reader$(EXEEXT) \
     vds_env$(EXEEXT) \
     vds_swmr_gen$(EXEEXT) vds_swmr_reader$(EXEEXT) vds_swmr_writer$(EXEEXT)
 if HAVE_SHARED_CONDITIONAL
@@ -96,6 +97,7 @@ check_PROGRAMS=$(TEST_PROG) error_test err_compat tcheck_version \
     vfd_swmr_generator vfd_swmr_reader vfd_swmr_writer \
     vfd_swmr_remove_reader vfd_swmr_remove_writer vfd_swmr_addrem_writer \
     vfd_swmr_sparse_reader vfd_swmr_sparse_writer \
+    vfd_swmr_vlstr vfd_swmr_vlstr_reader \
     swmr_check_compat_vfd vds_env vds_swmr_gen vds_swmr_reader vds_swmr_writer
 if HAVE_SHARED_CONDITIONAL
   check_PROGRAMS+= filter_plugin vol_plugin
diff --git a/test/vfd_swmr_vlstr.c b/test/vfd_swmr_vlstr.c
new file mode 100644
index 0000000..3c73fd2
--- /dev/null
+++ b/test/vfd_swmr_vlstr.c
@@ -0,0 +1,328 @@
+/*
+ * Copyright by The HDF Group.
+ * Copyright by the Board of Trustees of the University of Illinois.
+ * All rights reserved.
+ *
+ * This file is part of HDF5.  The full HDF5 copyright notice, including
+ * terms governing use, modification, and redistribution, is contained in
+ * the COPYING file, which can be found at the root of the source code
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
+ * If you do not have access to either file, you may request a copy from
+ * help@hdfgroup.org.
+ */
+
+#include <err.h>
+#include <time.h> /* nanosleep(2) */
+#include <unistd.h> /* getopt(3) */
+
+#define H5C_FRIEND              /*suppress error about including H5Cpkg   */
+#define H5F_FRIEND              /*suppress error about including H5Fpkg   */
+
+#include "hdf5.h"
+
+#include "H5Cpkg.h"
+#include "H5Fpkg.h"
+// #include "H5Iprivate.h"
+#include "H5HGprivate.h"
+#include "H5VLprivate.h"
+
+#include "testhdf5.h"
+#include "vfd_swmr_common.h"
+
+enum _step {
+  CREATE = 0
+, LENGTHEN
+, SHORTEN
+, DELETE
+, NSTEPS
+} step_t;
+
+static const hid_t badhid = H5I_INVALID_HID; // abbreviate
+static bool caught_out_of_bounds = false;
+
+static void
+write_vl_dset(hid_t dset, hid_t type, hid_t space, char *data)
+{
+    if (H5Dwrite(dset, type, space, space, H5P_DEFAULT, &data) < 0)
+        errx(EXIT_FAILURE, "%s: H5Dwrite", __func__);
+    if (H5Dflush(dset) < 0)
+        errx(EXIT_FAILURE, "%s: H5Dflush", __func__);
+}
+
+#if 0
+static hid_t
+initialize_dset(hid_t file, hid_t type, hid_t space, const char *name,
+    void *data)
+{
+    hid_t dset;
+
+    dset = H5Dcreate2(file, name, type, space, H5P_DEFAULT, H5P_DEFAULT,
+        H5P_DEFAULT);
+
+    if (dset == badhid)
+        errx(EXIT_FAILURE, "H5Dcreate2");
+
+    if (H5Dwrite(dset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0)
+        errx(EXIT_FAILURE, "H5Dwrite");
+
+    if (H5Dflush(dset) < 0)
+        errx(EXIT_FAILURE, "%s: H5Dflush", __func__);
+
+    return dset;
+}
+
+static void
+rewrite_dset(hid_t dset, hid_t type, char *data)
+{
+    if (H5Dwrite(dset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0)
+        errx(EXIT_FAILURE, "%s: H5Dwrite", __func__);
+    if (H5Dflush(dset) < 0)
+        errx(EXIT_FAILURE, "%s: H5Dflush", __func__);
+}
+#endif
+
+static hid_t
+create_vl_dset(hid_t file, hid_t type, hid_t space, const char *name)
+{
+    hid_t dset;
+
+    dset = H5Dcreate2(file, name, type, space, H5P_DEFAULT, H5P_DEFAULT,
+        H5P_DEFAULT);
+
+    if (dset == badhid)
+        errx(EXIT_FAILURE, "H5Dcreate2");
+
+    return dset;
+}
+
+static void
+print_cache_hits(H5C_t *cache)
+{
+    int i;
+
+    for (i = 0; i < H5AC_NTYPES; i++) {
+        printf("type-%d cache hits %" PRId64 "%s\n",
+            i, cache->hits[i], (i == H5AC_GHEAP_ID) ? " *" : "");
+    }
+    printf("\n");
+}
+
+static void
+usage(const char *progname)
+{
+    fprintf(stderr, "usage: %s [-W] [-V]\n", progname);
+    fprintf(stderr, "\n  -W: do not wait for SIGUSR1\n");
+    fprintf(stderr,   "  -f: use fixed-length string\n");
+    fprintf(stderr,   "      (default: variable-length string)\n");
+    fprintf(stderr,   "  -n: number of test steps to perform\n");
+    exit(EXIT_FAILURE);
+}
+
+bool
+H5HG_trap(const char *reason)
+{
+    if (strcmp(reason, "out of bounds") == 0) {
+        caught_out_of_bounds = true;
+        return false;
+    }
+    return true;
+}
+
+int
+main(int argc, char **argv)
+{
+    hid_t fapl, fcpl, fid, space, type;
+    hid_t dset[2];
+    char content[2][96];
+    char name[2][96];
+    H5F_t *f;
+    H5C_t *cache;
+    H5F_vfd_swmr_config_t config;
+    sigset_t oldsigs;
+    herr_t ret;
+    bool variable = true, wait_for_signal = true;
+    const hsize_t dims = 1;
+    int ch, i, ntimes = 100;
+    unsigned long tmp;
+    char *end;
+    const struct timespec delay =
+        {.tv_sec = 0, .tv_nsec = 1000 * 1000 * 1000 / 10};
+
+    assert(H5T_C_S1 != badhid);
+
+    while ((ch = getopt(argc, argv, "Wfn:")) != -1) {
+        switch(ch) {
+        case 'W':
+            wait_for_signal = false;
+            break;
+        case 'f':
+            variable = false;
+            break;
+        case 'n':
+            errno = 0;
+            tmp = strtoul(optarg, &end, 0);
+            if (end == optarg || *end != '\0')
+                errx(EXIT_FAILURE, "couldn't parse `-n` argument `%s`", optarg);
+            else if (errno != 0)
+                err(EXIT_FAILURE, "couldn't parse `-n` argument `%s`", optarg);
+            else if (tmp > INT_MAX)
+                errx(EXIT_FAILURE, "`-n` argument `%lu` too large", tmp);
+            ntimes = (int)tmp;
+            break;
+        default:
+            usage(argv[0]);
+            break;
+        }
+    }
+    argv += optind;
+    argc -= optind;
+
+    if (argc > 0)
+        errx(EXIT_FAILURE, "unexpected command-line arguments");
+
+    /* Create file access property list */
+    if((fapl = h5_fileaccess()) < 0)
+        errx(EXIT_FAILURE, "h5_fileaccess");
+
+    /* FOR NOW: set to use latest format, the "old" parameter is not used */
+    if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
+        errx(EXIT_FAILURE, "H5Pset_libver_bounds");
+
+    /*
+     * Set up to open the file with VFD SWMR configured.
+     */
+
+    /* Enable page buffering */
+    if(H5Pset_page_buffer_size(fapl, 4096, 100, 0) < 0)
+        errx(EXIT_FAILURE, "H5Pset_page_buffer_size");
+
+    memset(&config, 0, sizeof(config));
+
+    config.version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
+    config.tick_len = 1;
+    config.max_lag = 5;
+    config.writer = true;
+    config.md_pages_reserved = 128;
+    HDstrcpy(config.md_file_path, "./my_md_file");
+
+    /* Enable VFD SWMR configuration */
+    if(H5Pset_vfd_swmr_config(fapl, &config) < 0)
+        errx(EXIT_FAILURE, "H5Pset_vfd_swmr_config");
+
+    if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
+        errx(EXIT_FAILURE, "H5Pcreate");
+
+    ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, false, 1);
+    if (ret < 0)
+        errx(EXIT_FAILURE, "H5Pset_file_space_strategy");
+
+    fid = H5Fcreate("vfd_swmr_vlstr.h5", H5F_ACC_TRUNC, fcpl, fapl);
+
+    /* Create the VL string datatype and a scalar dataspace, or a
+     * fixed-length string datatype and a simple dataspace.
+     */
+    if ((type = H5Tcopy(H5T_C_S1)) == badhid)
+        errx(EXIT_FAILURE, "H5Tcopy");
+
+    /* Create the VL string datatype and a scalar dataspace */
+    if ((type = H5Tcopy(H5T_C_S1)) == badhid)
+        errx(EXIT_FAILURE, "H5Tcopy");
+
+    if (!variable) {
+        if (H5Tset_size(type, 32) < 0)
+            errx(EXIT_FAILURE, "H5Tset_size");
+        space = H5Screate_simple(1, &dims, NULL);
+    } else {
+        if (H5Tset_size(type, H5T_VARIABLE) < 0)
+            errx(EXIT_FAILURE, "H5Tset_size");
+        space = H5Screate(H5S_SCALAR);
+    }
+
+    if (space == badhid)
+        errx(EXIT_FAILURE, "H5Screate");
+
+    if ((f = H5VL_object_verify(fid, H5I_FILE)) == NULL)
+        errx(EXIT_FAILURE, "H5VL_object_verify");
+
+    cache = f->shared->cache;
+
+    if (fid == badhid)
+        errx(EXIT_FAILURE, "H5Fcreate");
+
+    block_signals(&oldsigs);
+
+    print_cache_hits(cache);
+
+    /* content 1 seq 1 short
+     * content 1 seq 1 long long long long long long long long
+     * content 1 seq 1 medium medium medium
+     */
+    for (i = 0; i < ntimes; i++) {
+        const int ndsets = 2;
+        const int step = i % NSTEPS;
+        const int which = (i / NSTEPS) % ndsets;
+        const int seq = i / (ndsets * NSTEPS);
+        fprintf(stderr, "iteration %d which %d step %d seq %d\n",
+            i, which, step, seq);
+        switch (step) {
+        case CREATE:
+            (void)snprintf(name[which], sizeof(name[which]),
+                "dset-%d", which);
+            (void)snprintf(content[which], sizeof(content[which]),
+                "content %d seq %d short", which, seq);
+            dset[which] =
+                create_vl_dset(fid, type, space, name[which]);
+            write_vl_dset(dset[which], type, space, content[which]);
+            break;
+        case LENGTHEN:
+            (void)snprintf(content[which], sizeof(content[which]),
+                "content %d seq %d long long long long long long long long",
+                which, seq);
+            write_vl_dset(dset[which], type, space, content[which]);
+            break;
+        case SHORTEN:
+            (void)snprintf(content[which], sizeof(content[which]),
+                "content %d seq %d medium medium medium",
+                which, seq);
+            write_vl_dset(dset[which], type, space, content[which]);
+            break;
+        case DELETE:
+            if (H5Dclose(dset[which]) < 0)
+                errx(EXIT_FAILURE, "H5Dclose");
+            if (H5Ldelete(fid, name[which], H5P_DEFAULT) < 0) {
+                errx(EXIT_FAILURE, "%s: H5Ldelete(, \"%s\", ) failed",
+                    __func__, name[which]);
+            }
+            break;
+        default:
+            errx(EXIT_FAILURE, "%s: unknown step %d", __func__, step);
+        }
+        if (caught_out_of_bounds) {
+            fprintf(stderr, "caught out of bounds\n");
+            break;
+        }
+        nanosleep(&delay, NULL);
+    }
+
+    if (wait_for_signal)
+        await_signal(fid);
+
+    restore_signals(&oldsigs);
+
+    if (H5Pclose(fapl) < 0)
+        errx(EXIT_FAILURE, "H5Pclose(fapl)");
+
+    if (H5Pclose(fcpl) < 0)
+        errx(EXIT_FAILURE, "H5Pclose(fcpl)");
+
+    if (H5Tclose(type) < 0)
+        errx(EXIT_FAILURE, "H5Tclose");
+
+    if (H5Sclose(space) < 0)
+        errx(EXIT_FAILURE, "H5Sclose");
+
+    if (H5Fclose(fid) < 0)
+        errx(EXIT_FAILURE, "H5Fclose");
+
+    return EXIT_SUCCESS;
+}
diff --git a/test/vfd_swmr_vlstr_reader.c b/test/vfd_swmr_vlstr_reader.c
new file mode 100644
index 0000000..dcd4468
--- /dev/null
+++ b/test/vfd_swmr_vlstr_reader.c
@@ -0,0 +1,223 @@
+/*
+ * Copyright by The HDF Group.
+ * Copyright by the Board of Trustees of the University of Illinois.
+ * All rights reserved.
+ *
+ * This file is part of HDF5.  The full HDF5 copyright notice, including
+ * terms governing use, modification, and redistribution, is contained in
+ * the COPYING file, which can be found at the root of the source code
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
+ * If you do not have access to either file, you may request a copy from
+ * help@hdfgroup.org.
+ */
+
+#include <err.h>
+#include <time.h> /* nanosleep(2) */
+#include <unistd.h> /* getopt(3) */
+
+#define H5C_FRIEND              /*suppress error about including H5Cpkg   */
+#define H5F_FRIEND              /*suppress error about including H5Fpkg   */
+
+#include "hdf5.h"
+
+#include "H5Cpkg.h"
+#include "H5Fpkg.h"
+// #include "H5Iprivate.h"
+#include "H5HGprivate.h"
+#include "H5VLprivate.h"
+
+#include "testhdf5.h"
+#include "vfd_swmr_common.h"
+
+enum _step {
+  CREATE = 0
+, LENGTHEN
+, SHORTEN
+, DELETE
+, NSTEPS
+} step_t;
+
+static const hid_t badhid = H5I_INVALID_HID; // abbreviate
+static bool caught_out_of_bounds = false;
+
+static void
+read_vl_dset(hid_t dset, hid_t type, hid_t space, char *data)
+{
+    if (H5Dread(dset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, &data) < 0)
+        errx(EXIT_FAILURE, "%s: H5Dwrite", __func__);
+}
+
+static hid_t
+open_vl_dset(hid_t file, hid_t type, const char *name)
+{
+    hid_t dset;
+
+    dset = H5Dopen(file, name, H5P_DEFAULT);
+
+    if (dset == badhid)
+        errx(EXIT_FAILURE, "H5Dopen");
+
+    return dset;
+}
+
+static void
+usage(const char *progname)
+{
+    fprintf(stderr, "usage: %s [-W] [-V]\n", progname);
+    fprintf(stderr, "\n  -W: do not wait for SIGUSR1\n");
+    fprintf(stderr,   "  -n: number of test steps to perform\n");
+    exit(EXIT_FAILURE);
+}
+
+bool
+H5HG_trap(const char *reason)
+{
+    if (strcmp(reason, "out of bounds") == 0) {
+        caught_out_of_bounds = true;
+        return false;
+    }
+    return true;
+}
+
+int
+main(int argc, char **argv)
+{
+    hid_t fapl, fid, space, type;
+    hid_t dset[2];
+    char content[2][96];
+    char name[2][96];
+    H5F_vfd_swmr_config_t config;
+    sigset_t oldsigs;
+    herr_t ret;
+    bool wait_for_signal = true;
+    const hsize_t dims = 1;
+    int ch, i, ntimes = 100;
+    unsigned long tmp;
+    char *end;
+    const struct timespec delay =
+        {.tv_sec = 0, .tv_nsec = 1000 * 1000 * 1000 / 10};
+
+    assert(H5T_C_S1 != badhid);
+
+    while ((ch = getopt(argc, argv, "Wn:")) != -1) {
+        switch(ch) {
+        case 'W':
+            wait_for_signal = false;
+            break;
+        case 'n':
+            errno = 0;
+            tmp = strtoul(optarg, &end, 0);
+            if (end == optarg || *end != '\0')
+                errx(EXIT_FAILURE, "couldn't parse `-n` argument `%s`", optarg);
+            else if (errno != 0)
+                err(EXIT_FAILURE, "couldn't parse `-n` argument `%s`", optarg);
+            else if (tmp > INT_MAX)
+                errx(EXIT_FAILURE, "`-n` argument `%lu` too large", tmp);
+            ntimes = (int)tmp;
+            break;
+        default:
+            usage(argv[0]);
+            break;
+        }
+    }
+    argv += optind;
+    argc -= optind;
+
+    if (argc > 0)
+        errx(EXIT_FAILURE, "unexpected command-line arguments");
+
+    /* Create file access property list */
+    if((fapl = h5_fileaccess()) < 0)
+        errx(EXIT_FAILURE, "h5_fileaccess");
+
+    /* FOR NOW: set to use latest format, the "old" parameter is not used */
+    if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
+        errx(EXIT_FAILURE, "H5Pset_libver_bounds");
+
+    /*
+     * Set up to open the file with VFD SWMR configured.
+     */
+
+    /* Enable page buffering */
+    if(H5Pset_page_buffer_size(fapl, 4096, 100, 0) < 0)
+        errx(EXIT_FAILURE, "H5Pset_page_buffer_size");
+
+    memset(&config, 0, sizeof(config));
+
+    config.version = H5F__CURR_VFD_SWMR_CONFIG_VERSION;
+    config.tick_len = 1;
+    config.max_lag = 5;
+    config.writer = true;
+    config.md_pages_reserved = 128;
+    HDstrcpy(config.md_file_path, "./my_md_file");
+
+    /* Enable VFD SWMR configuration */
+    if(H5Pset_vfd_swmr_config(fapl, &config) < 0)
+        errx(EXIT_FAILURE, "H5Pset_vfd_swmr_config");
+
+    fid = H5Fopen("vfd_swmr_vlstr.h5", H5F_ACC_RDONLY, fapl);
+
+    /* Create the VL string datatype and a scalar dataspace, or a
+     * fixed-length string datatype and a simple dataspace.
+     */
+    if ((type = H5Tcopy(H5T_C_S1)) == badhid)
+        errx(EXIT_FAILURE, "H5Tcopy");
+
+    /* Create the VL string datatype and a scalar dataspace */
+    if ((type = H5Tcopy(H5T_C_S1)) == badhid)
+        errx(EXIT_FAILURE, "H5Tcopy");
+
+    if (H5Tset_size(type, H5T_VARIABLE) < 0)
+        errx(EXIT_FAILURE, "H5Tset_size");
+    space = H5Screate(H5S_SCALAR);
+
+    if (space == badhid)
+        errx(EXIT_FAILURE, "H5Screate");
+
+    if (fid == badhid)
+        errx(EXIT_FAILURE, "H5Fcreate");
+
+    block_signals(&oldsigs);
+
+    /* content 1 seq 1 short
+     * content 1 seq 1 long long long long long long long long
+     * content 1 seq 1 medium medium medium
+     */
+    for (i = 0; i < ntimes; i++) {
+        const int ndsets = 2;
+        const int which = i % ndsets;
+        fprintf(stderr, "iteration %d which %d\n", i, which);
+        (void)snprintf(name[which], sizeof(name[which]),
+            "dset-%d", which);
+        dset[which] = open_vl_dset(fid, type, name[which]);
+        read_vl_dset(dset[which], type, space, content[which]);
+#if 0
+        (void)snprintf(content[which], sizeof(content[which]),
+            "content %d seq %d %s", which, seq, tail);
+#endif
+        if (caught_out_of_bounds) {
+            fprintf(stderr, "caught out of bounds\n");
+            break;
+        }
+        nanosleep(&delay, NULL);
+    }
+
+    if (wait_for_signal)
+        await_signal(fid);
+
+    restore_signals(&oldsigs);
+
+    if (H5Pclose(fapl) < 0)
+        errx(EXIT_FAILURE, "H5Pclose(fapl)");
+
+    if (H5Tclose(type) < 0)
+        errx(EXIT_FAILURE, "H5Tclose");
+
+    if (H5Sclose(space) < 0)
+        errx(EXIT_FAILURE, "H5Sclose");
+
+    if (H5Fclose(fid) < 0)
+        errx(EXIT_FAILURE, "H5Fclose");
+
+    return EXIT_SUCCESS;
+}