Perform the dataset opens in reverse order to their creation, and if

H5Dopen fails, rapidly retry up to 9,999 times.  Log H5Dopen failures,
but log no more than once every five seconds to avoid spamming the
terminal.

With these changes, it's easier for the reader to open the last dataset
before the writer created it, but the reader recovers instead of
quitting with an error.  It should only be necessary to retry opening
the *last* dataset; all previous datasets should open on one try if the
last is open.

1 files changed, 49 insertions, 5 deletions

diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c
index 6d65c8b..701c6bc 100644
--- a/test/vfd_swmr_bigset_writer.c
+++ b/test/vfd_swmr_bigset_writer.c
@@ -640,22 +640,66 @@ close_extensible_dset(state_t *s, unsigned int which)
     }
 }
 
+static bool
+below_speed_limit(struct timespec *last, const struct timespec *ival)
+{
+    struct timespec now;
+    bool result;
+
+    assert(0 <= last->tv_nsec && last->tv_nsec < 1000000000L);
+    assert(0 <= ival->tv_nsec && ival->tv_nsec < 1000000000L);
+
+    if (clock_gettime(CLOCK_MONOTONIC, &now) == -1)
+        err(EXIT_FAILURE, "%s: clock_gettime", __func__);
+
+    if (now.tv_sec - last->tv_sec > ival->tv_sec)
+        result = true;
+    else if (now.tv_sec - last->tv_sec < ival->tv_sec)
+        return false;
+    else
+        result = (now.tv_nsec - last->tv_nsec > ival->tv_nsec);
+
+    *last = now;
+
+    return result;
+}
+
 static void
 open_extensible_dset(state_t *s, unsigned int which)
 {
     hsize_t dims[RANK], maxdims[RANK];
     char dname[sizeof("/dataset-9999999999")];
     hid_t ds, filespace, ty;
+    const int tries = 10000;
+    int i;
+    struct timespec last = {0, 0};
+    static const struct timespec ival = {5, 0};
+    estack_state_t es;
 
     assert(which < s->ndatasets);
     assert(s->dataset[which] == badhid);
 
     esnprintf(dname, sizeof(dname), "/dataset-%d", which);
 
-    ds = H5Dopen(s->file[0], dname, s->dapl);
+    es = disable_estack();
+    for (i = 0; i < tries; i++) {
 
-    if (ds < 0)
-        errx(EXIT_FAILURE, "H5Dopen(, \"%s\", ) failed", dname);
+        ds = H5Dopen(s->file[0], dname, s->dapl);
+
+        if (ds >= 0)
+            break;
+
+        if (below_speed_limit(&last, &ival)) {
+            warnx("H5Dopen(, \"%s\", ) failed, %d retries remain",
+                dname, tries - i - 1);
+        }
+    }
+    restore_estack(es);
+
+    if (i == tries) {
+        errx(EXIT_FAILURE, "H5Dopen(, \"%s\", ) failed after %d tries",
+            dname, tries - i - 1);
+    }
 
     if ((ty = H5Dget_type(ds)) < 0)
         errx(EXIT_FAILURE, "H5Dget_type failed");
@@ -1070,8 +1114,8 @@ main(int argc, char **argv)
                 nanosleep(&s.update_interval, NULL);
         }
     } else {
-        for (which = 0; which < s.ndatasets; which++)
-            open_extensible_dset(&s, which);
+        for (which = s.ndatasets; which > 0; which--)
+            open_extensible_dset(&s, which - 1);
 
         for (step = 0; hang_back + step < s.nsteps;) {
             for (which = s.ndatasets; which-- > 0; ) {