From 79fc79c416e2650a0d28304cc5c09b85543b3b96 Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Thu, 3 Jun 2021 10:52:56 -0500 Subject: Some changes to the big set test: 1. changed the signal handling to named pipes for communication between the writer and reader; 2. the writer writes as many chunks within a tick then notify the reader to verify the data; 3. some refactoring work. --- test/testvfdswmr.sh.in | 30 +- test/vfd_swmr_bigset_writer.c | 1624 ++++++++++++++++++++++++++++++----------- 2 files changed, 1219 insertions(+), 435 deletions(-) diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in index ea2fd7b..1fccfe6 100644 --- a/test/testvfdswmr.sh.in +++ b/test/testvfdswmr.sh.in @@ -53,7 +53,7 @@ fi ##Default setting BIGSET_n=25 # -n option: # of iterations BIGSET_few_s=20 # -s option: # of datasets (for few_big test) -BIGSET_many_s=500 # -s option: # of datasets (for many_small test) +BIGSET_many_s=200 # -s option: # of datasets (for many_small test) GROUP_n=40 # -n option: # of groups (for group test) GROUP_attr_n=1 # -n option: # of groups (for group attribute test) GROUP_op_n=1 # -n option: # of groups (for group attribute test) @@ -61,7 +61,7 @@ GROUP_op_n=1 # -n option: # of groups (for group attribut if [[ "$HDF5TestExpress" -eq 0 ]] ; then # Setting for exhaustive run BIGSET_n=50 BIGSET_few_s=40 - BIGSET_many_s=1000 + BIGSET_many_s=400 GROUP_n=400 GROUP_attr_n=2 GROUP_op_n=2 @@ -1032,8 +1032,7 @@ for options in ${os_grp_op_list[*]}; do rm -f vfd_swmr_group_reader.*.{out,rc} done - - +# bigset test for smaller chunks for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do if [ ${do_many_small:-no} = no ]; then continue @@ -1041,7 +1040,7 @@ for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F # # Test many small datasets of one and two dimensions. # - # Perform 50 iterations on 1000 extensible datasets configured with + # Perform 25 iterations on 200 extensible datasets configured with # 16x16 chunks of 32-bit unsigned integer elements, # expanding each dataset by a chunk in one dimension (up to 50x1 # 16x16 chunks) on each iteration. @@ -1051,19 +1050,18 @@ for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F # echo launch vfd_swmr_bigset_writer many small, options $options catch_out_err_and_rc vfd_swmr_bigset_writer \ - ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_many_s -r 16 -c 16 -q & + ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_many_s -r 16 -c 16 -q -l 6 & pid_writer=$! catch_out_err_and_rc vfd_swmr_bigset_reader \ - ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_many_s -r 16 -c 16 -q -W & + ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_many_s -r 16 -c 16 -q -l 6 & pid_reader=$! - # Wait for the reader to finish before signalling the - # writer to quit: the writer holds the file open so that the + # Wait for the reader to finish before the + # writer quits: the writer holds the file open so that the # reader will find the shadow file when it opens # the .h5 file. wait $pid_reader - kill -USR1 $(cat vfd_swmr_bigset_writer.pid) wait $pid_writer # Collect exit code of the reader @@ -1083,11 +1081,12 @@ for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F rm -f vfd_swmr_bigset_reader.*.{out,rc} done +# bigset test for bigger chunks for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do # # Test a few big datasets of one and two dimensions. # - # Perform 50 iterations on 5 extensible datasets configured with + # Perform 25 iterations on 20 extensible datasets configured with # 256x256 chunks of 32-bit unsigned integer elements, # expanding each dataset by a chunk in one dimension (up to 50x1 # 256x256 chunks) on each iteration. @@ -1100,19 +1099,18 @@ for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F fi echo launch vfd_swmr_bigset_writer few big, options $options catch_out_err_and_rc vfd_swmr_bigset_writer \ - ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_few_s -r 256 -c 256 -q & + ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_few_s -r 256 -c 256 -q -l 3 & pid_writer=$! catch_out_err_and_rc vfd_swmr_bigset_reader \ - ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_few_s -r 256 -c 256 -q -W & + ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_few_s -r 256 -c 256 -q -l 3 & pid_reader=$! - # Wait for the reader to finish before signalling the - # writer to quit: the writer holds the file open so that the + # Wait for the reader to finish before the + # writer quits: the writer holds the file open so that the # reader will find the shadow file when it opens # the .h5 file. wait $pid_reader - kill -USR1 $(cat vfd_swmr_bigset_writer.pid) wait $pid_writer # Collect exit code of the reader diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c index 614a499..138b796 100644 --- a/test/vfd_swmr_bigset_writer.c +++ b/test/vfd_swmr_bigset_writer.c @@ -81,14 +81,13 @@ #ifndef H5_HAVE_WIN32_API -#include -#include - -#define ROWS 256 -#define COLS 512 -#define RANK 2 - -static const unsigned int hang_back = 3; +#define MAX_READ_LEN_IN_SECONDS 2 +#define TICK_LEN 4 +#define MAX_LAG 7 +#define ROWS 256 +#define COLS 512 +#define RANK 2 +#define NUM_ATTEMPTS 100 typedef struct _base { hsize_t row, col; @@ -121,7 +120,6 @@ typedef struct { unsigned ndatasets; const char * filename[MANY_FILES]; char progname[PATH_MAX]; - struct timespec update_interval; struct { quadrant_t ul, ur, bl, br, src; } quadrants; @@ -129,15 +127,41 @@ typedef struct { unsigned int asteps; unsigned int nsteps; bool two_dee; - bool wait_for_signal; enum { vds_off, vds_single, vds_multi } vds; bool use_vfd_swmr; + bool use_named_pipe; bool writer; bool fixed_array; hsize_t chunk_dims[RANK]; hsize_t one_dee_max_dims[RANK]; + struct timespec ival; } state_t; +/* Structure to hold info for named pipes */ +typedef struct { + const char *fifo_writer_to_reader; /* Name of fifo for writer to reader */ + const char *fifo_reader_to_writer; /* Name of fifo for reader to writer */ + int fd_writer_to_reader; /* File ID for fifo from writer to reader */ + int fd_reader_to_writer; /* File ID for fifo from reader to writer */ + int notify; /* Value to notify between writer and reader */ + int verify; /* Value to verify between writer and reader */ +} np_state_t; + +typedef struct { + unsigned step; + struct timespec time; +} exchange_info_t; + +/* Initializations for np_state_t */ +#define NP_INITIALIZER (np_state_t) { \ + .fifo_writer_to_reader = "./fifo_bigset_writer_to_reader" \ + , .fifo_reader_to_writer = "./fifo_bigset_reader_to_writer" \ + , .fd_writer_to_reader = -1 \ + , .fd_reader_to_writer = -1 \ + , .notify = 0 \ + , .verify = 0 \ +} + static inline state_t state_initializer(void) { @@ -154,18 +178,17 @@ state_initializer(void) .nsteps = 100, .filename = {"", "", "", ""}, .two_dee = false, - .wait_for_signal = true, .vds = vds_off, .use_vfd_swmr = true, + .use_named_pipe = true, .writer = true, .fixed_array = false, .one_dee_max_dims = {ROWS, H5S_UNLIMITED}, .chunk_dims = {ROWS, COLS}, - .update_interval = - (struct timespec){.tv_sec = 0, .tv_nsec = 1000000000UL / 30 /* 1/30 second */}}; + .ival = (struct timespec){.tv_sec = MAX_READ_LEN_IN_SECONDS, .tv_nsec = 0}}; } -static void state_init(state_t *, int, char **); +static bool state_init(state_t *, int, char **); static const hid_t badhid = H5I_INVALID_HID; @@ -174,17 +197,23 @@ static hsize_t two_dee_max_dims[RANK]; static uint32_t matget(const mat_t *mat, unsigned i, unsigned j) { - assert(i < mat->rows && j < mat->cols); - return mat->elt[i * mat->cols + j]; } -static void +static bool matset(mat_t *mat, unsigned i, unsigned j, uint32_t v) { - assert(i < mat->rows && j < mat->cols); + if (i >= mat->rows || j >= mat->cols) { + fprintf(stderr, "index out of boundary\n"); + TEST_ERROR; + } mat->elt[i * mat->cols + j] = v; + + return true; + +error: + return false; } static mat_t * @@ -192,15 +221,20 @@ newmat(unsigned rows, unsigned cols) { mat_t *mat; - mat = malloc(sizeof(*mat) + (rows * cols - 1) * sizeof(mat->elt[0])); + mat = HDmalloc(sizeof(*mat) + (rows * cols - 1) * sizeof(mat->elt[0])); - if (mat == NULL) - err(EXIT_FAILURE, "%s: malloc", __func__); + if (mat == NULL) { + fprintf(stderr, "HDmalloc failed\n"); + TEST_ERROR; + } mat->rows = rows; mat->cols = cols; return mat; + +error: + return NULL; } static void @@ -209,7 +243,7 @@ usage(const char *progname) fprintf(stderr, "usage: %s [-F] [-M] [-S] [-V] [-W] [-a steps] [-b] [-c cols]\n" " [-d dims]\n" - " [-n iterations] [-r rows] [-s datasets]\n" + " [-l tick_num] [-n iterations] [-r rows] [-s datasets]\n" " [-u milliseconds]\n" "\n" "-F: fixed maximal dimension for the chunked datasets\n" @@ -218,56 +252,72 @@ usage(const char *progname) "-S: do not use VFD SWMR\n" "-V: use virtual datasets and a single\n" " source file\n" - "-W: do not wait for a signal before\n" - " exiting\n" "-a steps: `steps` between adding attributes\n" "-b: write data in big-endian byte order\n" "-c cols: `cols` columns per chunk\n" "-d 1|one|2|two|both: select dataset expansion in one or\n" " both dimensions\n" + "-l tick_num: expected maximal number of ticks from\n" + " the writer's finishing creation to the reader's finishing validation\n" + "-N: do not use named pipes\n" "-n iterations: how many times to expand each dataset\n" "-r rows: `rows` rows per chunk\n" "-s datasets: number of datasets to create\n" - "-u ms: milliseconds interval between updates\n" - " to %s.h5\n" "\n", - progname, progname); + progname); exit(EXIT_FAILURE); } -static void +static bool make_quadrant_dataspace(state_t *s, quadrant_t *q) { - q->space = H5Screate_simple(NELMTS(s->chunk_dims), s->chunk_dims, - s->two_dee ? two_dee_max_dims : s->one_dee_max_dims); + if ((q->space = H5Screate_simple(NELMTS(s->chunk_dims), s->chunk_dims, + s->two_dee ? two_dee_max_dims : s->one_dee_max_dims)) < 0) { + fprintf(stderr, "H5Screate_simple failed\n"); + TEST_ERROR; + } - if (q->space < 0) { - errx(EXIT_FAILURE, "%s.%d: H5Screate_simple failed", __func__, __LINE__); + if (H5Sselect_hyperslab(q->space, H5S_SELECT_SET, q->start, q->stride, q->count, q->block) < 0) { + fprintf(stderr, "H5Sselect_hyperslab failed\n"); + TEST_ERROR; } - if (H5Sselect_hyperslab(q->space, H5S_SELECT_SET, q->start, q->stride, q->count, q->block) < 0) - errx(EXIT_FAILURE, "%s: H5Sselect_hyperslab failed", __func__); + return true; + +error: + H5E_BEGIN_TRY { + H5Sclose(q->space); + } H5E_END_TRY; + + return false; } -static void +static bool state_init(state_t *s, int argc, char **argv) { unsigned long tmp; int ch; unsigned i; const hsize_t dims = 1; - char tfile[PATH_MAX]; + char *tfile = NULL; char * end; - unsigned long millis; quadrant_t *const ul = &s->quadrants.ul, *const ur = &s->quadrants.ur, *const bl = &s->quadrants.bl, *const br = &s->quadrants.br, *const src = &s->quadrants.src; const char *personality; *s = state_initializer(); - esnprintf(tfile, sizeof(tfile), "%s", argv[0]); - esnprintf(s->progname, sizeof(s->progname), "%s", basename(tfile)); - while ((ch = getopt(argc, argv, "FMSVWa:bc:d:n:qr:s:u:")) != -1) { + if (H5_basename(argv[0], &tfile) < 0) { + fprintf(stderr, "H5_basename failed\n"); + TEST_ERROR; + } + + esnprintf(s->progname, sizeof(s->progname), "%s", tfile); + + if (tfile) + HDfree(tfile); + + while ((ch = getopt(argc, argv, "FMNSVa:bc:d:l:n:qr:s:")) != -1) { switch (ch) { case 'F': /* The flag to indicate whether the maximal dimension of the chunked datasets is fixed or @@ -283,8 +333,9 @@ state_init(state_t *s, int argc, char **argv) case 'V': s->vds = vds_single; break; - case 'W': - s->wait_for_signal = false; + case 'N': + /* Disable named pipes, mainly for running the writer and reader seperately */ + s->use_named_pipe = false; break; case 'd': if (strcmp(optarg, "1") == 0 || strcmp(optarg, "one") == 0) @@ -293,34 +344,49 @@ state_init(state_t *s, int argc, char **argv) strcmp(optarg, "both") == 0) s->two_dee = true; else { - errx(EXIT_FAILURE, "bad -d argument \"%s\"", optarg); + fprintf(stderr, "bad -d argument %s\n", optarg); + TEST_ERROR; } break; case 'a': case 'c': + case 'l': case 'n': case 'r': case 's': errno = 0; - tmp = strtoul(optarg, &end, 0); + tmp = HDstrtoul(optarg, &end, 0); if (end == optarg || *end != '\0') { - errx(EXIT_FAILURE, "couldn't parse `-%c` argument `%s`", ch, optarg); + fprintf(stderr, "couldn't parse -%c argument %s\n", ch, optarg); + TEST_ERROR; } else if (errno != 0) { - err(EXIT_FAILURE, "couldn't parse `-%c` argument `%s`", ch, optarg); + fprintf(stderr, "couldn't parse -%c argument %s\n", ch, optarg); + TEST_ERROR; + } + else if (tmp > UINT_MAX) { + fprintf(stderr, "-%c argument %lu too large", ch, tmp); + TEST_ERROR; } - else if (tmp > UINT_MAX) - errx(EXIT_FAILURE, "`-%c` argument `%lu` too large", ch, tmp); if ((ch == 'c' || ch == 'r') && tmp == 0) { - errx(EXIT_FAILURE, "`-%c` argument `%lu` must be >= 1", ch, tmp); + fprintf(stderr, "-%c argument %lu must be >= 1", ch, tmp); + TEST_ERROR; } if (ch == 'a') s->asteps = (unsigned)tmp; else if (ch == 'c') s->cols = (unsigned)tmp; - else if (ch == 'n') + else if (ch == 'l') { + /* Translate the tick number to time represented by the timespec struct */ + float time = (float)(((unsigned)tmp * TICK_LEN) / 10.0); + unsigned sec = (unsigned)time; + unsigned nsec = (unsigned)((time - sec) * 10 * 1000 * 1000); + + s->ival.tv_sec = sec; + s->ival.tv_nsec = nsec; + } else if (ch == 'n') s->nsteps = (unsigned)tmp; else if (ch == 'r') s->rows = (unsigned)tmp; @@ -333,18 +399,6 @@ state_init(state_t *s, int argc, char **argv) case 'q': verbosity = 0; break; - case 'u': - errno = 0; - millis = strtoul(optarg, &end, 0); - if (millis == ULONG_MAX && errno == ERANGE) { - err(EXIT_FAILURE, "option -p argument \"%s\"", optarg); - } - else if (*end != '\0') { - errx(EXIT_FAILURE, "garbage after -p argument \"%s\"", optarg); - } - s->update_interval.tv_sec = (time_t)(millis / 1000UL); - s->update_interval.tv_nsec = (long)((millis * 1000000UL) % 1000000000UL); - break; case '?': default: usage(s->progname); @@ -354,11 +408,14 @@ state_init(state_t *s, int argc, char **argv) argc -= optind; argv += optind; - if (argc > 0) - errx(EXIT_FAILURE, "unexpected command-line arguments"); + if (argc > 0) { + fprintf(stderr, "unexpected command-line arguments\n"); + TEST_ERROR; + } if (s->vds != vds_off && s->two_dee) { - errx(EXIT_FAILURE, "virtual datasets and 2D datasets are mutually exclusive"); + fprintf(stderr, "virtual datasets and 2D datasets are mutually exclusive\n"); + TEST_ERROR; } s->chunk_dims[0] = s->rows; @@ -388,11 +445,14 @@ state_init(state_t *s, int argc, char **argv) } if ((s->quadrant_dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) { - errx(EXIT_FAILURE, "%s.%d: H5Pcreate failed", __func__, __LINE__); + fprintf(stderr, "H5Pcreate failed\n"); + TEST_ERROR; } - if (H5Pset_chunk(s->quadrant_dcpl, RANK, half_chunk_dims) < 0) - errx(EXIT_FAILURE, "H5Pset_chunk failed"); + if (H5Pset_chunk(s->quadrant_dcpl, RANK, half_chunk_dims) < 0) { + fprintf(stderr, "H5Pset_chunk failed\n"); + TEST_ERROR; + } *ul = (quadrant_t){.start = {0, 0}, .stride = {s->rows, s->cols}, @@ -414,72 +474,86 @@ state_init(state_t *s, int argc, char **argv) .block = {s->rows / 2, s->cols / 2}, .count = {1, H5S_UNLIMITED}}; - make_quadrant_dataspace(s, ul); - make_quadrant_dataspace(s, ur); - make_quadrant_dataspace(s, bl); - make_quadrant_dataspace(s, br); + if (!make_quadrant_dataspace(s, ul)) { + fprintf(stderr, "make_quadrant_dataspace failed\n"); + TEST_ERROR; + } + + if (!make_quadrant_dataspace(s, ur)) { + fprintf(stderr, "make_quadrant_dataspace failed\n"); + TEST_ERROR; + } + + if (!make_quadrant_dataspace(s, bl)) { + fprintf(stderr, "make_quadrant_dataspace failed\n"); + TEST_ERROR; + } + + if (!make_quadrant_dataspace(s, br)) { + fprintf(stderr, "make_quadrant_dataspace failed\n"); + TEST_ERROR; + } *src = (quadrant_t){.start = {0, 0}, .stride = {s->rows / 2, s->cols / 2}, .block = {s->rows / 2, s->cols / 2}, .count = {1, H5S_UNLIMITED}}; - src->space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims); - - if (src->space < 0) { - errx(EXIT_FAILURE, "%s.%d: H5Screate_simple failed", __func__, __LINE__); + if ((src->space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims)) < 0) { + fprintf(stderr, "H5Screate_simple failed\n"); + TEST_ERROR; } - if (H5Sselect_hyperslab(src->space, H5S_SELECT_SET, src->start, src->stride, src->count, src->block) < - 0) - errx(EXIT_FAILURE, "%s: H5Sselect_hyperslab failed", __func__); - - ul->src_space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims); - - if (ul->src_space < 0) { - errx(EXIT_FAILURE, "%s.%d: H5Screate_simple failed", __func__, __LINE__); + if (H5Sselect_hyperslab(src->space, H5S_SELECT_SET, src->start, src->stride, src->count, src->block) < 0) { + fprintf(stderr, "H5Sselect_hyperslab failed\n"); + TEST_ERROR; } - ur->src_space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims); - - if (ur->src_space < 0) { - errx(EXIT_FAILURE, "%s.%d: H5Screate_simple failed", __func__, __LINE__); + if ((ul->src_space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims)) < 0) { + fprintf(stderr, "H5Screate_simple failed\n"); + TEST_ERROR; } - bl->src_space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims); - - if (bl->src_space < 0) { - errx(EXIT_FAILURE, "%s.%d: H5Screate_simple failed", __func__, __LINE__); + if ((ur->src_space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims)) < 0) { + fprintf(stderr, "H5Screate_simple failed\n"); + TEST_ERROR; } - br->src_space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims); + if ((bl->src_space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims)) < 0) { + fprintf(stderr, "H5Screate_simple failed\n"); + TEST_ERROR; + } - if (br->src_space < 0) { - errx(EXIT_FAILURE, "%s.%d: H5Screate_simple failed", __func__, __LINE__); + if ((br->src_space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims)) < 0) { + fprintf(stderr, "H5Screate_simple failed\n"); + TEST_ERROR; } } /* space for attributes */ - if ((s->one_by_one_sid = H5Screate_simple(1, &dims, &dims)) < 0) - errx(EXIT_FAILURE, "H5Screate_simple failed"); + if ((s->one_by_one_sid = H5Screate_simple(1, &dims, &dims)) < 0) { + fprintf(stderr, "H5Screate_simple failed\n"); + TEST_ERROR; + } - s->dataset = malloc(sizeof(*s->dataset) * s->ndatasets); - if (s->dataset == NULL) - err(EXIT_FAILURE, "could not allocate dataset handles"); + if ((s->dataset = HDmalloc(sizeof(*s->dataset) * s->ndatasets)) == NULL) { + fprintf(stderr, "HDmalloc failed\n"); + TEST_ERROR; + } - s->sources = malloc(sizeof(*s->sources) * s->ndatasets); - if (s->sources == NULL) - err(EXIT_FAILURE, "could not allocate quadrant dataset handles"); + if ((s->sources = HDmalloc(sizeof(*s->sources) * s->ndatasets)) == NULL) { + fprintf(stderr, "HDmalloc failed\n"); + TEST_ERROR; + } for (i = 0; i < s->ndatasets; i++) { s->dataset[i] = badhid; s->sources[i].ul = s->sources[i].ur = s->sources[i].bl = s->sources[i].br = badhid; } - s->memspace = H5Screate_simple(RANK, s->chunk_dims, NULL); - - if (s->memspace < 0) { - errx(EXIT_FAILURE, "%s.%d: H5Screate_simple failed", __func__, __LINE__); + if ((s->memspace = H5Screate_simple(RANK, s->chunk_dims, NULL)) < 0) { + fprintf(stderr, "H5Screate_simple failed\n"); + TEST_ERROR; } s->filename[0] = "vfd_swmr_bigset.h5"; @@ -494,52 +568,102 @@ state_init(state_t *s, int argc, char **argv) s->filename[3] = s->filename[0]; } - personality = strstr(s->progname, "vfd_swmr_bigset_"); + personality = HDstrstr(s->progname, "vfd_swmr_bigset_"); - if (personality != NULL && strcmp(personality, "vfd_swmr_bigset_writer") == 0) + if (personality != NULL && HDstrcmp(personality, "vfd_swmr_bigset_writer") == 0) s->writer = true; - else if (personality != NULL && strcmp(personality, "vfd_swmr_bigset_reader") == 0) + else if (personality != NULL && HDstrcmp(personality, "vfd_swmr_bigset_reader") == 0) s->writer = false; else { - errx(EXIT_FAILURE, "unknown personality, expected vfd_swmr_bigset_{reader,writer}"); + fprintf(stderr, "unknown personality, expected vfd_swmr_bigset_{reader,writer}\n"); + TEST_ERROR; + } + + if ((s->dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) { + fprintf(stderr, "H5Pcreate failed\n"); + TEST_ERROR; } - if ((s->dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) - errx(EXIT_FAILURE, "%s.%d: H5Pcreate failed", __func__, __LINE__); + if (H5Pset_chunk_cache(s->dapl, 0, 0, H5D_CHUNK_CACHE_W0_DEFAULT) < 0) { + fprintf(stderr, "H5Pset_chunk_cache failed\n"); + TEST_ERROR; + } - if (H5Pset_chunk_cache(s->dapl, 0, 0, H5D_CHUNK_CACHE_W0_DEFAULT) < 0) - errx(EXIT_FAILURE, "H5Pset_chunk_cache failed"); + if (s->vds != vds_off && H5Pset_virtual_view(s->dapl, H5D_VDS_FIRST_MISSING) < 0) { + fprintf(stderr, "H5Pset_virtual_view failed\n"); + TEST_ERROR; + } - if (s->vds != vds_off && H5Pset_virtual_view(s->dapl, H5D_VDS_FIRST_MISSING) < 0) - errx(EXIT_FAILURE, "H5Pset_virtual_view failed"); + return true; + +error: + H5E_BEGIN_TRY { + H5Pclose(s->quadrant_dcpl); + H5Sclose(ul->space); + H5Sclose(ur->space); + H5Sclose(bl->space); + H5Sclose(br->space); + H5Sclose(ul->src_space); + H5Sclose(ur->src_space); + H5Sclose(bl->src_space); + H5Sclose(br->src_space); + H5Sclose(src->space); + H5Sclose(s->one_by_one_sid); + H5Sclose(s->memspace); + } H5E_END_TRY; + + if (tfile) + HDfree(tfile); + + if (s->dataset) + HDfree(s->dataset); + + if (s->sources) + HDfree(s->sources); + + return false; } -static void +static bool state_destroy(state_t *s) { size_t i; - if (H5Pclose(s->dapl) < 0) - errx(EXIT_FAILURE, "H5Pclose(fapl)"); - - s->dapl = badhid; + if (H5Pclose(s->dapl) < 0) { + fprintf(stderr, "H5Pclose failed\n"); + TEST_ERROR; + } if (s->vds != vds_off) { quadrant_t *const ul = &s->quadrants.ul, *const ur = &s->quadrants.ur, *const bl = &s->quadrants.bl, *const br = &s->quadrants.br; if (H5Sclose(ul->src_space) < 0 || H5Sclose(ur->src_space) < 0 || H5Sclose(bl->src_space) < 0 || - H5Sclose(br->src_space) < 0) - errx(EXIT_FAILURE, "H5Sclose failed"); + H5Sclose(br->src_space) < 0) { + fprintf(stderr, "H5Sclose failed\n"); + TEST_ERROR; + } - ul->src_space = ur->src_space = bl->src_space = br->src_space = badhid; + if (H5Sclose(ul->space) < 0 || H5Sclose(ur->space) < 0 || H5Sclose(bl->space) < 0 || + H5Sclose(br->space) < 0) { + fprintf(stderr, "H5Sclose failed\n"); + TEST_ERROR; + } - if (H5Pclose(s->quadrant_dcpl) < 0) - errx(EXIT_FAILURE, "H5Pclose(dcpl)"); + if (H5Pclose(s->quadrant_dcpl) < 0) { + fprintf(stderr, "H5Pclose failed\n"); + TEST_ERROR; + } + } - s->quadrant_dcpl = badhid; + if (H5Sclose(s->one_by_one_sid) < 0) { + fprintf(stderr, "H5Sclose failed\n"); + TEST_ERROR; + } - /* TBD destroy spaces belonging to quadrants */ + if (H5Sclose(s->memspace) < 0) { + fprintf(stderr, "H5Sclose failed\n"); + TEST_ERROR; } for (i = 0; i < NELMTS(s->file); i++) { @@ -550,12 +674,265 @@ state_destroy(state_t *s) if (s->vds != vds_multi && i > 0) continue; - if (H5Fclose(fid) < 0) - errx(EXIT_FAILURE, "H5Fclose"); + if (H5Fclose(fid) < 0) { + fprintf(stderr, "H5Fclose failed\n"); + TEST_ERROR; + } } + + if (s->dataset) + HDfree(s->dataset); + + if (s->sources) + HDfree(s->sources); + + return true; + +error: + H5E_BEGIN_TRY { + H5Pclose(s->quadrant_dcpl); + H5Sclose(s->one_by_one_sid); + H5Sclose(s->memspace); + } H5E_END_TRY; + + if (s->dataset) + HDfree(s->dataset); + + if (s->sources) + HDfree(s->sources); + + return false; } -static void +/* + * Initialize the named pipes for test synchronization. + * (This function can go to vfd_swmr_common.c. It's the same as + * the one in vfd_swmr_attrdset_writer.c) + */ +static bool +np_init(np_state_t *np, bool writer) +{ + *np = NP_INITIALIZER; + + /* + * Use two named pipes(FIFO) to coordinate the writer and reader for + * two-way communication so that the two sides can move forward together. + * One is for the writer to write to the reader. + * The other one is for the reader to signal the writer. + */ + if (writer) { + /* If the named pipes are present at the start of the test, remove them */ + if (HDaccess(np->fifo_writer_to_reader, F_OK) == 0) + if(HDremove(np->fifo_writer_to_reader) != 0) { + fprintf(stderr, "HDremove fifo_writer_to_reader failed\n"); + TEST_ERROR; + } + + if (HDaccess(np->fifo_reader_to_writer, F_OK) == 0) + if(HDremove(np->fifo_reader_to_writer) != 0) { + fprintf(stderr, "HDremove fifo_reader_to_writer failed\n"); + TEST_ERROR; + } + + /* Writer creates two named pipes(FIFO) */ + if (HDmkfifo(np->fifo_writer_to_reader, 0600) < 0) { + fprintf(stderr, "HDmkfifo fifo_writer_to_reader failed\n"); + TEST_ERROR; + } + + if (HDmkfifo(np->fifo_reader_to_writer, 0600) < 0) { + fprintf(stderr, "HDmkfifo fifo_reader_to_writer failed\n"); + TEST_ERROR; + } + } + + /* Both the writer and reader open the pipes */ + if ((np->fd_writer_to_reader = HDopen(np->fifo_writer_to_reader, O_RDWR)) < 0) { + fprintf(stderr, "HDopen fifo_writer_to_reader failed\n"); + TEST_ERROR; + } + + if ((np->fd_reader_to_writer = HDopen(np->fifo_reader_to_writer, O_RDWR)) < 0) { + fprintf(stderr, "HDopen fifo_reader_to_writer failed\n"); + TEST_ERROR; + } + + return true; + +error: + return false; +} /* np_init() */ + +/* + * Close and remove the named pipes. + * (This function can go to vfd_swmr_common.c. It's the same as + * the one in vfd_swmr_attrdset_writer.c) + */ +static bool +np_close(np_state_t *np, bool writer) +{ + /* Both the writer and reader close the named pipes */ + if (HDclose(np->fd_writer_to_reader) < 0) { + fprintf(stderr, "HDclose fd_writer_to_reader failed\n"); + TEST_ERROR; + } + + if (HDclose(np->fd_reader_to_writer) < 0) { + fprintf(stderr, "HDclose fd_reader_to_writer failed\n"); + TEST_ERROR; + } + + /* Reader finishes last and deletes the named pipes */ + if(!writer) { + if(HDremove(np->fifo_writer_to_reader) != 0) { + fprintf(stderr, "HDremove fifo_writer_to_reader failed\n"); + TEST_ERROR; + } + + if(HDremove(np->fifo_reader_to_writer) != 0) { + fprintf(stderr, "HDremove fifo_reader_to_writer failed\n"); + TEST_ERROR; + } + } + return true; + +error: + return false; +} /* np_close() */ + +/* Wait for the writer's notice before starting to zoo validation */ +static int +reader_verify(np_state_t np, int verify) +{ + int notify; + + if (HDread(np.fd_writer_to_reader, ¬ify, sizeof(int)) < 0) { + fprintf(stderr, "HDread failed\n"); + TEST_ERROR; + } + + if (notify != verify) { + fprintf(stderr, "expected %d but read %d\n", verify, notify); + TEST_ERROR; + } + + return 0; + +error: + return -1; +} + +/* Notify the reader of finishing zoo creation by sending the timestamp + * and wait for the reader to finish validation before proceeding */ +static int +notify_and_wait_for_reader(state_t *s, np_state_t *np) +{ + int notify; + unsigned int i; + struct timespec last = {0, 0}; + + /* Get the time when finishing zoo creation */ + if (HDclock_gettime(CLOCK_MONOTONIC, &last) < 0) { + fprintf(stderr, "HDclock_gettime failed\n"); + TEST_ERROR; + } + + /* Notify the reader of finishing zoo creation by sending the timestamp */ + if (HDwrite(np->fd_writer_to_reader, &last, sizeof(last)) < 0) { + fprintf(stderr, "HDwrite failed\n"); + TEST_ERROR; + } + + /* During the wait, writer makes repeated HDF5 API calls so as to trigger + * EOT at approximately the correct time */ + for(i = 0; i < MAX_LAG + 1; i++) { + decisleep(TICK_LEN); + + H5E_BEGIN_TRY { + H5Aexists(s->file[0], "nonexistent"); + } H5E_END_TRY; + } + + /* Wait until the reader finishes validating zoo creation */ + if (HDread(np->fd_reader_to_writer, ¬ify, sizeof(int)) < 0) { + fprintf(stderr, "HDread failed\n"); + TEST_ERROR; + } + + if (notify != np->verify) { + fprintf(stderr, "expected %d but read %d\n", np->verify, notify); + TEST_ERROR; + } + + return 0; + +error: + return -1; +} + +/* Receive the notice of the writer finishing dataset creation (timestamp) + * Make sure the dataset validation doesn't take longer than the expected time. + * This time period is from the writer finishing dataset creation to the reader finishing + * the validation of dataset creation */ +static int +reader_check_time_and_notify_writer(np_state_t *np, state_t s) +{ + struct timespec last = {0, 0}; + + /* Receive the notice of the writer finishing zoo creation (timestamp) */ + if (HDread(np->fd_writer_to_reader, &last, sizeof(last)) < 0) { + fprintf(stderr, "HDread failed\n"); + TEST_ERROR; + } + + /* Make sure the dataset validation doesn't take longer than the expected time. + * This time period is from the writer finishing dataset creation to the reader finishing + * the validation of dataset creation */ + if (below_speed_limit(&last, &(s.ival))) { + AT(); + fprintf(stderr, "dataset validation took too long to finish\n"); + } + + /* Notify the writer that dataset validation is finished */ + if (HDwrite(np->fd_reader_to_writer, &(np->notify), sizeof(int)) < 0) { + fprintf(stderr, "HDwrite failed\n"); + TEST_ERROR; + } + + return 0; + +error: + return -1; +} + +/* Notify the reader by sending the timestamp and the number of chunks written */ +static int +notify_reader(np_state_t *np, unsigned step) +{ + exchange_info_t last; + + /* Get the time */ + if (HDclock_gettime(CLOCK_MONOTONIC, &(last.time)) < 0) { + fprintf(stderr, "HDclock_gettime failed\n"); + TEST_ERROR; + } + + last.step = step; + + /* Notify the reader by sending the timestamp and the number of chunks written */ + if (HDwrite(np->fd_writer_to_reader, &last, sizeof(last)) < 0) { + H5_FAILED(); AT(); + fprintf(stderr, "HDwrite failed"); + goto error; + } + + return 0; + +error: + return -1; +} + +static bool create_extensible_dset(state_t *s, unsigned int which) { quadrant_t *const ul = &s->quadrants.ul, *const ur = &s->quadrants.ur, *const bl = &s->quadrants.bl, @@ -563,19 +940,19 @@ create_extensible_dset(state_t *s, unsigned int which) char dname[sizeof("/dataset-9999999999")]; char ul_dname[sizeof("/ul-dataset-9999999999")], ur_dname[sizeof("/ur-dataset-9999999999")], bl_dname[sizeof("/bl-dataset-9999999999")], br_dname[sizeof("/br-dataset-9999999999")]; - hid_t dcpl, ds, filespace; - - assert(which < s->ndatasets); - assert(s->dataset[which] == badhid); + hid_t dcpl = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, filespace = H5I_INVALID_HID; esnprintf(dname, sizeof(dname), "/dataset-%d", which); if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) { - errx(EXIT_FAILURE, "%s.%d: H5Pcreate failed", __func__, __LINE__); + fprintf(stderr, "H5Pcreate failed\n"); + TEST_ERROR; } - if (H5Pset_chunk(dcpl, RANK, s->chunk_dims) < 0) - errx(EXIT_FAILURE, "H5Pset_chunk failed"); + if (H5Pset_chunk(dcpl, RANK, s->chunk_dims) < 0) { + fprintf(stderr, "H5Pset_chunk failed\n"); + TEST_ERROR; + } if (s->vds != vds_off) { sources_t *const srcs = &s->sources[which]; @@ -585,78 +962,101 @@ create_extensible_dset(state_t *s, unsigned int which) esnprintf(bl_dname, sizeof(bl_dname), "/bl-dataset-%d", which); esnprintf(br_dname, sizeof(br_dname), "/br-dataset-%d", which); - srcs->ul = H5Dcreate2(s->file[0], ul_dname, s->filetype, ul->src_space, H5P_DEFAULT, s->quadrant_dcpl, - s->dapl); - - if (srcs->ul < 0) - errx(EXIT_FAILURE, "H5Dcreate(, \"%s\", ) failed", ul_dname); - - srcs->ur = H5Dcreate2(s->file[1], ur_dname, s->filetype, ur->src_space, H5P_DEFAULT, s->quadrant_dcpl, - s->dapl); - - if (srcs->ur < 0) - errx(EXIT_FAILURE, "H5Dcreate(, \"%s\", ) failed", ur_dname); + if ((srcs->ul = H5Dcreate2(s->file[0], ul_dname, s->filetype, ul->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { + fprintf(stderr, "H5Dcreate2 failed\n"); + TEST_ERROR; + } - srcs->bl = H5Dcreate2(s->file[2], bl_dname, s->filetype, bl->src_space, H5P_DEFAULT, s->quadrant_dcpl, - s->dapl); + if ((srcs->ur = H5Dcreate2(s->file[1], ur_dname, s->filetype, ur->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { + fprintf(stderr, "H5Dcreate2 failed\n"); + TEST_ERROR; + } - if (srcs->bl < 0) - errx(EXIT_FAILURE, "H5Dcreate(, \"%s\", ) failed", bl_dname); + if ((srcs->bl = H5Dcreate2(s->file[2], bl_dname, s->filetype, bl->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { + fprintf(stderr, "H5Dcreate2 failed\n"); + TEST_ERROR; + } - srcs->br = H5Dcreate2(s->file[3], br_dname, s->filetype, br->src_space, H5P_DEFAULT, s->quadrant_dcpl, - s->dapl); + if ((srcs->br = H5Dcreate2(s->file[3], br_dname, s->filetype, br->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { + fprintf(stderr, "H5Dcreate2 failed\n"); + TEST_ERROR; + } - if (srcs->br < 0) - errx(EXIT_FAILURE, "H5Dcreate(, \"%s\", ) failed", br_dname); + if (H5Pset_virtual(dcpl, ul->space, s->filename[0], ul_dname, src->space) < 0) { + fprintf(stderr, "H5Pset_virtual failed\n"); + TEST_ERROR; + } - if (H5Pset_virtual(dcpl, ul->space, s->filename[0], ul_dname, src->space) < 0) - errx(EXIT_FAILURE, "%s: H5Pset_virtual failed", __func__); + if (H5Pset_virtual(dcpl, ur->space, s->filename[1], ur_dname, src->space) < 0) { + fprintf(stderr, "H5Pset_virtual failed\n"); + TEST_ERROR; + } - if (H5Pset_virtual(dcpl, ur->space, s->filename[1], ur_dname, src->space) < 0) - errx(EXIT_FAILURE, "%s: H5Pset_virtual failed", __func__); + if (H5Pset_virtual(dcpl, bl->space, s->filename[2], bl_dname, src->space) < 0) { + fprintf(stderr, "H5Pset_virtual failed\n"); + TEST_ERROR; + } - if (H5Pset_virtual(dcpl, bl->space, s->filename[2], bl_dname, src->space) < 0) - errx(EXIT_FAILURE, "%s: H5Pset_virtual failed", __func__); + if (H5Pset_virtual(dcpl, br->space, s->filename[3], br_dname, src->space) < 0) { + fprintf(stderr, "H5Pset_virtual failed\n"); + TEST_ERROR; + } + } - if (H5Pset_virtual(dcpl, br->space, s->filename[3], br_dname, src->space) < 0) - errx(EXIT_FAILURE, "%s: H5Pset_virtual failed", __func__); + if ((filespace = H5Screate_simple(NELMTS(s->chunk_dims), s->chunk_dims, + s->two_dee ? two_dee_max_dims : s->one_dee_max_dims)) < 0) { + fprintf(stderr, "H5Screate_simple failed\n"); + TEST_ERROR; } - filespace = H5Screate_simple(NELMTS(s->chunk_dims), s->chunk_dims, - s->two_dee ? two_dee_max_dims : s->one_dee_max_dims); + if ((dset_id = H5Dcreate2(s->file[0], dname, s->filetype, filespace, H5P_DEFAULT, dcpl, s->dapl)) < 0) { + fprintf(stderr, "H5Dcreate2 failed\n"); + TEST_ERROR; + } - if (filespace < 0) { - errx(EXIT_FAILURE, "%s.%d: H5Screate_simple failed", __func__, __LINE__); + if (H5Sclose(filespace) < 0) { + fprintf(stderr, "H5Sclose failed\n"); + TEST_ERROR; } - ds = H5Dcreate2(s->file[0], dname, s->filetype, filespace, H5P_DEFAULT, dcpl, s->dapl); + if (H5Pclose(dcpl) < 0) { + fprintf(stderr, "H5Pclose failed\n"); + TEST_ERROR; + } - if (ds < 0) - errx(EXIT_FAILURE, "H5Dcreate(, \"%s\", ) failed", dname); + s->dataset[which] = dset_id; - if (H5Sclose(filespace) < 0) - errx(EXIT_FAILURE, "%s: H5Sclose failed", __func__); + return true; - if (H5Pclose(dcpl) < 0) - errx(EXIT_FAILURE, "%s: H5Pclose failed", __func__); +error: + H5E_BEGIN_TRY { + H5Dclose(dset_id); + H5Pclose(dcpl); + H5Sclose(filespace); + } H5E_END_TRY; - s->dataset[which] = ds; + return false; } -static void +static bool close_extensible_dset(state_t *s, unsigned int which) { char dname[sizeof("/dataset-9999999999")]; - hid_t ds; + hid_t dset_id = H5I_INVALID_HID; - assert(which < s->ndatasets); + if (which >= s->ndatasets) { + fprintf(stderr, "index is out of range\n"); + TEST_ERROR; + } esnprintf(dname, sizeof(dname), "/dataset-%d", which); - ds = s->dataset[which]; + dset_id = s->dataset[which]; - if (H5Dclose(ds) < 0) - errx(EXIT_FAILURE, "H5Dclose failed for \"%s\"", dname); + if (H5Dclose(dset_id) < 0) { + fprintf(stderr, "H5Dclose failed\n"); + TEST_ERROR; + } s->dataset[which] = badhid; @@ -664,86 +1064,114 @@ close_extensible_dset(state_t *s, unsigned int which) sources_t *const srcs = &s->sources[which]; if (H5Dclose(srcs->ul) < 0 || H5Dclose(srcs->ur) < 0 || H5Dclose(srcs->bl) < 0 || - H5Dclose(srcs->br) < 0) - errx(EXIT_FAILURE, "H5Dclose failed"); - - srcs->ul = srcs->ur = srcs->bl = srcs->br = badhid; + H5Dclose(srcs->br) < 0) { + fprintf(stderr, "H5Dclose failed\n"); + TEST_ERROR; + } } + + return true; + +error: + H5E_BEGIN_TRY { + H5Dclose(dset_id); + } H5E_END_TRY; + + return false; } -static void -open_extensible_dset(state_t *s, unsigned int which) +static bool +open_extensible_dset(state_t *s) { hsize_t dims[RANK], maxdims[RANK]; char dname[sizeof("/dataset-9999999999")]; - hid_t ds, filespace, ty; - const int tries = 10000; - int i; - struct timespec last = {0, 0}; - static const struct timespec ival = {5, 0}; - estack_state_t es; + hid_t dset_id, filespace, dtype; + unsigned int which, i; - assert(which < s->ndatasets); - assert(s->dataset[which] == badhid); + for (which = 0; which < s->ndatasets; which++) { + esnprintf(dname, sizeof(dname), "/dataset-%d", which); - esnprintf(dname, sizeof(dname), "/dataset-%d", which); + /* Tries to open the dataset repeatedly until successful. After trying + * NUM_ATTEMPTS times without success, report it as a failure + */ + for (i = 0; i < NUM_ATTEMPTS; i++) { + H5E_BEGIN_TRY { + dset_id = H5Dopen2(s->file[0], dname, s->dapl); + } H5E_END_TRY; - es = disable_estack(); - for (i = 0; i < tries; i++) { - struct timespec one_tenth = {.tv_sec = 0, .tv_nsec = 1000000000L / 10}; + if (dset_id >= 0) + break; + else + decisleep(1); + } - ds = H5Dopen2(s->file[0], dname, s->dapl); + if (i == NUM_ATTEMPTS) { + fprintf(stderr, "chunk verification reached the maximal number of attempts\n"); + TEST_ERROR; + } - if (ds >= 0) - break; + if ((dtype = H5Dget_type(dset_id)) < 0) { + fprintf(stderr, "H5Dget_type failed\n"); + TEST_ERROR; + } - if (below_speed_limit(&last, &ival)) { - warnx("H5Dopen(, \"%s\", ) transient failure, %d retries remain", dname, tries - i - 1); + if (H5Tequal(dtype, s->filetype) <= 0) { + fprintf(stderr, "Unexpected data type\n"); + TEST_ERROR; } - while (nanosleep(&one_tenth, &one_tenth) == -1 && errno == EINTR) - ; // do nothing - } - restore_estack(es); - if (i == tries) { - errx(EXIT_FAILURE, "H5Dopen(, \"%s\", ) failed after %d tries", dname, tries); - } + if ((filespace = H5Dget_space(dset_id)) < 0) { + fprintf(stderr, "H5Dget_space failed\n"); + TEST_ERROR; + } - if ((ty = H5Dget_type(ds)) < 0) - errx(EXIT_FAILURE, "H5Dget_type failed"); + if (H5Sget_simple_extent_ndims(filespace) != RANK) { + fprintf(stderr, "Unexpected data rank\n"); + TEST_ERROR; + } - if (H5Tequal(ty, s->filetype) <= 0) - errx(EXIT_FAILURE, "Unexpected data type"); + if (H5Sget_simple_extent_dims(filespace, dims, maxdims) < 0) { + fprintf(stderr, "H5Sget_simple_extent_dims failed\n"); + TEST_ERROR; + } - if ((filespace = H5Dget_space(ds)) < 0) - errx(EXIT_FAILURE, "H5Dget_space failed"); + if (H5Sclose(filespace) < 0) { + fprintf(stderr, "H5Sclose failed\n"); + TEST_ERROR; + } - if (H5Sget_simple_extent_ndims(filespace) != RANK) - errx(EXIT_FAILURE, "Unexpected rank"); + if (H5Tclose(dtype) < 0) { + fprintf(stderr, "H5Tclose failed\n"); + TEST_ERROR; + } - if (H5Sget_simple_extent_dims(filespace, dims, maxdims) < 0) - errx(EXIT_FAILURE, "H5Sget_simple_extent_dims failed"); + if (s->two_dee) { + if (maxdims[0] != two_dee_max_dims[0] || maxdims[1] != two_dee_max_dims[1] || + maxdims[0] != maxdims[1]) { + fprintf(stderr, "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE, maxdims[0], + maxdims[1]); + TEST_ERROR; + } + } else if (maxdims[0] != s->one_dee_max_dims[0] || maxdims[1] != s->one_dee_max_dims[1] || + dims[0] != s->chunk_dims[0]) { + fprintf(stderr, + "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE " or columns %" PRIuHSIZE, + maxdims[0], maxdims[1], dims[1]); + } - if (H5Sclose(filespace) < 0) - errx(EXIT_FAILURE, "H5Sclose failed"); + s->dataset[which] = dset_id; + } - filespace = badhid; + return true; - if (s->two_dee) { - if (maxdims[0] != two_dee_max_dims[0] || maxdims[1] != two_dee_max_dims[1] || - maxdims[0] != maxdims[1]) { - errx(EXIT_FAILURE, "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE, maxdims[0], - maxdims[1]); - } - } - else if (maxdims[0] != s->one_dee_max_dims[0] || maxdims[1] != s->one_dee_max_dims[1] || - dims[0] != s->chunk_dims[0]) { - errx(EXIT_FAILURE, - "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE " or columns %" PRIuHSIZE, - maxdims[0], maxdims[1], dims[1]); - } +error: + H5E_BEGIN_TRY { + H5Dclose(dset_id); + H5Tclose(dtype); + H5Sclose(filespace); + } H5E_END_TRY; - s->dataset[which] = ds; + return false; } /* Write or verify the dataset test pattern in the matrix `mat`. @@ -772,10 +1200,11 @@ open_extensible_dset(state_t *s, unsigned int which) * * In an actual pattern, the dataset number, `which`, is added to each integer. */ -static void +static bool set_or_verify_matrix(mat_t *mat, unsigned int which, base_t base, bool do_set) { unsigned row, col; + bool ret = true; for (row = 0; row < mat->rows; row++) { for (col = 0; col < mat->cols; col++) { @@ -787,85 +1216,138 @@ set_or_verify_matrix(mat_t *mat, unsigned int which, base_t base, bool do_set) else u = j * j + i; - assert(UINT32_MAX - u >= which); v = (uint32_t)(u + which); - if (do_set) - matset(mat, row, col, v); - else if (matget(mat, row, col) != v) { - errx(EXIT_FAILURE, - "matrix mismatch " - "at %" PRIuHSIZE ", %" PRIuHSIZE " (%u, %u), " - "read %" PRIu32 " expecting %" PRIu32, - i, j, row, col, matget(mat, row, col), v); + if (do_set) { + if (!matset(mat, row, col, v)) { + fprintf(stderr, "data initialization failed\n"); + ret = false; + break; + } + } else if (matget(mat, row, col) != v) { + /* If the data doesn't match, simply return false and + * let the caller repeat this step + */ + ret = false; + break; } } } + + return ret; } -static void +static bool init_matrix(mat_t *mat, unsigned int which, base_t base) { - set_or_verify_matrix(mat, which, base, true); + return set_or_verify_matrix(mat, which, base, true); } -static void +static bool verify_matrix(mat_t *mat, unsigned int which, base_t base) { - set_or_verify_matrix(mat, which, base, false); + return set_or_verify_matrix(mat, which, base, false); } -static void +static bool verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t base) { hsize_t offset[RANK] = {base.row, base.col}; herr_t status; - hid_t ds; + hid_t dset_id; - assert(which < s->ndatasets); + if (which >= s->ndatasets) { + fprintf(stderr, "the dataset order is bigger than the number of datasets"); + TEST_ERROR; + } dbgf(1, "verifying chunk %" PRIuHSIZE ", %" PRIuHSIZE "\n", base.row, base.col); - ds = s->dataset[which]; + dset_id = s->dataset[which]; - status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, s->chunk_dims, NULL); + if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, s->chunk_dims, NULL) < 0) { + fprintf(stderr, "H5Sselect_hyperslab failed\n"); + TEST_ERROR; + } + + /* A failure to read the data may indicate the data isn't ready yet. Instead of displaying the error stack, + * simply return false and let the caller repeat this step. + */ + H5E_BEGIN_TRY { + status = H5Dread(dset_id, H5T_NATIVE_UINT32, s->memspace, filespace, H5P_DEFAULT, mat->elt); + } H5E_END_TRY; if (status < 0) - errx(EXIT_FAILURE, "H5Sselect_hyperslab failed"); + TEST_ERROR; - status = H5Dread(ds, H5T_NATIVE_UINT32, s->memspace, filespace, H5P_DEFAULT, mat->elt); + return verify_matrix(mat, which, base); - if (status < 0) - errx(EXIT_FAILURE, "H5Dread failed"); +error: + return false; +} + +static bool +repeat_verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t base) +{ + hid_t dset_id = s->dataset[which]; + unsigned i; + + /* If the chunk data isn't good after reading it NUM_ATTEMPTS times, report it as a failure */ + for (i = 0; i < NUM_ATTEMPTS; i++) { + if (verify_chunk(s, filespace, mat, which, base)) + break; + else { + decisleep(1); + + /* Refresh the dataset and try it again */ + if (H5Drefresh(dset_id) < 0) { + fprintf(stderr, "H5Drefresh failed\n"); + TEST_ERROR; + } + } + } + + if (i == NUM_ATTEMPTS) { + fprintf(stderr, "chunk verification reached the maximal number of attempts\n"); + TEST_ERROR; + } + + return true; - verify_matrix(mat, which, base); +error: + return false; } -static void +static bool init_and_write_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t base) { hsize_t offset[RANK] = {base.row, base.col}; - herr_t status; - hid_t ds; - - assert(which < s->ndatasets); + hid_t dset_id; - ds = s->dataset[which]; + dset_id = s->dataset[which]; - init_matrix(mat, which, base); + if (!init_matrix(mat, which, base)) { + fprintf(stderr, "data initialization failed\n"); + TEST_ERROR; + } - status = H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, s->chunk_dims, NULL); + if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, s->chunk_dims, NULL) < 0) { + fprintf(stderr, "H5Sselect_hyperslab failed\n"); + TEST_ERROR; + } - if (status < 0) - errx(EXIT_FAILURE, "H5Sselect_hyperslab failed"); + if (H5Dwrite(dset_id, H5T_NATIVE_UINT32, s->memspace, filespace, H5P_DEFAULT, mat->elt) < 0) { + fprintf(stderr, "H5Dwrite failed\n"); + TEST_ERROR; + } - status = H5Dwrite(ds, H5T_NATIVE_UINT32, s->memspace, filespace, H5P_DEFAULT, mat->elt); + return true; - if (status < 0) - errx(EXIT_FAILURE, "H5Dwrite failed"); +error: + return false; } -static void -verify_dset_attribute(hid_t ds, unsigned int which, unsigned int step) +static bool +verify_dset_attribute(hid_t dset_id, unsigned int which, unsigned int step) { unsigned int read_step; hid_t aid; @@ -875,64 +1357,97 @@ verify_dset_attribute(hid_t ds, unsigned int which, unsigned int step) dbgf(1, "verifying attribute %s on dataset %u equals %u\n", name, which, step); - if ((aid = H5Aopen(ds, name, H5P_DEFAULT)) < 0) - errx(EXIT_FAILURE, "H5Aopen failed"); + if ((aid = H5Aopen(dset_id, name, H5P_DEFAULT)) < 0) { + fprintf(stderr, "H5Aopen failed\n"); + TEST_ERROR; + } + + if (H5Aread(aid, H5T_NATIVE_UINT, &read_step) < 0) { + fprintf(stderr, "H5Aread failed\n"); + TEST_ERROR; + } - if (H5Aread(aid, H5T_NATIVE_UINT, &read_step) < 0) - errx(EXIT_FAILURE, "H5Aread failed"); + if (H5Aclose(aid) < 0) { + fprintf(stderr, "H5Aclose failed\n"); + TEST_ERROR; + } - if (H5Aclose(aid) < 0) - errx(EXIT_FAILURE, "H5Aclose failed"); + if (read_step != step) { + fprintf(stderr, "expected %u read %u\n", step, read_step); + TEST_ERROR; + } - if (read_step != step) - errx(EXIT_FAILURE, "expected %u read %u", step, read_step); + return true; + +error: + H5E_BEGIN_TRY { + H5Aclose(aid); + } H5E_END_TRY; + + return false; } -static void -verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned *stepp) +static bool +verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned finished_step, unsigned last_step) { - hid_t ds, filespace; + hid_t dset_id = H5I_INVALID_HID, filespace = H5I_INVALID_HID; hsize_t size[RANK]; base_t base, last; - unsigned int ncols, last_step, step; + unsigned int ncols, step; + int i; - assert(which < s->ndatasets); - - ds = s->dataset[which]; + if (which >= s->ndatasets) { + fprintf(stderr, "the dataset order is bigger than the number of datasets"); + TEST_ERROR; + } - if (H5Drefresh(ds) < 0) - errx(EXIT_FAILURE, "H5Drefresh failed"); + dset_id = s->dataset[which]; - filespace = H5Dget_space(ds); + /* Attempt to check the availablity of the chunks for a number time before reporting it as a failure */ + for (i = 0; i < NUM_ATTEMPTS; i++) { + if (H5Drefresh(dset_id) < 0) { + fprintf(stderr, "H5Drefresh failed\n"); + TEST_ERROR; + } - if (filespace == badhid) - errx(EXIT_FAILURE, "H5Dget_space failed"); + if ((filespace = H5Dget_space(dset_id)) < 0) { + fprintf(stderr, "H5Dget_space failed\n"); + TEST_ERROR; + } - if (H5Sget_simple_extent_dims(filespace, size, NULL) < 0) - errx(EXIT_FAILURE, "H5Sget_simple_extent_dims failed"); + if (H5Sget_simple_extent_dims(filespace, size, NULL) < 0) { + fprintf(stderr, "H5Sget_simple_extent_dims failed\n"); + TEST_ERROR; + } - ncols = (unsigned)(size[1] / s->chunk_dims[1]); - if (ncols < hang_back) - goto out; + ncols = (unsigned)(size[1] / s->chunk_dims[1]); - last_step = ncols - hang_back; + /* Make sure the chunks show up on the reader side. Otherwise sleep a while and try again */ + if (ncols >= last_step) + break; + else + decisleep(1); + } - for (step = *stepp; step <= last_step; step++) { - const unsigned ofs = step % 2; + if (i == NUM_ATTEMPTS) { + fprintf(stderr, "chunk verification reached the maximal number of attempts"); + TEST_ERROR; + } + for (step = finished_step; step < last_step; step++) { dbgf(1, "%s: which %u step %u\n", __func__, which, step); if (s->two_dee) { size[0] = s->chunk_dims[0] * (1 + step); size[1] = s->chunk_dims[1] * (1 + step); - last.row = s->chunk_dims[0] * step + ofs; - last.col = s->chunk_dims[1] * step + ofs; + last.row = s->chunk_dims[0] * step; + last.col = s->chunk_dims[1] * step; } else { size[0] = s->chunk_dims[0]; size[1] = s->chunk_dims[1] * (1 + step); last.row = 0; - last.col = s->chunk_dims[1] * step + ofs; + last.col = s->chunk_dims[1] * step; } dbgf(1, "new size %" PRIuHSIZE ", %" PRIuHSIZE "\n", size[0], size[1]); @@ -942,33 +1457,108 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned *ste /* Down the right side, intersecting the bottom row. */ base.col = last.col; - for (base.row = ofs; base.row <= last.row; base.row += s->chunk_dims[0]) { - verify_chunk(s, filespace, mat, which, base); + for (base.row = 0; base.row <= last.row; base.row += s->chunk_dims[0]) { + if (!repeat_verify_chunk(s, filespace, mat, which, base)) { + fprintf(stderr, "chunk verification failed\n"); + TEST_ERROR; + } } /* Across the bottom, stopping before the last column to * avoid re-reading the bottom-right chunk. */ base.row = last.row; - for (base.col = ofs; base.col < last.col; base.col += s->chunk_dims[1]) { - verify_chunk(s, filespace, mat, which, base); + for (base.col = 0; base.col < last.col; base.col += s->chunk_dims[1]) { + if (!repeat_verify_chunk(s, filespace, mat, which, base)) { + fprintf(stderr, "chunk verification failed\n"); + TEST_ERROR; + } } } else { - verify_chunk(s, filespace, mat, which, last); + if (!repeat_verify_chunk(s, filespace, mat, which, last)) { + fprintf(stderr, "chunk verification failed\n"); + TEST_ERROR; + } + } + + if (s->asteps != 0 && step % s->asteps == 0) { + if (!verify_dset_attribute(dset_id, which, step)) { + fprintf(stderr, "verify_dset_attribute failed\n"); + TEST_ERROR; + } } - if (s->asteps != 0 && step % s->asteps == 0) - verify_dset_attribute(ds, which, step); } - *stepp = step; + return true; + +error: + H5E_BEGIN_TRY { + H5Sclose(filespace); + } H5E_END_TRY; -out: - if (H5Sclose(filespace) < 0) - errx(EXIT_FAILURE, "H5Sclose failed"); + return false; } -static void +static bool +verify_dsets(state_t s, np_state_t *np, mat_t *mat) +{ + unsigned *nextstep = NULL; + unsigned finished_step = 0; + unsigned which; + exchange_info_t last; + + if (!(nextstep = HDcalloc(s.ndatasets, sizeof(*nextstep)))) { + fprintf(stderr, "memory allocation failed\n"); + TEST_ERROR; + } + + do { + /* Receive the notice of the writer finishing zoo creation, + * including the number of chunks finished and the timestamp + */ + if (s.use_named_pipe && HDread(np->fd_writer_to_reader, &last, sizeof(last)) < 0) { + fprintf(stderr, "HDread failed\n"); + TEST_ERROR; + } + + for (which = 0; which < s.ndatasets; which++) { + dbgf(1, "step %d which %d\n", nextstep[which], which); + + /* Verify the chunks starting from the finished one in last round + * to the ones written in this round + */ + if (!verify_extensible_dset(&s, which, mat, finished_step, last.step)) { + fprintf(stderr, "verify_extensible_dset failed\n"); + TEST_ERROR; + } + + /* Reset the finished one */ + finished_step = last.step; + } + + /* Make sure the chunk verification doesn't take longer than the expected time. + * This time period is from the writer finishing chunks to the reader finishing + * the validation of the chunks */ + if (s.use_named_pipe && below_speed_limit(&(last.time), &(s.ival))) { + AT(); + fprintf(stderr, "verify_extensible_dset took too long to finish\n"); + } + } while (finished_step < s.nsteps); + + if (nextstep) + HDfree(nextstep); + + return true; + +error: + if (nextstep) + HDfree(nextstep); + + return false; +} + +static bool add_dset_attribute(const state_t *s, hid_t ds, hid_t sid, unsigned int which, unsigned int step) { hid_t aid; @@ -978,30 +1568,56 @@ add_dset_attribute(const state_t *s, hid_t ds, hid_t sid, unsigned int which, un dbgf(1, "setting attribute %s on dataset %u to %u\n", name, which, step); - if ((aid = H5Acreate2(ds, name, s->filetype, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) - errx(EXIT_FAILURE, "H5Acreate2 failed"); + if ((aid = H5Acreate2(ds, name, s->filetype, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) { + fprintf(stderr, "H5Acreate2 failed\n"); + TEST_ERROR; + } + + if (H5Awrite(aid, H5T_NATIVE_UINT, &step) < 0) { + fprintf(stderr, "H5Awrite failed\n"); + TEST_ERROR; + } + + if (H5Aclose(aid) < 0) { + fprintf(stderr, "H5Aclose failed\n"); + TEST_ERROR; + } + + return true; - if (H5Awrite(aid, H5T_NATIVE_UINT, &step) < 0) - errx(EXIT_FAILURE, "H5Awrite failed"); - if (H5Aclose(aid) < 0) - errx(EXIT_FAILURE, "H5Aclose failed"); +error: + H5E_BEGIN_TRY { + H5Aclose(aid); + } H5E_END_TRY; + + return false; } -static void +static bool write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t *mat) { - hid_t ds, filespace; + hid_t ds = H5I_INVALID_HID, filespace = H5I_INVALID_HID; hsize_t size[RANK]; base_t base, last; + char dname[sizeof("/dataset-9999999999")]; + + esnprintf(dname, sizeof(dname), "/dataset-%d", which); dbgf(1, "%s: which %u step %u\n", __func__, which, step); - assert(which < s->ndatasets); + if (which >= s->ndatasets) { + fprintf(stderr, "index is out of range\n"); + TEST_ERROR; + } ds = s->dataset[which]; - if (s->asteps != 0 && step % s->asteps == 0) - add_dset_attribute(s, ds, s->one_by_one_sid, which, step); + if (s->asteps != 0 && step % s->asteps == 0) { + if (!add_dset_attribute(s, ds, s->one_by_one_sid, which, step)) { + fprintf(stderr, "add_dset_attribute failed\n"); + TEST_ERROR; + } + } if (s->two_dee) { size[0] = s->chunk_dims[0] * (1 + step); @@ -1023,69 +1639,164 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * sources_t *const srcs = &s->sources[which]; if (H5Dset_extent(srcs->ul, half_size) < 0) { - errx(EXIT_FAILURE, "%s.%d: H5Dset_extent failed", __func__, __LINE__); + fprintf(stderr, "H5Dset_extent failed\n"); + TEST_ERROR; } + if (H5Dset_extent(srcs->ur, half_size) < 0) { - errx(EXIT_FAILURE, "%s.%d: H5Dset_extent failed", __func__, __LINE__); + fprintf(stderr, "H5Dset_extent failed\n"); + TEST_ERROR; } + if (H5Dset_extent(srcs->bl, half_size) < 0) { - errx(EXIT_FAILURE, "%s.%d: H5Dset_extent failed", __func__, __LINE__); + fprintf(stderr, "H5Dset_extent failed\n"); + TEST_ERROR; } + if (H5Dset_extent(srcs->br, half_size) < 0) { - errx(EXIT_FAILURE, "%s.%d: H5Dset_extent failed", __func__, __LINE__); + fprintf(stderr, "H5Dset_extent failed\n"); + TEST_ERROR; } + } else if (H5Dset_extent(ds, size) < 0) { + fprintf(stderr, "H5Dset_extent failed\n"); + TEST_ERROR; } - else if (H5Dset_extent(ds, size) < 0) - errx(EXIT_FAILURE, "H5Dset_extent failed"); - - filespace = H5Dget_space(ds); - if (filespace == badhid) - errx(EXIT_FAILURE, "H5Dget_space failed"); + if ((filespace = H5Dget_space(ds)) < 0) { + fprintf(stderr, "H5Dget_space failed\n"); + TEST_ERROR; + } if (s->two_dee) { base.col = last.col; for (base.row = 0; base.row <= last.row; base.row += s->chunk_dims[0]) { dbgf(1, "writing chunk %" PRIuHSIZE ", %" PRIuHSIZE "\n", base.row, base.col); - init_and_write_chunk(s, filespace, mat, which, base); + if (!init_and_write_chunk(s, filespace, mat, which, base)) { + fprintf(stderr, "init_and_write_chunk failed\n"); + TEST_ERROR; + } } base.row = last.row; for (base.col = 0; base.col < last.col; base.col += s->chunk_dims[1]) { dbgf(1, "writing chunk %" PRIuHSIZE ", %" PRIuHSIZE "\n", base.row, base.col); - init_and_write_chunk(s, filespace, mat, which, base); + if (!init_and_write_chunk(s, filespace, mat, which, base)) { + fprintf(stderr, "init_and_write_chunk failed\n"); + TEST_ERROR; + } } } else { - init_and_write_chunk(s, filespace, mat, which, last); + if (!init_and_write_chunk(s, filespace, mat, which, last)) { + fprintf(stderr, "init_and_write_chunk failed\n"); + TEST_ERROR; + } } - if (H5Sclose(filespace) < 0) - errx(EXIT_FAILURE, "H5Sclose failed"); + if (H5Sclose(filespace) < 0) { + fprintf(stderr, "H5Sclose failed\n"); + TEST_ERROR; + } + + return true; + +error: + H5E_BEGIN_TRY { + H5Sclose(filespace); + } H5E_END_TRY; + + return false; +} + +static bool +write_dsets(state_t s, np_state_t *np, mat_t *mat) +{ + unsigned last_step, step, which; + unsigned long long old_tick_num; + H5F_t *f = NULL; + + if (NULL == (f = (H5F_t *)H5VL_object(s.file[0]))) { + fprintf(stderr, "H5VL_object failed\n"); + TEST_ERROR; + } + + old_tick_num = f->shared->tick_num; + + /* Write as many as chunks within the same tick number before notifying + * the reader to verify them. + */ + for (step = 0; step < s.nsteps; step++) { + /* Write as many as chunks before the tick number changes */ + if (f->shared->tick_num == old_tick_num) { + for (which = 0; which < s.ndatasets; which++) { + dbgf(2, "step %d which %d\n", step, which); + if (!write_extensible_dset(&s, which, step, mat)) { + fprintf(stderr, "write_extensible_dset failed\n"); + TEST_ERROR; + } + } + } + + /* After finishing writing all the chunks, end the tick */ + if(s.use_vfd_swmr && step == (s.nsteps - 1)) { + unsigned long i; + + if (s.vds != vds_multi) + H5Fvfd_swmr_end_tick(s.file[0]); + else + for (i = 0; i < NELMTS(s.file); i++) + H5Fvfd_swmr_end_tick(s.file[i]); + } + + /* Notify the reader to start verification by + * sending the timestamp and the number of chunks written + */ + if (f->shared->tick_num > old_tick_num || step == (s.nsteps - 1)) { + last_step = step + 1; + if (s.use_named_pipe && notify_reader(np, last_step) < 0) { + fprintf(stderr, "notify_reader failed\n"); + TEST_ERROR; + } + + old_tick_num = f->shared->tick_num; + } + } + + return true; + +error: + return false; } int main(int argc, char **argv) { mat_t * mat; - hid_t fcpl; - sigset_t oldsigs; - herr_t ret; - unsigned step, which; + hid_t fcpl = H5I_INVALID_HID; + unsigned which; state_t s; + np_state_t np; size_t i; - state_init(&s, argc, argv); + if (!state_init(&s, argc, argv)) { + fprintf(stderr, "state_init failed\n"); + TEST_ERROR; + } - if ((mat = newmat(s.rows, s.cols)) == NULL) - err(EXIT_FAILURE, "%s: could not allocate matrix", __func__); + if ((mat = newmat(s.rows, s.cols)) == NULL) { + fprintf(stderr, "could not allocate matrix\n"); + TEST_ERROR; + } - if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - errx(EXIT_FAILURE, "H5Pcreate"); + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) { + fprintf(stderr, "H5Pcreate failed\n"); + TEST_ERROR; + } - ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, false, 1); - if (ret < 0) - errx(EXIT_FAILURE, "H5Pset_file_space_strategy"); + if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, false, 1) < 0) { + fprintf(stderr, "H5Pset_file_space_strategy failed\n"); + TEST_ERROR; + } for (i = 0; i < NELMTS(s.file); i++) { hid_t fapl; @@ -1097,89 +1808,164 @@ main(int argc, char **argv) } /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(&config, 4, 7, s.writer, FALSE, 128, "./bigset-shadow-%zu", i); + init_vfd_swmr_config(&config, TICK_LEN, MAX_LAG, s.writer, FALSE, 128, "./bigset-shadow-%zu", i); /* use_latest_format, use_vfd_swmr, only_meta_page, config */ - fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, &config); - - if (fapl < 0) - errx(EXIT_FAILURE, "vfd_swmr_create_fapl"); + if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, &config)) < 0) { + fprintf(stderr, "vfd_swmr_create_fapl failed\n"); + TEST_ERROR; + } s.file[i] = s.writer ? H5Fcreate(s.filename[i], H5F_ACC_TRUNC, fcpl, fapl) : H5Fopen(s.filename[i], H5F_ACC_RDONLY, fapl); - if (s.file[i] == badhid) - errx(EXIT_FAILURE, s.writer ? "H5Fcreate" : "H5Fopen"); + if (s.file[i] == badhid) { + fprintf(stderr, s.writer ? "H5Fcreate failed" : "H5Fopen failed"); + TEST_ERROR; + } - if (H5Pclose(fapl) < 0) - errx(EXIT_FAILURE, "H5Pclose(fapl)"); + if (H5Pclose(fapl) < 0) { + fprintf(stderr, "H5Pclose failed\n"); + TEST_ERROR; + } } - block_signals(&oldsigs); + /* Initiailze named pipes */ + if(s.use_named_pipe && !np_init(&np, s.writer)) { + fprintf(stderr, "np_init() failed\n"); + TEST_ERROR; + } if (s.writer) { + /* Writer tells reader to start */ + np.notify = 1; + if (s.use_named_pipe && HDwrite(np.fd_writer_to_reader, &np.notify, sizeof(int)) < 0) { + fprintf(stderr, "HDwrite failed\n"); + TEST_ERROR; + } + + /* Create NDATASETS datasets as the reader is doing verification. So no communication with + * the reader during the creation of datasets. + */ for (which = 0; which < s.ndatasets; which++) - create_extensible_dset(&s, which); + if (!create_extensible_dset(&s, which)) { + fprintf(stderr, "create_extensible_dset failed: number %u\n", which); + TEST_ERROR; + } - for (step = 0; step < s.nsteps; step++) { - for (which = 0; which < s.ndatasets; which++) { - dbgf(2, "step %d which %d\n", step, which); - write_extensible_dset(&s, which, step, mat); - if (s.ndatasets <= s.nsteps) - nanosleep(&s.update_interval, NULL); - } - if (s.ndatasets > s.nsteps) - nanosleep(&s.update_interval, NULL); + /* Call H5Fvfd_swmr_end_tick to end the tick. No communication with the reader in this step */ + if(s.use_vfd_swmr && s.use_named_pipe) { + unsigned long j; + + if (s.vds != vds_multi) { + if (H5Fvfd_swmr_end_tick(s.file[0]) < 0) { + fprintf(stderr, "H5Fvfd_swmr_end_tick failed\n"); + TEST_ERROR; + } + } else { + for (j = 0; j < NELMTS(s.file); j++) + if (H5Fvfd_swmr_end_tick(s.file[j]) < 0) { + fprintf(stderr, "H5Fvfd_swmr_end_tick failed\n"); + TEST_ERROR; + } + } + } + + /* Notify the reader of finishing dataset creation by sending the timestamp + * and wait for the reader to finish validation before proceeding */ + np.verify = 2; + if (s.use_named_pipe && notify_and_wait_for_reader(&s, &np) < 0) { + fprintf(stderr, "notify_and_wait_for_reader failed\n"); + TEST_ERROR; + } + + /* Start to write chunks. The writer writes as many chunks as possible within a tick, then + * notify the reader. But it doesn't receive back the reader's notice. */ + if (!write_dsets(s, &np, mat)) { + fprintf(stderr, "write_dsets failed"); + TEST_ERROR; + } + } else { + /* Wait for the writer's notice before starting the validation of dataset creation */ + np.verify = 1; + if (s.use_named_pipe && reader_verify(np, np.verify) < 0) { + fprintf(stderr, "reader_verify failed\n"); + TEST_ERROR; + } + + /* Open all the datasets as the writer is creating them. No communication with + * the writer during this step. + */ + if (!open_extensible_dset(&s)) { + fprintf(stderr, "open_extensible_dset failed\n"); + TEST_ERROR; } - } - else { - unsigned *nextstep = calloc(s.ndatasets, sizeof(*nextstep)); - unsigned finished_step; - - if (nextstep == NULL) - err(EXIT_FAILURE, "could not allocate `nextstep` array"); - - for (which = s.ndatasets; which > 0; which--) - open_extensible_dset(&s, which - 1); - - do { - finished_step = UINT_MAX; /* the greatest step finished on - * *all* datasets - */ - - for (which = s.ndatasets; which-- > 0;) { - dbgf(2, "step %d which %d\n", nextstep[which], which); - verify_extensible_dset(&s, which, mat, &nextstep[which]); - if (nextstep[which] < finished_step) - finished_step = nextstep[which]; - if (s.ndatasets <= s.nsteps) - nanosleep(&s.update_interval, NULL); - } - if (s.ndatasets > s.nsteps) - nanosleep(&s.update_interval, NULL); - } while (hang_back + finished_step < s.nsteps); - free(nextstep); + /* Receive the notice of the writer finishing dataset creation (timestamp) + * Make sure the dataset creation doesn't take longer than the expected time. + * This time period is from the writer finishing dataset creation to the reader finishing + * the validation of dataset creation */ + np.notify = 2; + if (s.use_named_pipe && reader_check_time_and_notify_writer(&np, s) < 0) { + fprintf(stderr, "reader_check_time_and_notify_writer failed\n"); + TEST_ERROR; + } + + /* Once the reader starts to verify the datasets, it doesn't notify the writer any info. + * Both the reader and writer finish by themselves. + */ + if (!verify_dsets(s, &np, mat)) { + fprintf(stderr, "verify_dsets failed\n"); + TEST_ERROR; + } } for (which = 0; which < s.ndatasets; which++) - close_extensible_dset(&s, which); - - if (s.use_vfd_swmr && s.wait_for_signal) - await_signal(s.file[0]); + if (!close_extensible_dset(&s, which)) { + fprintf(stderr, "close_extensible_dset failed\n"); + TEST_ERROR; + } - restore_signals(&oldsigs); + if (H5Pclose(fcpl) < 0) { + fprintf(stderr, "H5Pclose failed\n"); + TEST_ERROR; + } - if (H5Pclose(fcpl) < 0) - errx(EXIT_FAILURE, "H5Pclose(fcpl)"); + if(s.use_named_pipe && !np_close(&np, s.writer)) { + fprintf(stderr, "np_close() failed\n"); + TEST_ERROR; + } - state_destroy(&s); + if (!state_destroy(&s)) { + fprintf(stderr, "state_destroy failed\n"); + TEST_ERROR; + } - free(mat); + HDfree(mat); return EXIT_SUCCESS; -} +error: + H5E_BEGIN_TRY { + H5Pclose(fcpl); + + for (i = 0; i < NELMTS(s.file); i++) + H5Fclose(s.file[i]); + } H5E_END_TRY; + + if (s.use_named_pipe && np.fd_writer_to_reader >= 0) + HDclose(np.fd_writer_to_reader); + + if (s.use_named_pipe && np.fd_reader_to_writer >= 0) + HDclose(np.fd_reader_to_writer); + + if(s.use_named_pipe && !s.writer) { + HDremove(np.fifo_writer_to_reader); + HDremove(np.fifo_reader_to_writer); + } + + return EXIT_FAILURE; +} #else /* H5_HAVE_WIN32_API */ int -- cgit v0.12 From a30dbdea29ecae0d07cf9c49b76bf0951ed01a88 Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Mon, 7 Jun 2021 10:06:02 -0500 Subject: Some simple format changes. --- test/vfd_swmr_bigset_writer.c | 108 +++++++++++++++++++++--------------------- 1 file changed, 53 insertions(+), 55 deletions(-) diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c index 138b796..65e3881 100644 --- a/test/vfd_swmr_bigset_writer.c +++ b/test/vfd_swmr_bigset_writer.c @@ -84,10 +84,10 @@ #define MAX_READ_LEN_IN_SECONDS 2 #define TICK_LEN 4 #define MAX_LAG 7 -#define ROWS 256 -#define COLS 512 +#define ROWS 256 +#define COLS 512 #define RANK 2 -#define NUM_ATTEMPTS 100 +#define NUM_ATTEMPTS 100 typedef struct _base { hsize_t row, col; @@ -113,13 +113,13 @@ typedef struct _sources { #define MANY_FILES 4 typedef struct { - hid_t * dataset; - sources_t * sources; - hid_t file[MANY_FILES]; - hid_t dapl, filetype, memspace, one_by_one_sid, quadrant_dcpl; - unsigned ndatasets; - const char * filename[MANY_FILES]; - char progname[PATH_MAX]; + hid_t *dataset; + sources_t *sources; + hid_t file[MANY_FILES]; + hid_t dapl, filetype, memspace, one_by_one_sid, quadrant_dcpl; + unsigned ndatasets; + const char *filename[MANY_FILES]; + char progname[PATH_MAX]; struct { quadrant_t ul, ur, bl, br, src; } quadrants; @@ -141,10 +141,10 @@ typedef struct { typedef struct { const char *fifo_writer_to_reader; /* Name of fifo for writer to reader */ const char *fifo_reader_to_writer; /* Name of fifo for reader to writer */ - int fd_writer_to_reader; /* File ID for fifo from writer to reader */ - int fd_reader_to_writer; /* File ID for fifo from reader to writer */ - int notify; /* Value to notify between writer and reader */ - int verify; /* Value to verify between writer and reader */ + int fd_writer_to_reader; /* File ID for fifo from writer to reader */ + int fd_reader_to_writer; /* File ID for fifo from reader to writer */ + int notify; /* Value to notify between writer and reader */ + int verify; /* Value to verify between writer and reader */ } np_state_t; typedef struct { @@ -154,12 +154,12 @@ typedef struct { /* Initializations for np_state_t */ #define NP_INITIALIZER (np_state_t) { \ - .fifo_writer_to_reader = "./fifo_bigset_writer_to_reader" \ - , .fifo_reader_to_writer = "./fifo_bigset_reader_to_writer" \ - , .fd_writer_to_reader = -1 \ - , .fd_reader_to_writer = -1 \ - , .notify = 0 \ - , .verify = 0 \ + .fifo_writer_to_reader = "./fifo_bigset_writer_to_reader", \ + .fifo_reader_to_writer = "./fifo_bigset_reader_to_writer", \ + .fd_writer_to_reader = -1, \ + .fd_reader_to_writer = -1, \ + .notify = 0, \ + .verify = 0 \ } static inline state_t @@ -380,7 +380,7 @@ state_init(state_t *s, int argc, char **argv) s->cols = (unsigned)tmp; else if (ch == 'l') { /* Translate the tick number to time represented by the timespec struct */ - float time = (float)(((unsigned)tmp * TICK_LEN) / 10.0); + float time = (float)(((unsigned)tmp * TICK_LEN) / 10.0); unsigned sec = (unsigned)time; unsigned nsec = (unsigned)((time - sec) * 10 * 1000 * 1000); @@ -723,13 +723,13 @@ np_init(np_state_t *np, bool writer) if (writer) { /* If the named pipes are present at the start of the test, remove them */ if (HDaccess(np->fifo_writer_to_reader, F_OK) == 0) - if(HDremove(np->fifo_writer_to_reader) != 0) { + if (HDremove(np->fifo_writer_to_reader) != 0) { fprintf(stderr, "HDremove fifo_writer_to_reader failed\n"); TEST_ERROR; } if (HDaccess(np->fifo_reader_to_writer, F_OK) == 0) - if(HDremove(np->fifo_reader_to_writer) != 0) { + if (HDremove(np->fifo_reader_to_writer) != 0) { fprintf(stderr, "HDremove fifo_reader_to_writer failed\n"); TEST_ERROR; } @@ -827,8 +827,8 @@ error: static int notify_and_wait_for_reader(state_t *s, np_state_t *np) { - int notify; - unsigned int i; + int notify; + unsigned int i; struct timespec last = {0, 0}; /* Get the time when finishing zoo creation */ @@ -845,7 +845,7 @@ notify_and_wait_for_reader(state_t *s, np_state_t *np) /* During the wait, writer makes repeated HDF5 API calls so as to trigger * EOT at approximately the correct time */ - for(i = 0; i < MAX_LAG + 1; i++) { + for (i = 0; i < MAX_LAG + 1; i++) { decisleep(TICK_LEN); H5E_BEGIN_TRY { @@ -921,9 +921,8 @@ notify_reader(np_state_t *np, unsigned step) /* Notify the reader by sending the timestamp and the number of chunks written */ if (HDwrite(np->fd_writer_to_reader, &last, sizeof(last)) < 0) { - H5_FAILED(); AT(); fprintf(stderr, "HDwrite failed"); - goto error; + TEST_ERROR; } return 0; @@ -1083,10 +1082,10 @@ error: static bool open_extensible_dset(state_t *s) { - hsize_t dims[RANK], maxdims[RANK]; - char dname[sizeof("/dataset-9999999999")]; - hid_t dset_id, filespace, dtype; - unsigned int which, i; + hsize_t dims[RANK], maxdims[RANK]; + char dname[sizeof("/dataset-9999999999")]; + hid_t dset_id, filespace, dtype; + unsigned int which, i; for (which = 0; which < s->ndatasets; which++) { esnprintf(dname, sizeof(dname), "/dataset-%d", which); @@ -1204,7 +1203,7 @@ static bool set_or_verify_matrix(mat_t *mat, unsigned int which, base_t base, bool do_set) { unsigned row, col; - bool ret = true; + bool ret = true; for (row = 0; row < mat->rows; row++) { for (col = 0; col < mat->cols; col++) { @@ -1288,7 +1287,7 @@ error: static bool repeat_verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t base) { - hid_t dset_id = s->dataset[which]; + hid_t dset_id = s->dataset[which]; unsigned i; /* If the chunk data isn't good after reading it NUM_ATTEMPTS times, report it as a failure */ @@ -1503,9 +1502,9 @@ error: static bool verify_dsets(state_t s, np_state_t *np, mat_t *mat) { - unsigned *nextstep = NULL; - unsigned finished_step = 0; - unsigned which; + unsigned *nextstep = NULL; + unsigned finished_step = 0; + unsigned which; exchange_info_t last; if (!(nextstep = HDcalloc(s.ndatasets, sizeof(*nextstep)))) { @@ -1596,10 +1595,10 @@ error: static bool write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t *mat) { - hid_t ds = H5I_INVALID_HID, filespace = H5I_INVALID_HID; + hid_t dset_id = H5I_INVALID_HID, filespace = H5I_INVALID_HID; hsize_t size[RANK]; base_t base, last; - char dname[sizeof("/dataset-9999999999")]; + char dname[sizeof("/dataset-9999999999")]; esnprintf(dname, sizeof(dname), "/dataset-%d", which); @@ -1610,10 +1609,10 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * TEST_ERROR; } - ds = s->dataset[which]; + dset_id = s->dataset[which]; if (s->asteps != 0 && step % s->asteps == 0) { - if (!add_dset_attribute(s, ds, s->one_by_one_sid, which, step)) { + if (!add_dset_attribute(s, dset_id, s->one_by_one_sid, which, step)) { fprintf(stderr, "add_dset_attribute failed\n"); TEST_ERROR; } @@ -1624,8 +1623,7 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * size[1] = s->chunk_dims[1] * (1 + step); last.row = s->chunk_dims[0] * step; last.col = s->chunk_dims[1] * step; - } - else { + } else { size[0] = s->chunk_dims[0]; size[1] = s->chunk_dims[1] * (1 + step); last.row = 0; @@ -1657,12 +1655,12 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * fprintf(stderr, "H5Dset_extent failed\n"); TEST_ERROR; } - } else if (H5Dset_extent(ds, size) < 0) { + } else if (H5Dset_extent(dset_id, size) < 0) { fprintf(stderr, "H5Dset_extent failed\n"); TEST_ERROR; } - if ((filespace = H5Dget_space(ds)) < 0) { + if ((filespace = H5Dget_space(dset_id)) < 0) { fprintf(stderr, "H5Dget_space failed\n"); TEST_ERROR; } @@ -1738,7 +1736,7 @@ write_dsets(state_t s, np_state_t *np, mat_t *mat) } /* After finishing writing all the chunks, end the tick */ - if(s.use_vfd_swmr && step == (s.nsteps - 1)) { + if (s.use_vfd_swmr && step == (s.nsteps - 1)) { unsigned long i; if (s.vds != vds_multi) @@ -1771,12 +1769,12 @@ error: int main(int argc, char **argv) { - mat_t * mat; - hid_t fcpl = H5I_INVALID_HID; - unsigned which; - state_t s; + mat_t *mat; + hid_t fcpl = H5I_INVALID_HID; + unsigned which; + state_t s; np_state_t np; - size_t i; + size_t i; if (!state_init(&s, argc, argv)) { fprintf(stderr, "state_init failed\n"); @@ -1831,7 +1829,7 @@ main(int argc, char **argv) } /* Initiailze named pipes */ - if(s.use_named_pipe && !np_init(&np, s.writer)) { + if (s.use_named_pipe && !np_init(&np, s.writer)) { fprintf(stderr, "np_init() failed\n"); TEST_ERROR; } @@ -1854,7 +1852,7 @@ main(int argc, char **argv) } /* Call H5Fvfd_swmr_end_tick to end the tick. No communication with the reader in this step */ - if(s.use_vfd_swmr && s.use_named_pipe) { + if (s.use_vfd_swmr && s.use_named_pipe) { unsigned long j; if (s.vds != vds_multi) { @@ -1931,7 +1929,7 @@ main(int argc, char **argv) TEST_ERROR; } - if(s.use_named_pipe && !np_close(&np, s.writer)) { + if (s.use_named_pipe && !np_close(&np, s.writer)) { fprintf(stderr, "np_close() failed\n"); TEST_ERROR; } @@ -1959,7 +1957,7 @@ error: if (s.use_named_pipe && np.fd_reader_to_writer >= 0) HDclose(np.fd_reader_to_writer); - if(s.use_named_pipe && !s.writer) { + if (s.use_named_pipe && !s.writer) { HDremove(np.fifo_writer_to_reader); HDremove(np.fifo_reader_to_writer); } -- cgit v0.12 From da0ea555dff447a68bb6773d3e64888386fa692d Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 7 Jun 2021 15:09:33 +0000 Subject: Committing clang-format changes --- test/vfd_swmr_bigset_writer.c | 249 ++++++++++++++++++++++++------------------ 1 file changed, 144 insertions(+), 105 deletions(-) diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c index 65e3881..e5fae1f 100644 --- a/test/vfd_swmr_bigset_writer.c +++ b/test/vfd_swmr_bigset_writer.c @@ -113,13 +113,13 @@ typedef struct _sources { #define MANY_FILES 4 typedef struct { - hid_t *dataset; - sources_t *sources; - hid_t file[MANY_FILES]; - hid_t dapl, filetype, memspace, one_by_one_sid, quadrant_dcpl; - unsigned ndatasets; + hid_t * dataset; + sources_t * sources; + hid_t file[MANY_FILES]; + hid_t dapl, filetype, memspace, one_by_one_sid, quadrant_dcpl; + unsigned ndatasets; const char *filename[MANY_FILES]; - char progname[PATH_MAX]; + char progname[PATH_MAX]; struct { quadrant_t ul, ur, bl, br, src; } quadrants; @@ -128,23 +128,23 @@ typedef struct { unsigned int nsteps; bool two_dee; enum { vds_off, vds_single, vds_multi } vds; - bool use_vfd_swmr; - bool use_named_pipe; - bool writer; - bool fixed_array; - hsize_t chunk_dims[RANK]; - hsize_t one_dee_max_dims[RANK]; + bool use_vfd_swmr; + bool use_named_pipe; + bool writer; + bool fixed_array; + hsize_t chunk_dims[RANK]; + hsize_t one_dee_max_dims[RANK]; struct timespec ival; } state_t; /* Structure to hold info for named pipes */ typedef struct { - const char *fifo_writer_to_reader; /* Name of fifo for writer to reader */ - const char *fifo_reader_to_writer; /* Name of fifo for reader to writer */ - int fd_writer_to_reader; /* File ID for fifo from writer to reader */ - int fd_reader_to_writer; /* File ID for fifo from reader to writer */ - int notify; /* Value to notify between writer and reader */ - int verify; /* Value to verify between writer and reader */ + const char *fifo_writer_to_reader; /* Name of fifo for writer to reader */ + const char *fifo_reader_to_writer; /* Name of fifo for reader to writer */ + int fd_writer_to_reader; /* File ID for fifo from writer to reader */ + int fd_reader_to_writer; /* File ID for fifo from reader to writer */ + int notify; /* Value to notify between writer and reader */ + int verify; /* Value to verify between writer and reader */ } np_state_t; typedef struct { @@ -153,14 +153,13 @@ typedef struct { } exchange_info_t; /* Initializations for np_state_t */ -#define NP_INITIALIZER (np_state_t) { \ - .fifo_writer_to_reader = "./fifo_bigset_writer_to_reader", \ - .fifo_reader_to_writer = "./fifo_bigset_reader_to_writer", \ - .fd_writer_to_reader = -1, \ - .fd_reader_to_writer = -1, \ - .notify = 0, \ - .verify = 0 \ -} +#define NP_INITIALIZER \ + (np_state_t) \ + { \ + .fifo_writer_to_reader = "./fifo_bigset_writer_to_reader", \ + .fifo_reader_to_writer = "./fifo_bigset_reader_to_writer", .fd_writer_to_reader = -1, \ + .fd_reader_to_writer = -1, .notify = 0, .verify = 0 \ + } static inline state_t state_initializer(void) @@ -272,7 +271,7 @@ static bool make_quadrant_dataspace(state_t *s, quadrant_t *q) { if ((q->space = H5Screate_simple(NELMTS(s->chunk_dims), s->chunk_dims, - s->two_dee ? two_dee_max_dims : s->one_dee_max_dims)) < 0) { + s->two_dee ? two_dee_max_dims : s->one_dee_max_dims)) < 0) { fprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } @@ -285,9 +284,11 @@ make_quadrant_dataspace(state_t *s, quadrant_t *q) return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Sclose(q->space); - } H5E_END_TRY; + } + H5E_END_TRY; return false; } @@ -298,8 +299,8 @@ state_init(state_t *s, int argc, char **argv) unsigned long tmp; int ch; unsigned i; - const hsize_t dims = 1; - char *tfile = NULL; + const hsize_t dims = 1; + char * tfile = NULL; char * end; quadrant_t *const ul = &s->quadrants.ul, *const ur = &s->quadrants.ur, *const bl = &s->quadrants.bl, *const br = &s->quadrants.br, *const src = &s->quadrants.src; @@ -381,12 +382,13 @@ state_init(state_t *s, int argc, char **argv) else if (ch == 'l') { /* Translate the tick number to time represented by the timespec struct */ float time = (float)(((unsigned)tmp * TICK_LEN) / 10.0); - unsigned sec = (unsigned)time; + unsigned sec = (unsigned)time; unsigned nsec = (unsigned)((time - sec) * 10 * 1000 * 1000); - s->ival.tv_sec = sec; + s->ival.tv_sec = sec; s->ival.tv_nsec = nsec; - } else if (ch == 'n') + } + else if (ch == 'n') s->nsteps = (unsigned)tmp; else if (ch == 'r') s->rows = (unsigned)tmp; @@ -504,7 +506,8 @@ state_init(state_t *s, int argc, char **argv) TEST_ERROR; } - if (H5Sselect_hyperslab(src->space, H5S_SELECT_SET, src->start, src->stride, src->count, src->block) < 0) { + if (H5Sselect_hyperslab(src->space, H5S_SELECT_SET, src->start, src->stride, src->count, src->block) < + 0) { fprintf(stderr, "H5Sselect_hyperslab failed\n"); TEST_ERROR; } @@ -597,7 +600,8 @@ state_init(state_t *s, int argc, char **argv) return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Pclose(s->quadrant_dcpl); H5Sclose(ul->space); H5Sclose(ur->space); @@ -610,7 +614,8 @@ error: H5Sclose(src->space); H5Sclose(s->one_by_one_sid); H5Sclose(s->memspace); - } H5E_END_TRY; + } + H5E_END_TRY; if (tfile) HDfree(tfile); @@ -689,11 +694,13 @@ state_destroy(state_t *s) return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Pclose(s->quadrant_dcpl); H5Sclose(s->one_by_one_sid); H5Sclose(s->memspace); - } H5E_END_TRY; + } + H5E_END_TRY; if (s->dataset) HDfree(s->dataset); @@ -783,13 +790,13 @@ np_close(np_state_t *np, bool writer) } /* Reader finishes last and deletes the named pipes */ - if(!writer) { - if(HDremove(np->fifo_writer_to_reader) != 0) { + if (!writer) { + if (HDremove(np->fifo_writer_to_reader) != 0) { fprintf(stderr, "HDremove fifo_writer_to_reader failed\n"); TEST_ERROR; } - if(HDremove(np->fifo_reader_to_writer) != 0) { + if (HDremove(np->fifo_reader_to_writer) != 0) { fprintf(stderr, "HDremove fifo_reader_to_writer failed\n"); TEST_ERROR; } @@ -848,9 +855,11 @@ notify_and_wait_for_reader(state_t *s, np_state_t *np) for (i = 0; i < MAX_LAG + 1; i++) { decisleep(TICK_LEN); - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Aexists(s->file[0], "nonexistent"); - } H5E_END_TRY; + } + H5E_END_TRY; } /* Wait until the reader finishes validating zoo creation */ @@ -961,22 +970,26 @@ create_extensible_dset(state_t *s, unsigned int which) esnprintf(bl_dname, sizeof(bl_dname), "/bl-dataset-%d", which); esnprintf(br_dname, sizeof(br_dname), "/br-dataset-%d", which); - if ((srcs->ul = H5Dcreate2(s->file[0], ul_dname, s->filetype, ul->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { + if ((srcs->ul = H5Dcreate2(s->file[0], ul_dname, s->filetype, ul->src_space, H5P_DEFAULT, + s->quadrant_dcpl, s->dapl)) < 0) { fprintf(stderr, "H5Dcreate2 failed\n"); TEST_ERROR; } - if ((srcs->ur = H5Dcreate2(s->file[1], ur_dname, s->filetype, ur->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { + if ((srcs->ur = H5Dcreate2(s->file[1], ur_dname, s->filetype, ur->src_space, H5P_DEFAULT, + s->quadrant_dcpl, s->dapl)) < 0) { fprintf(stderr, "H5Dcreate2 failed\n"); TEST_ERROR; } - if ((srcs->bl = H5Dcreate2(s->file[2], bl_dname, s->filetype, bl->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { + if ((srcs->bl = H5Dcreate2(s->file[2], bl_dname, s->filetype, bl->src_space, H5P_DEFAULT, + s->quadrant_dcpl, s->dapl)) < 0) { fprintf(stderr, "H5Dcreate2 failed\n"); TEST_ERROR; } - if ((srcs->br = H5Dcreate2(s->file[3], br_dname, s->filetype, br->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { + if ((srcs->br = H5Dcreate2(s->file[3], br_dname, s->filetype, br->src_space, H5P_DEFAULT, + s->quadrant_dcpl, s->dapl)) < 0) { fprintf(stderr, "H5Dcreate2 failed\n"); TEST_ERROR; } @@ -1003,7 +1016,7 @@ create_extensible_dset(state_t *s, unsigned int which) } if ((filespace = H5Screate_simple(NELMTS(s->chunk_dims), s->chunk_dims, - s->two_dee ? two_dee_max_dims : s->one_dee_max_dims)) < 0) { + s->two_dee ? two_dee_max_dims : s->one_dee_max_dims)) < 0) { fprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } @@ -1028,11 +1041,13 @@ create_extensible_dset(state_t *s, unsigned int which) return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Dclose(dset_id); H5Pclose(dcpl); H5Sclose(filespace); - } H5E_END_TRY; + } + H5E_END_TRY; return false; } @@ -1072,9 +1087,11 @@ close_extensible_dset(state_t *s, unsigned int which) return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Dclose(dset_id); - } H5E_END_TRY; + } + H5E_END_TRY; return false; } @@ -1094,9 +1111,11 @@ open_extensible_dset(state_t *s) * NUM_ATTEMPTS times without success, report it as a failure */ for (i = 0; i < NUM_ATTEMPTS; i++) { - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { dset_id = H5Dopen2(s->file[0], dname, s->dapl); - } H5E_END_TRY; + } + H5E_END_TRY; if (dset_id >= 0) break; @@ -1148,14 +1167,15 @@ open_extensible_dset(state_t *s) if (maxdims[0] != two_dee_max_dims[0] || maxdims[1] != two_dee_max_dims[1] || maxdims[0] != maxdims[1]) { fprintf(stderr, "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE, maxdims[0], - maxdims[1]); + maxdims[1]); TEST_ERROR; } - } else if (maxdims[0] != s->one_dee_max_dims[0] || maxdims[1] != s->one_dee_max_dims[1] || - dims[0] != s->chunk_dims[0]) { + } + else if (maxdims[0] != s->one_dee_max_dims[0] || maxdims[1] != s->one_dee_max_dims[1] || + dims[0] != s->chunk_dims[0]) { fprintf(stderr, - "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE " or columns %" PRIuHSIZE, - maxdims[0], maxdims[1], dims[1]); + "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE " or columns %" PRIuHSIZE, + maxdims[0], maxdims[1], dims[1]); } s->dataset[which] = dset_id; @@ -1164,11 +1184,13 @@ open_extensible_dset(state_t *s) return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Dclose(dset_id); H5Tclose(dtype); H5Sclose(filespace); - } H5E_END_TRY; + } + H5E_END_TRY; return false; } @@ -1222,7 +1244,8 @@ set_or_verify_matrix(mat_t *mat, unsigned int which, base_t base, bool do_set) ret = false; break; } - } else if (matget(mat, row, col) != v) { + } + else if (matget(mat, row, col) != v) { /* If the data doesn't match, simply return false and * let the caller repeat this step */ @@ -1268,12 +1291,14 @@ verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t bas TEST_ERROR; } - /* A failure to read the data may indicate the data isn't ready yet. Instead of displaying the error stack, - * simply return false and let the caller repeat this step. + /* A failure to read the data may indicate the data isn't ready yet. Instead of displaying the error + * stack, simply return false and let the caller repeat this step. */ - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { status = H5Dread(dset_id, H5T_NATIVE_UINT32, s->memspace, filespace, H5P_DEFAULT, mat->elt); - } H5E_END_TRY; + } + H5E_END_TRY; if (status < 0) TEST_ERROR; @@ -1379,9 +1404,11 @@ verify_dset_attribute(hid_t dset_id, unsigned int which, unsigned int step) return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Aclose(aid); - } H5E_END_TRY; + } + H5E_END_TRY; return false; } @@ -1492,9 +1519,11 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Sclose(filespace); - } H5E_END_TRY; + } + H5E_END_TRY; return false; } @@ -1502,7 +1531,7 @@ error: static bool verify_dsets(state_t s, np_state_t *np, mat_t *mat) { - unsigned *nextstep = NULL; + unsigned * nextstep = NULL; unsigned finished_step = 0; unsigned which; exchange_info_t last; @@ -1585,9 +1614,11 @@ add_dset_attribute(const state_t *s, hid_t ds, hid_t sid, unsigned int which, un return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Aclose(aid); - } H5E_END_TRY; + } + H5E_END_TRY; return false; } @@ -1623,7 +1654,8 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * size[1] = s->chunk_dims[1] * (1 + step); last.row = s->chunk_dims[0] * step; last.col = s->chunk_dims[1] * step; - } else { + } + else { size[0] = s->chunk_dims[0]; size[1] = s->chunk_dims[1] * (1 + step); last.row = 0; @@ -1655,7 +1687,8 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * fprintf(stderr, "H5Dset_extent failed\n"); TEST_ERROR; } - } else if (H5Dset_extent(dset_id, size) < 0) { + } + else if (H5Dset_extent(dset_id, size) < 0) { fprintf(stderr, "H5Dset_extent failed\n"); TEST_ERROR; } @@ -1699,9 +1732,11 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Sclose(filespace); - } H5E_END_TRY; + } + H5E_END_TRY; return false; } @@ -1709,9 +1744,9 @@ error: static bool write_dsets(state_t s, np_state_t *np, mat_t *mat) { - unsigned last_step, step, which; + unsigned last_step, step, which; unsigned long long old_tick_num; - H5F_t *f = NULL; + H5F_t * f = NULL; if (NULL == (f = (H5F_t *)H5VL_object(s.file[0]))) { fprintf(stderr, "H5VL_object failed\n"); @@ -1737,13 +1772,13 @@ write_dsets(state_t s, np_state_t *np, mat_t *mat) /* After finishing writing all the chunks, end the tick */ if (s.use_vfd_swmr && step == (s.nsteps - 1)) { - unsigned long i; + unsigned long i; - if (s.vds != vds_multi) - H5Fvfd_swmr_end_tick(s.file[0]); - else - for (i = 0; i < NELMTS(s.file); i++) - H5Fvfd_swmr_end_tick(s.file[i]); + if (s.vds != vds_multi) + H5Fvfd_swmr_end_tick(s.file[0]); + else + for (i = 0; i < NELMTS(s.file); i++) + H5Fvfd_swmr_end_tick(s.file[i]); } /* Notify the reader to start verification by @@ -1769,7 +1804,7 @@ error: int main(int argc, char **argv) { - mat_t *mat; + mat_t * mat; hid_t fcpl = H5I_INVALID_HID; unsigned which; state_t s; @@ -1849,24 +1884,25 @@ main(int argc, char **argv) if (!create_extensible_dset(&s, which)) { fprintf(stderr, "create_extensible_dset failed: number %u\n", which); TEST_ERROR; - } + } /* Call H5Fvfd_swmr_end_tick to end the tick. No communication with the reader in this step */ if (s.use_vfd_swmr && s.use_named_pipe) { - unsigned long j; - - if (s.vds != vds_multi) { - if (H5Fvfd_swmr_end_tick(s.file[0]) < 0) { - fprintf(stderr, "H5Fvfd_swmr_end_tick failed\n"); - TEST_ERROR; - } - } else { - for (j = 0; j < NELMTS(s.file); j++) - if (H5Fvfd_swmr_end_tick(s.file[j]) < 0) { - fprintf(stderr, "H5Fvfd_swmr_end_tick failed\n"); - TEST_ERROR; - } - } + unsigned long j; + + if (s.vds != vds_multi) { + if (H5Fvfd_swmr_end_tick(s.file[0]) < 0) { + fprintf(stderr, "H5Fvfd_swmr_end_tick failed\n"); + TEST_ERROR; + } + } + else { + for (j = 0; j < NELMTS(s.file); j++) + if (H5Fvfd_swmr_end_tick(s.file[j]) < 0) { + fprintf(stderr, "H5Fvfd_swmr_end_tick failed\n"); + TEST_ERROR; + } + } } /* Notify the reader of finishing dataset creation by sending the timestamp @@ -1883,7 +1919,8 @@ main(int argc, char **argv) fprintf(stderr, "write_dsets failed"); TEST_ERROR; } - } else { + } + else { /* Wait for the writer's notice before starting the validation of dataset creation */ np.verify = 1; if (s.use_named_pipe && reader_verify(np, np.verify) < 0) { @@ -1944,12 +1981,14 @@ main(int argc, char **argv) return EXIT_SUCCESS; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Pclose(fcpl); for (i = 0; i < NELMTS(s.file); i++) H5Fclose(s.file[i]); - } H5E_END_TRY; + } + H5E_END_TRY; if (s.use_named_pipe && np.fd_writer_to_reader >= 0) HDclose(np.fd_writer_to_reader); -- cgit v0.12 From 4f0a03189b6d92cdb6c893fa86e83a05953cae0e Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Thu, 8 Jul 2021 11:17:44 -0500 Subject: 1. Added the test for 3D datasets; 2. Added performance measurement. --- test/vfd_swmr_bigset_writer.c | 813 +++++++++++++++++++++++++++--------------- 1 file changed, 518 insertions(+), 295 deletions(-) diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c index e5fae1f..48981d5 100644 --- a/test/vfd_swmr_bigset_writer.c +++ b/test/vfd_swmr_bigset_writer.c @@ -86,11 +86,20 @@ #define MAX_LAG 7 #define ROWS 256 #define COLS 512 -#define RANK 2 +#define DEPTHS 1 +#define RANK2 2 +#define RANK3 3 #define NUM_ATTEMPTS 100 +/* Calculate the time passed in seconds. + * X is the beginning time; Y is the ending time. + * Expects X, Y to be struct timespec from the function call HDclock_gettime. + */ +#define TIME_PASSED(X,Y) \ + ((double)((Y.tv_sec - X.tv_sec) * 1000000000 + (Y.tv_nsec - X.tv_nsec))) / 1000000000.0 + typedef struct _base { - hsize_t row, col; + hsize_t depth, row, col; } base_t; typedef struct _mat { @@ -99,10 +108,10 @@ typedef struct _mat { } mat_t; typedef struct _quadrant { - hsize_t start[RANK]; - hsize_t stride[RANK]; - hsize_t block[RANK]; - hsize_t count[RANK]; + hsize_t start[RANK2]; + hsize_t stride[RANK2]; + hsize_t block[RANK2]; + hsize_t count[RANK2]; hid_t space, src_space; } quadrant_t; @@ -113,38 +122,41 @@ typedef struct _sources { #define MANY_FILES 4 typedef struct { - hid_t * dataset; - sources_t * sources; - hid_t file[MANY_FILES]; - hid_t dapl, filetype, memspace, one_by_one_sid, quadrant_dcpl; - unsigned ndatasets; + hid_t *dataset; + sources_t *sources; + hid_t file[MANY_FILES]; + hid_t dapl, filetype, memspace, one_by_one_sid, quadrant_dcpl; + unsigned ndatasets; const char *filename[MANY_FILES]; - char progname[PATH_MAX]; + char progname[PATH_MAX]; struct { quadrant_t ul, ur, bl, br, src; } quadrants; unsigned int cols, rows; unsigned int asteps; unsigned int nsteps; - bool two_dee; + bool expand_2d; + bool test_3d; enum { vds_off, vds_single, vds_multi } vds; - bool use_vfd_swmr; - bool use_named_pipe; - bool writer; - bool fixed_array; - hsize_t chunk_dims[RANK]; - hsize_t one_dee_max_dims[RANK]; + bool use_vfd_swmr; + bool use_named_pipe; + bool do_perf; + bool cross_chunks; + bool writer; + bool fixed_array; + hsize_t chunk_dims[RANK2]; + hsize_t one_dee_max_dims[RANK2]; struct timespec ival; } state_t; /* Structure to hold info for named pipes */ typedef struct { - const char *fifo_writer_to_reader; /* Name of fifo for writer to reader */ - const char *fifo_reader_to_writer; /* Name of fifo for reader to writer */ - int fd_writer_to_reader; /* File ID for fifo from writer to reader */ - int fd_reader_to_writer; /* File ID for fifo from reader to writer */ - int notify; /* Value to notify between writer and reader */ - int verify; /* Value to verify between writer and reader */ + const char *fifo_writer_to_reader; /* Name of fifo for writer to reader */ + const char *fifo_reader_to_writer; /* Name of fifo for reader to writer */ + int fd_writer_to_reader; /* File ID for fifo from writer to reader */ + int fd_reader_to_writer; /* File ID for fifo from reader to writer */ + int notify; /* Value to notify between writer and reader */ + int verify; /* Value to verify between writer and reader */ } np_state_t; typedef struct { @@ -153,13 +165,14 @@ typedef struct { } exchange_info_t; /* Initializations for np_state_t */ -#define NP_INITIALIZER \ - (np_state_t) \ - { \ - .fifo_writer_to_reader = "./fifo_bigset_writer_to_reader", \ - .fifo_reader_to_writer = "./fifo_bigset_reader_to_writer", .fd_writer_to_reader = -1, \ - .fd_reader_to_writer = -1, .notify = 0, .verify = 0 \ - } +#define NP_INITIALIZER (np_state_t) { \ + .fifo_writer_to_reader = "./fifo_bigset_writer_to_reader", \ + .fifo_reader_to_writer = "./fifo_bigset_reader_to_writer", \ + .fd_writer_to_reader = -1, \ + .fd_reader_to_writer = -1, \ + .notify = 0, \ + .verify = 0 \ +} static inline state_t state_initializer(void) @@ -176,10 +189,13 @@ state_initializer(void) .asteps = 10, .nsteps = 100, .filename = {"", "", "", ""}, - .two_dee = false, + .expand_2d = false, + .test_3d = false, .vds = vds_off, .use_vfd_swmr = true, .use_named_pipe = true, + .do_perf = false, + .cross_chunks = false, .writer = true, .fixed_array = false, .one_dee_max_dims = {ROWS, H5S_UNLIMITED}, @@ -191,7 +207,7 @@ static bool state_init(state_t *, int, char **); static const hid_t badhid = H5I_INVALID_HID; -static hsize_t two_dee_max_dims[RANK]; +static hsize_t two_dee_max_dims[RANK2], three_dee_max_dims[RANK3]; static uint32_t matget(const mat_t *mat, unsigned i, unsigned j) @@ -240,28 +256,32 @@ static void usage(const char *progname) { fprintf(stderr, - "usage: %s [-F] [-M] [-S] [-V] [-W] [-a steps] [-b] [-c cols]\n" + "usage: %s [-C] [-F] [-M] [-P] [-S] [-V] [-W] [-a steps] [-b] [-c cols]\n" " [-d dims]\n" " [-l tick_num] [-n iterations] [-r rows] [-s datasets]\n" - " [-u milliseconds]\n" + " [-t] [-u milliseconds]\n" "\n" + "-C: cross-over chunks during chunk verification\n" "-F: fixed maximal dimension for the chunked datasets\n" - "-M: use virtual datasets and many source\n" + "-M: use virtual datasets and many source\n" " files\n" - "-S: do not use VFD SWMR\n" - "-V: use virtual datasets and a single\n" + "-P: do the performance measurement" + "-S: do not use VFD SWMR\n" + "-V: use virtual datasets and a single\n" " source file\n" - "-a steps: `steps` between adding attributes\n" - "-b: write data in big-endian byte order\n" - "-c cols: `cols` columns per chunk\n" + "-a steps: `steps` between adding attributes\n" + "-b: write data in big-endian byte order\n" + "-c cols: `cols` columns per chunk\n" "-d 1|one|2|two|both: select dataset expansion in one or\n" " both dimensions\n" "-l tick_num: expected maximal number of ticks from\n" " the writer's finishing creation to the reader's finishing validation\n" "-N: do not use named pipes\n" "-n iterations: how many times to expand each dataset\n" - "-r rows: `rows` rows per chunk\n" + "-r rows: `rows` rows per chunk\n" "-s datasets: number of datasets to create\n" + "-t: enable test for 3D datasets (dataset expansion is along one dimension)\n" + " currently, 3D datasets isn't tested with VDS\n" "\n", progname); exit(EXIT_FAILURE); @@ -271,7 +291,7 @@ static bool make_quadrant_dataspace(state_t *s, quadrant_t *q) { if ((q->space = H5Screate_simple(NELMTS(s->chunk_dims), s->chunk_dims, - s->two_dee ? two_dee_max_dims : s->one_dee_max_dims)) < 0) { + s->expand_2d ? two_dee_max_dims : s->one_dee_max_dims)) < 0) { fprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } @@ -284,11 +304,9 @@ make_quadrant_dataspace(state_t *s, quadrant_t *q) return true; error: - H5E_BEGIN_TRY - { + H5E_BEGIN_TRY { H5Sclose(q->space); - } - H5E_END_TRY; + } H5E_END_TRY; return false; } @@ -299,8 +317,8 @@ state_init(state_t *s, int argc, char **argv) unsigned long tmp; int ch; unsigned i; - const hsize_t dims = 1; - char * tfile = NULL; + const hsize_t dims = 1; + char *tfile = NULL; char * end; quadrant_t *const ul = &s->quadrants.ul, *const ur = &s->quadrants.ur, *const bl = &s->quadrants.bl, *const br = &s->quadrants.br, *const src = &s->quadrants.src; @@ -315,11 +333,14 @@ state_init(state_t *s, int argc, char **argv) esnprintf(s->progname, sizeof(s->progname), "%s", tfile); - if (tfile) - HDfree(tfile); + HDfree(tfile); - while ((ch = getopt(argc, argv, "FMNSVa:bc:d:l:n:qr:s:")) != -1) { + while ((ch = getopt(argc, argv, "CFMNPSVa:bc:d:l:n:qr:s:t")) != -1) { switch (ch) { + case 'C': + /* This flag indicates cross-over chunk read during data validation */ + s->cross_chunks = true; + break; case 'F': /* The flag to indicate whether the maximal dimension of the chunked datasets is fixed or * unlimited */ @@ -328,6 +349,9 @@ state_init(state_t *s, int argc, char **argv) case 'M': s->vds = vds_multi; break; + case 'P': + s->do_perf = true; + break; case 'S': s->use_vfd_swmr = false; break; @@ -340,10 +364,10 @@ state_init(state_t *s, int argc, char **argv) break; case 'd': if (strcmp(optarg, "1") == 0 || strcmp(optarg, "one") == 0) - s->two_dee = false; + s->expand_2d = false; else if (strcmp(optarg, "2") == 0 || strcmp(optarg, "two") == 0 || strcmp(optarg, "both") == 0) - s->two_dee = true; + s->expand_2d = true; else { fprintf(stderr, "bad -d argument %s\n", optarg); TEST_ERROR; @@ -382,19 +406,21 @@ state_init(state_t *s, int argc, char **argv) else if (ch == 'l') { /* Translate the tick number to time represented by the timespec struct */ float time = (float)(((unsigned)tmp * TICK_LEN) / 10.0); - unsigned sec = (unsigned)time; + unsigned sec = (unsigned)time; unsigned nsec = (unsigned)((time - sec) * 10 * 1000 * 1000); - s->ival.tv_sec = sec; + s->ival.tv_sec = sec; s->ival.tv_nsec = nsec; - } - else if (ch == 'n') + } else if (ch == 'n') s->nsteps = (unsigned)tmp; else if (ch == 'r') s->rows = (unsigned)tmp; else s->ndatasets = (unsigned)tmp; break; + case 't': + s->test_3d = true; + break; case 'b': s->filetype = H5T_STD_U32BE; break; @@ -415,27 +441,56 @@ state_init(state_t *s, int argc, char **argv) TEST_ERROR; } - if (s->vds != vds_off && s->two_dee) { + if (s->vds != vds_off && s->expand_2d) { fprintf(stderr, "virtual datasets and 2D datasets are mutually exclusive\n"); TEST_ERROR; } + if (s->test_3d) { + if (s->expand_2d) { + fprintf(stderr, "3D dataset test doesn't support 2D expansion\n"); + TEST_ERROR; + } + + if (s->cross_chunks) { + fprintf(stderr, "3D dataset test doesn't support cross-over chunks during verification\n"); + TEST_ERROR; + } + + if (s->vds != vds_off) { + fprintf(stderr, "3D dataset test doesn't support VDS\n"); + TEST_ERROR; + } + } + s->chunk_dims[0] = s->rows; s->chunk_dims[1] = s->cols; + s->one_dee_max_dims[0] = s->rows; if (s->fixed_array) { s->one_dee_max_dims[1] = s->cols * s->nsteps; two_dee_max_dims[0] = s->rows * s->nsteps; two_dee_max_dims[1] = s->cols * s->nsteps; - } - else { + + if(s->test_3d) { + three_dee_max_dims[0] = s->nsteps; + three_dee_max_dims[1] = s->rows; + three_dee_max_dims[2] = s->cols; + } + } else { s->one_dee_max_dims[1] = H5S_UNLIMITED; two_dee_max_dims[0] = two_dee_max_dims[1] = H5S_UNLIMITED; + + if(s->test_3d) { + three_dee_max_dims[0] = H5S_UNLIMITED; + three_dee_max_dims[1] = s->rows; + three_dee_max_dims[2] = s->cols; + } } if (s->vds != vds_off) { - const hsize_t half_chunk_dims[RANK] = {s->rows / 2, s->cols / 2}; - hsize_t half_max_dims[RANK]; + const hsize_t half_chunk_dims[RANK2] = {s->rows / 2, s->cols / 2}; + hsize_t half_max_dims[RANK2]; if (s->fixed_array) { half_max_dims[0] = s->rows / 2; @@ -451,7 +506,7 @@ state_init(state_t *s, int argc, char **argv) TEST_ERROR; } - if (H5Pset_chunk(s->quadrant_dcpl, RANK, half_chunk_dims) < 0) { + if (H5Pset_chunk(s->quadrant_dcpl, RANK2, half_chunk_dims) < 0) { fprintf(stderr, "H5Pset_chunk failed\n"); TEST_ERROR; } @@ -501,33 +556,32 @@ state_init(state_t *s, int argc, char **argv) .block = {s->rows / 2, s->cols / 2}, .count = {1, H5S_UNLIMITED}}; - if ((src->space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims)) < 0) { + if ((src->space = H5Screate_simple(RANK2, half_chunk_dims, half_max_dims)) < 0) { fprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } - if (H5Sselect_hyperslab(src->space, H5S_SELECT_SET, src->start, src->stride, src->count, src->block) < - 0) { + if (H5Sselect_hyperslab(src->space, H5S_SELECT_SET, src->start, src->stride, src->count, src->block) < 0) { fprintf(stderr, "H5Sselect_hyperslab failed\n"); TEST_ERROR; } - if ((ul->src_space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims)) < 0) { + if ((ul->src_space = H5Screate_simple(RANK2, half_chunk_dims, half_max_dims)) < 0) { fprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } - if ((ur->src_space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims)) < 0) { + if ((ur->src_space = H5Screate_simple(RANK2, half_chunk_dims, half_max_dims)) < 0) { fprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } - if ((bl->src_space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims)) < 0) { + if ((bl->src_space = H5Screate_simple(RANK2, half_chunk_dims, half_max_dims)) < 0) { fprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } - if ((br->src_space = H5Screate_simple(RANK, half_chunk_dims, half_max_dims)) < 0) { + if ((br->src_space = H5Screate_simple(RANK2, half_chunk_dims, half_max_dims)) < 0) { fprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } @@ -554,9 +608,18 @@ state_init(state_t *s, int argc, char **argv) s->sources[i].ul = s->sources[i].ur = s->sources[i].bl = s->sources[i].br = badhid; } - if ((s->memspace = H5Screate_simple(RANK, s->chunk_dims, NULL)) < 0) { - fprintf(stderr, "H5Screate_simple failed\n"); - TEST_ERROR; + if (s->test_3d) { + hsize_t dims3[RANK3] = {1, s->chunk_dims[0], s->chunk_dims[1]}; + + if ((s->memspace = H5Screate_simple(RANK3, dims3, NULL)) < 0) { + fprintf(stderr, "H5Screate_simple failed\n"); + TEST_ERROR; + } + } else { + if ((s->memspace = H5Screate_simple(RANK2, s->chunk_dims, NULL)) < 0) { + fprintf(stderr, "H5Screate_simple failed\n"); + TEST_ERROR; + } } s->filename[0] = "vfd_swmr_bigset.h5"; @@ -600,8 +663,7 @@ state_init(state_t *s, int argc, char **argv) return true; error: - H5E_BEGIN_TRY - { + H5E_BEGIN_TRY { H5Pclose(s->quadrant_dcpl); H5Sclose(ul->space); H5Sclose(ur->space); @@ -614,17 +676,13 @@ error: H5Sclose(src->space); H5Sclose(s->one_by_one_sid); H5Sclose(s->memspace); - } - H5E_END_TRY; + } H5E_END_TRY; - if (tfile) - HDfree(tfile); + HDfree(tfile); - if (s->dataset) - HDfree(s->dataset); + HDfree(s->dataset); - if (s->sources) - HDfree(s->sources); + HDfree(s->sources); return false; } @@ -633,6 +691,7 @@ static bool state_destroy(state_t *s) { size_t i; + struct timespec start_time, end_time; if (H5Pclose(s->dapl) < 0) { fprintf(stderr, "H5Pclose failed\n"); @@ -671,6 +730,14 @@ state_destroy(state_t *s) TEST_ERROR; } + /* For checking the time spent in file close. It's for running the writer alone */ + if (s->do_perf) { + if (HDclock_gettime(CLOCK_MONOTONIC, &start_time) == -1) { + fprintf(stderr, "HDclock_gettime failed"); + TEST_ERROR; + } + } + for (i = 0; i < NELMTS(s->file); i++) { hid_t fid = s->file[i]; @@ -685,28 +752,32 @@ state_destroy(state_t *s) } } - if (s->dataset) - HDfree(s->dataset); + /* For checking the time spent in file close. It's for running the writer alone */ + if (s->do_perf) { + if (HDclock_gettime(CLOCK_MONOTONIC, &end_time) == -1) { + fprintf(stderr, "HDclock_gettime failed"); + TEST_ERROR; + } + + fprintf(stdout, "File close time (for running the writer alone) = %lf\n", TIME_PASSED(start_time, end_time)); + } + + HDfree(s->dataset); - if (s->sources) - HDfree(s->sources); + HDfree(s->sources); return true; error: - H5E_BEGIN_TRY - { + H5E_BEGIN_TRY { H5Pclose(s->quadrant_dcpl); H5Sclose(s->one_by_one_sid); H5Sclose(s->memspace); - } - H5E_END_TRY; + } H5E_END_TRY; - if (s->dataset) - HDfree(s->dataset); + HDfree(s->dataset); - if (s->sources) - HDfree(s->sources); + HDfree(s->sources); return false; } @@ -790,13 +861,13 @@ np_close(np_state_t *np, bool writer) } /* Reader finishes last and deletes the named pipes */ - if (!writer) { - if (HDremove(np->fifo_writer_to_reader) != 0) { + if(!writer) { + if(HDremove(np->fifo_writer_to_reader) != 0) { fprintf(stderr, "HDremove fifo_writer_to_reader failed\n"); TEST_ERROR; } - if (HDremove(np->fifo_reader_to_writer) != 0) { + if(HDremove(np->fifo_reader_to_writer) != 0) { fprintf(stderr, "HDremove fifo_reader_to_writer failed\n"); TEST_ERROR; } @@ -807,7 +878,7 @@ error: return false; } /* np_close() */ -/* Wait for the writer's notice before starting to zoo validation */ +/* Wait for the writer's notice before starting validation */ static int reader_verify(np_state_t np, int verify) { @@ -829,7 +900,7 @@ error: return -1; } -/* Notify the reader of finishing zoo creation by sending the timestamp +/* Notify the reader of finishing creation by sending the timestamp * and wait for the reader to finish validation before proceeding */ static int notify_and_wait_for_reader(state_t *s, np_state_t *np) @@ -838,13 +909,13 @@ notify_and_wait_for_reader(state_t *s, np_state_t *np) unsigned int i; struct timespec last = {0, 0}; - /* Get the time when finishing zoo creation */ + /* Get the time when finishing creation */ if (HDclock_gettime(CLOCK_MONOTONIC, &last) < 0) { fprintf(stderr, "HDclock_gettime failed\n"); TEST_ERROR; } - /* Notify the reader of finishing zoo creation by sending the timestamp */ + /* Notify the reader of finishing creation by sending the timestamp */ if (HDwrite(np->fd_writer_to_reader, &last, sizeof(last)) < 0) { fprintf(stderr, "HDwrite failed\n"); TEST_ERROR; @@ -855,14 +926,12 @@ notify_and_wait_for_reader(state_t *s, np_state_t *np) for (i = 0; i < MAX_LAG + 1; i++) { decisleep(TICK_LEN); - H5E_BEGIN_TRY - { + H5E_BEGIN_TRY { H5Aexists(s->file[0], "nonexistent"); - } - H5E_END_TRY; + } H5E_END_TRY; } - /* Wait until the reader finishes validating zoo creation */ + /* Wait until the reader finishes validating creation */ if (HDread(np->fd_reader_to_writer, ¬ify, sizeof(int)) < 0) { fprintf(stderr, "HDread failed\n"); TEST_ERROR; @@ -888,18 +957,18 @@ reader_check_time_and_notify_writer(np_state_t *np, state_t s) { struct timespec last = {0, 0}; - /* Receive the notice of the writer finishing zoo creation (timestamp) */ + /* Receive the notice of the writer finishing creation (timestamp) */ if (HDread(np->fd_writer_to_reader, &last, sizeof(last)) < 0) { fprintf(stderr, "HDread failed\n"); TEST_ERROR; } - /* Make sure the dataset validation doesn't take longer than the expected time. + /* If the dataset validation takes longer than the expected time, issue a warning. * This time period is from the writer finishing dataset creation to the reader finishing * the validation of dataset creation */ if (below_speed_limit(&last, &(s.ival))) { AT(); - fprintf(stderr, "dataset validation took too long to finish\n"); + fprintf(stderr, "Warning: dataset validation took too long to finish\n"); } /* Notify the writer that dataset validation is finished */ @@ -949,6 +1018,7 @@ create_extensible_dset(state_t *s, unsigned int which) char ul_dname[sizeof("/ul-dataset-9999999999")], ur_dname[sizeof("/ur-dataset-9999999999")], bl_dname[sizeof("/bl-dataset-9999999999")], br_dname[sizeof("/br-dataset-9999999999")]; hid_t dcpl = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, filespace = H5I_INVALID_HID; + hsize_t dims3[3] = {1, s->chunk_dims[0], s->chunk_dims[1]}; esnprintf(dname, sizeof(dname), "/dataset-%d", which); @@ -957,9 +1027,17 @@ create_extensible_dset(state_t *s, unsigned int which) TEST_ERROR; } - if (H5Pset_chunk(dcpl, RANK, s->chunk_dims) < 0) { - fprintf(stderr, "H5Pset_chunk failed\n"); - TEST_ERROR; + if (s->test_3d) { + /* The chunk is 1 x M x N and grows along the first dimension */ + if (H5Pset_chunk(dcpl, RANK3, dims3) < 0) { + fprintf(stderr, "H5Pset_chunk for 3D dataset failed\n"); + TEST_ERROR; + } + } else { + if (H5Pset_chunk(dcpl, RANK2, s->chunk_dims) < 0) { + fprintf(stderr, "H5Pset_chunk for 2D dataset failed\n"); + TEST_ERROR; + } } if (s->vds != vds_off) { @@ -970,26 +1048,22 @@ create_extensible_dset(state_t *s, unsigned int which) esnprintf(bl_dname, sizeof(bl_dname), "/bl-dataset-%d", which); esnprintf(br_dname, sizeof(br_dname), "/br-dataset-%d", which); - if ((srcs->ul = H5Dcreate2(s->file[0], ul_dname, s->filetype, ul->src_space, H5P_DEFAULT, - s->quadrant_dcpl, s->dapl)) < 0) { + if ((srcs->ul = H5Dcreate2(s->file[0], ul_dname, s->filetype, ul->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { fprintf(stderr, "H5Dcreate2 failed\n"); TEST_ERROR; } - if ((srcs->ur = H5Dcreate2(s->file[1], ur_dname, s->filetype, ur->src_space, H5P_DEFAULT, - s->quadrant_dcpl, s->dapl)) < 0) { + if ((srcs->ur = H5Dcreate2(s->file[1], ur_dname, s->filetype, ur->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { fprintf(stderr, "H5Dcreate2 failed\n"); TEST_ERROR; } - if ((srcs->bl = H5Dcreate2(s->file[2], bl_dname, s->filetype, bl->src_space, H5P_DEFAULT, - s->quadrant_dcpl, s->dapl)) < 0) { + if ((srcs->bl = H5Dcreate2(s->file[2], bl_dname, s->filetype, bl->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { fprintf(stderr, "H5Dcreate2 failed\n"); TEST_ERROR; } - if ((srcs->br = H5Dcreate2(s->file[3], br_dname, s->filetype, br->src_space, H5P_DEFAULT, - s->quadrant_dcpl, s->dapl)) < 0) { + if ((srcs->br = H5Dcreate2(s->file[3], br_dname, s->filetype, br->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { fprintf(stderr, "H5Dcreate2 failed\n"); TEST_ERROR; } @@ -1015,10 +1089,17 @@ create_extensible_dset(state_t *s, unsigned int which) } } - if ((filespace = H5Screate_simple(NELMTS(s->chunk_dims), s->chunk_dims, - s->two_dee ? two_dee_max_dims : s->one_dee_max_dims)) < 0) { - fprintf(stderr, "H5Screate_simple failed\n"); - TEST_ERROR; + if (s->test_3d) { + if ((filespace = H5Screate_simple(RANK3, dims3, three_dee_max_dims)) < 0) { + fprintf(stderr, "H5Screate_simple 3D dataspace failed\n"); + TEST_ERROR; + } + } else { + if ((filespace = H5Screate_simple(RANK2, s->chunk_dims, + s->expand_2d ? two_dee_max_dims : s->one_dee_max_dims)) < 0) { + fprintf(stderr, "H5Screate_simple 2D dataspace failed\n"); + TEST_ERROR; + } } if ((dset_id = H5Dcreate2(s->file[0], dname, s->filetype, filespace, H5P_DEFAULT, dcpl, s->dapl)) < 0) { @@ -1041,13 +1122,11 @@ create_extensible_dset(state_t *s, unsigned int which) return true; error: - H5E_BEGIN_TRY - { + H5E_BEGIN_TRY { H5Dclose(dset_id); H5Pclose(dcpl); H5Sclose(filespace); - } - H5E_END_TRY; + } H5E_END_TRY; return false; } @@ -1087,11 +1166,9 @@ close_extensible_dset(state_t *s, unsigned int which) return true; error: - H5E_BEGIN_TRY - { + H5E_BEGIN_TRY { H5Dclose(dset_id); - } - H5E_END_TRY; + } H5E_END_TRY; return false; } @@ -1099,9 +1176,11 @@ error: static bool open_extensible_dset(state_t *s) { - hsize_t dims[RANK], maxdims[RANK]; + hsize_t dims2[RANK2], maxdims2[RANK2]; + hsize_t dims3[RANK3], maxdims3[RANK3]; char dname[sizeof("/dataset-9999999999")]; hid_t dset_id, filespace, dtype; + int rank; unsigned int which, i; for (which = 0; which < s->ndatasets; which++) { @@ -1111,11 +1190,9 @@ open_extensible_dset(state_t *s) * NUM_ATTEMPTS times without success, report it as a failure */ for (i = 0; i < NUM_ATTEMPTS; i++) { - H5E_BEGIN_TRY - { + H5E_BEGIN_TRY { dset_id = H5Dopen2(s->file[0], dname, s->dapl); - } - H5E_END_TRY; + } H5E_END_TRY; if (dset_id >= 0) break; @@ -1143,16 +1220,28 @@ open_extensible_dset(state_t *s) TEST_ERROR; } - if (H5Sget_simple_extent_ndims(filespace) != RANK) { - fprintf(stderr, "Unexpected data rank\n"); + if ((rank = H5Sget_simple_extent_ndims(filespace)) < 0) { + fprintf(stderr, "H5Sget_simple_extent_ndims failed\n"); TEST_ERROR; } - if (H5Sget_simple_extent_dims(filespace, dims, maxdims) < 0) { - fprintf(stderr, "H5Sget_simple_extent_dims failed\n"); + if ((s->test_3d && rank != RANK3) || (!s->test_3d && rank != RANK2)) { + fprintf(stderr, "Unexpected data rank: %d\n", rank); TEST_ERROR; } + if (s->test_3d) { + if (H5Sget_simple_extent_dims(filespace, dims3, maxdims3) < 0) { + fprintf(stderr, "H5Sget_simple_extent_dims failed\n"); + TEST_ERROR; + } + } else { + if (H5Sget_simple_extent_dims(filespace, dims2, maxdims2) < 0) { + fprintf(stderr, "H5Sget_simple_extent_dims failed\n"); + TEST_ERROR; + } + } + if (H5Sclose(filespace) < 0) { fprintf(stderr, "H5Sclose failed\n"); TEST_ERROR; @@ -1163,19 +1252,27 @@ open_extensible_dset(state_t *s) TEST_ERROR; } - if (s->two_dee) { - if (maxdims[0] != two_dee_max_dims[0] || maxdims[1] != two_dee_max_dims[1] || - maxdims[0] != maxdims[1]) { - fprintf(stderr, "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE, maxdims[0], - maxdims[1]); + if (s->test_3d) { + if (maxdims3[0] != three_dee_max_dims[0] || maxdims3[1] != three_dee_max_dims[1] || + maxdims3[2] != three_dee_max_dims[2]) { + fprintf(stderr, "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE " x %" PRIuHSIZE, + maxdims3[0], maxdims3[1], maxdims3[2]); TEST_ERROR; } - } - else if (maxdims[0] != s->one_dee_max_dims[0] || maxdims[1] != s->one_dee_max_dims[1] || - dims[0] != s->chunk_dims[0]) { - fprintf(stderr, - "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE " or columns %" PRIuHSIZE, - maxdims[0], maxdims[1], dims[1]); + } else { + if (s->expand_2d) { + if (maxdims2[0] != two_dee_max_dims[0] || maxdims2[1] != two_dee_max_dims[1] || + maxdims2[0] != maxdims2[1]) { + fprintf(stderr, "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE, maxdims2[0], + maxdims2[1]); + TEST_ERROR; + } + } else if (maxdims2[0] != s->one_dee_max_dims[0] || maxdims2[1] != s->one_dee_max_dims[1] || + dims2[0] != s->chunk_dims[0]) { + fprintf(stderr, + "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE " or columns %" PRIuHSIZE, + maxdims2[0], maxdims2[1], dims2[1]); + } } s->dataset[which] = dset_id; @@ -1184,14 +1281,51 @@ open_extensible_dset(state_t *s) return true; error: - H5E_BEGIN_TRY - { + H5E_BEGIN_TRY { H5Dclose(dset_id); H5Tclose(dtype); H5Sclose(filespace); + } H5E_END_TRY; + + return false; +} + +static bool +create_dsets(state_t s) +{ + struct timespec start_time, end_time; + unsigned int which; + + /* For checking the time spent in dataset creation. It's for running the writer alone */ + if (s.do_perf) { + if (HDclock_gettime(CLOCK_MONOTONIC, &start_time) == -1) { + fprintf(stderr, "HDclock_gettime failed"); + TEST_ERROR; + } + } + + /* Create NDATASETS datasets as the reader is doing verification. So no communication with + * the reader during the creation of datasets. + */ + for (which = 0; which < s.ndatasets; which++) + if (!create_extensible_dset(&s, which)) { + fprintf(stderr, "create_extensible_dset failed: number %u\n", which); + TEST_ERROR; + } + + /* For checking the time spent in dataset creation. It's for running the writer alone */ + if (s.do_perf) { + if (HDclock_gettime(CLOCK_MONOTONIC, &end_time) == -1) { + fprintf(stderr, "HDclock_gettime failed"); + TEST_ERROR; + } + + fprintf(stdout, "Dataset creation time (for running the writer alone) = %lf\n", TIME_PASSED(start_time, end_time)); } - H5E_END_TRY; + return true; + +error: return false; } @@ -1244,8 +1378,7 @@ set_or_verify_matrix(mat_t *mat, unsigned int which, base_t base, bool do_set) ret = false; break; } - } - else if (matget(mat, row, col) != v) { + } else if (matget(mat, row, col) != v) { /* If the data doesn't match, simply return false and * let the caller repeat this step */ @@ -1273,7 +1406,9 @@ verify_matrix(mat_t *mat, unsigned int which, base_t base) static bool verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t base) { - hsize_t offset[RANK] = {base.row, base.col}; + hsize_t offset2[RANK2] = {base.row, base.col}; + hsize_t offset3[RANK3] = {base.depth, base.row, base.col}; + hsize_t count3[RANK3] = {1, s->chunk_dims[0], s->chunk_dims[1]}; herr_t status; hid_t dset_id; @@ -1282,23 +1417,31 @@ verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t bas TEST_ERROR; } - dbgf(1, "verifying chunk %" PRIuHSIZE ", %" PRIuHSIZE "\n", base.row, base.col); + if (s->test_3d) + dbgf(1, "verifying chunk %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE "\n", base.row, base.col, base.depth); + else + dbgf(1, "verifying chunk %" PRIuHSIZE ", %" PRIuHSIZE "\n", base.row, base.col); dset_id = s->dataset[which]; - if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, s->chunk_dims, NULL) < 0) { - fprintf(stderr, "H5Sselect_hyperslab failed\n"); - TEST_ERROR; + if (s->test_3d) { + if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset3, NULL, count3, NULL) < 0) { + fprintf(stderr, "H5Sselect_hyperslab failed\n"); + TEST_ERROR; + } + } else { + if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset2, NULL, s->chunk_dims, NULL) < 0) { + fprintf(stderr, "H5Sselect_hyperslab failed\n"); + TEST_ERROR; + } } - /* A failure to read the data may indicate the data isn't ready yet. Instead of displaying the error - * stack, simply return false and let the caller repeat this step. + /* A failure to read the data may indicate the data isn't ready yet. Instead of displaying the error stack, + * simply return false and let the caller repeat this step. */ - H5E_BEGIN_TRY - { + H5E_BEGIN_TRY { status = H5Dread(dset_id, H5T_NATIVE_UINT32, s->memspace, filespace, H5P_DEFAULT, mat->elt); - } - H5E_END_TRY; + } H5E_END_TRY; if (status < 0) TEST_ERROR; @@ -1309,6 +1452,7 @@ error: return false; } +/* Try to verify a chunk NUM_ATTEMPTS times until the data is correct */ static bool repeat_verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t base) { @@ -1344,7 +1488,9 @@ error: static bool init_and_write_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t base) { - hsize_t offset[RANK] = {base.row, base.col}; + hsize_t offset2[RANK2] = {base.row, base.col}; + hsize_t offset3[RANK3] = {base.depth, base.row, base.col}; + hsize_t count3[RANK3] = {1, s->chunk_dims[0], s->chunk_dims[1]}; hid_t dset_id; dset_id = s->dataset[which]; @@ -1354,9 +1500,17 @@ init_and_write_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, ba TEST_ERROR; } - if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset, NULL, s->chunk_dims, NULL) < 0) { - fprintf(stderr, "H5Sselect_hyperslab failed\n"); - TEST_ERROR; + if (s->test_3d) { + /* The chunk dimensions are 1 x M x N. It grows along the first dimension */ + if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset3, NULL, count3, NULL) < 0) { + fprintf(stderr, "H5Sselect_hyperslab for 2D dataset failed\n"); + TEST_ERROR; + } + } else { + if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset2, NULL, s->chunk_dims, NULL) < 0) { + fprintf(stderr, "H5Sselect_hyperslab for 2D dataset failed\n"); + TEST_ERROR; + } } if (H5Dwrite(dset_id, H5T_NATIVE_UINT32, s->memspace, filespace, H5P_DEFAULT, mat->elt) < 0) { @@ -1404,11 +1558,9 @@ verify_dset_attribute(hid_t dset_id, unsigned int which, unsigned int step) return true; error: - H5E_BEGIN_TRY - { + H5E_BEGIN_TRY { H5Aclose(aid); - } - H5E_END_TRY; + } H5E_END_TRY; return false; } @@ -1417,9 +1569,9 @@ static bool verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned finished_step, unsigned last_step) { hid_t dset_id = H5I_INVALID_HID, filespace = H5I_INVALID_HID; - hsize_t size[RANK]; + hsize_t size2[RANK2], size3[RANK3]; base_t base, last; - unsigned int ncols, step; + unsigned int nchunks, step, ofs; int i; if (which >= s->ndatasets) { @@ -1441,15 +1593,24 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini TEST_ERROR; } - if (H5Sget_simple_extent_dims(filespace, size, NULL) < 0) { - fprintf(stderr, "H5Sget_simple_extent_dims failed\n"); - TEST_ERROR; - } + if (s->test_3d) { + if (H5Sget_simple_extent_dims(filespace, size3, NULL) < 0) { + fprintf(stderr, "H5Sget_simple_extent_dims failed\n"); + TEST_ERROR; + } - ncols = (unsigned)(size[1] / s->chunk_dims[1]); + nchunks = (unsigned)size3[0]; + } else { + if (H5Sget_simple_extent_dims(filespace, size2, NULL) < 0) { + fprintf(stderr, "H5Sget_simple_extent_dims failed\n"); + TEST_ERROR; + } + + nchunks = (unsigned)(size2[1] / s->chunk_dims[1]); + } /* Make sure the chunks show up on the reader side. Otherwise sleep a while and try again */ - if (ncols >= last_step) + if (nchunks >= last_step) break; else decisleep(1); @@ -1463,24 +1624,52 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini for (step = finished_step; step < last_step; step++) { dbgf(1, "%s: which %u step %u\n", __func__, which, step); - if (s->two_dee) { - size[0] = s->chunk_dims[0] * (1 + step); - size[1] = s->chunk_dims[1] * (1 + step); - last.row = s->chunk_dims[0] * step; - last.col = s->chunk_dims[1] * step; - } - else { - size[0] = s->chunk_dims[0]; - size[1] = s->chunk_dims[1] * (1 + step); + /* Read data that randomly crosses over chunks. But it should not happen to + * the last chunk being written + */ + if (s->cross_chunks) { + if (step == last_step - 1) + ofs = 0; + else + ofs = step % 2; + } else + ofs = 0; + + if (s->test_3d) { + size3[0] = 1 + step; + size3[1] = s->chunk_dims[0]; + size3[2] = s->chunk_dims[1]; + last.depth = step; last.row = 0; - last.col = s->chunk_dims[1] * step; + last.col = 0; + } else { + if (s->expand_2d) { + size2[0] = s->chunk_dims[0] * (1 + step); + size2[1] = s->chunk_dims[1] * (1 + step); + last.row = s->chunk_dims[0] * step + ofs; + last.col = s->chunk_dims[1] * step + ofs; + } else { + size2[0] = s->chunk_dims[0]; + size2[1] = s->chunk_dims[1] * (1 + step); + last.row = 0; + last.col = s->chunk_dims[1] * step + ofs; + } } - dbgf(1, "new size %" PRIuHSIZE ", %" PRIuHSIZE "\n", size[0], size[1]); - dbgf(1, "last row %" PRIuHSIZE " col %" PRIuHSIZE "\n", last.row, last.col); - - if (s->two_dee) { + if (s->test_3d) { + dbgf(1, "new size3 %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE "\n", size3[0], size3[1], size3[2]); + dbgf(1, "last row %" PRIuHSIZE " col %" PRIuHSIZE " depth %" PRIuHSIZE "\n", last.row, last.col, last.depth); + } else { + dbgf(1, "new size2 %" PRIuHSIZE ", %" PRIuHSIZE "\n", size2[0], size2[1]); + dbgf(1, "last row %" PRIuHSIZE " col %" PRIuHSIZE "\n", last.row, last.col); + } + if (s->test_3d || !s->expand_2d) { + if (!repeat_verify_chunk(s, filespace, mat, which, last)) { + fprintf(stderr, "chunk verification failed\n"); + TEST_ERROR; + } + } else { /* Down the right side, intersecting the bottom row. */ base.col = last.col; for (base.row = 0; base.row <= last.row; base.row += s->chunk_dims[0]) { @@ -1501,12 +1690,6 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini } } } - else { - if (!repeat_verify_chunk(s, filespace, mat, which, last)) { - fprintf(stderr, "chunk verification failed\n"); - TEST_ERROR; - } - } if (s->asteps != 0 && step % s->asteps == 0) { if (!verify_dset_attribute(dset_id, which, step)) { @@ -1519,11 +1702,9 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini return true; error: - H5E_BEGIN_TRY - { + H5E_BEGIN_TRY { H5Sclose(filespace); - } - H5E_END_TRY; + } H5E_END_TRY; return false; } @@ -1531,18 +1712,15 @@ error: static bool verify_dsets(state_t s, np_state_t *np, mat_t *mat) { - unsigned * nextstep = NULL; unsigned finished_step = 0; unsigned which; + unsigned counter = 0; + double passed_time = 0.0, total_time = 0.0, min_time = 1000000.0, max_time = 0.0; exchange_info_t last; - - if (!(nextstep = HDcalloc(s.ndatasets, sizeof(*nextstep)))) { - fprintf(stderr, "memory allocation failed\n"); - TEST_ERROR; - } + struct timespec end_time; do { - /* Receive the notice of the writer finishing zoo creation, + /* Receive the notice of the writer finishing creation, * including the number of chunks finished and the timestamp */ if (s.use_named_pipe && HDread(np->fd_writer_to_reader, &last, sizeof(last)) < 0) { @@ -1551,8 +1729,6 @@ verify_dsets(state_t s, np_state_t *np, mat_t *mat) } for (which = 0; which < s.ndatasets; which++) { - dbgf(1, "step %d which %d\n", nextstep[which], which); - /* Verify the chunks starting from the finished one in last round * to the ones written in this round */ @@ -1572,17 +1748,36 @@ verify_dsets(state_t s, np_state_t *np, mat_t *mat) AT(); fprintf(stderr, "verify_extensible_dset took too long to finish\n"); } + + /* For checking the time lapse between the writer's finishing writing a batch of chunks + * within a tick and the reader's finishing verifying those chunks + */ + if (s.use_named_pipe && s.do_perf) { + if (HDclock_gettime(CLOCK_MONOTONIC, &end_time) == -1) { + fprintf(stderr, "HDclock_gettime failed"); + TEST_ERROR; + } + + counter++; + passed_time = TIME_PASSED(last.time, end_time); + + total_time += passed_time; + + if (passed_time > max_time) + max_time = passed_time; + + if (passed_time < min_time) + min_time = passed_time; + } } while (finished_step < s.nsteps); - if (nextstep) - HDfree(nextstep); + /* Print out the performance information */ + if (s.use_named_pipe && s.do_perf && counter) + fprintf(stdout, "Dataset verification: mean time = %lf, max time = %lf, min time = %lf\n", total_time / (double)counter, max_time, min_time); return true; error: - if (nextstep) - HDfree(nextstep); - return false; } @@ -1614,11 +1809,9 @@ add_dset_attribute(const state_t *s, hid_t ds, hid_t sid, unsigned int which, un return true; error: - H5E_BEGIN_TRY - { + H5E_BEGIN_TRY { H5Aclose(aid); - } - H5E_END_TRY; + } H5E_END_TRY; return false; } @@ -1627,7 +1820,7 @@ static bool write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t *mat) { hid_t dset_id = H5I_INVALID_HID, filespace = H5I_INVALID_HID; - hsize_t size[RANK]; + hsize_t size2[RANK2], size3[RANK3]; base_t base, last; char dname[sizeof("/dataset-9999999999")]; @@ -1649,23 +1842,35 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * } } - if (s->two_dee) { - size[0] = s->chunk_dims[0] * (1 + step); - size[1] = s->chunk_dims[1] * (1 + step); - last.row = s->chunk_dims[0] * step; - last.col = s->chunk_dims[1] * step; - } - else { - size[0] = s->chunk_dims[0]; - size[1] = s->chunk_dims[1] * (1 + step); + if (s->test_3d) { + size3[0] = 1 + step; + size3[1] = s->chunk_dims[0]; + size3[2] = s->chunk_dims[1]; + last.depth = step; last.row = 0; - last.col = s->chunk_dims[1] * step; + last.col = 0; + } else { + if (s->expand_2d) { + size2[0] = s->chunk_dims[0] * (1 + step); + size2[1] = s->chunk_dims[1] * (1 + step); + last.row = s->chunk_dims[0] * step; + last.col = s->chunk_dims[1] * step; + } else { + size2[0] = s->chunk_dims[0]; + size2[1] = s->chunk_dims[1] * (1 + step); + last.row = 0; + last.col = s->chunk_dims[1] * step; + } + last.depth = 0; } - dbgf(1, "new size %" PRIuHSIZE ", %" PRIuHSIZE "\n", size[0], size[1]); + if (s->test_3d) + dbgf(1, "new size %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE "\n", size3[0], size3[1], size3[2]); + else + dbgf(1, "new size %" PRIuHSIZE ", %" PRIuHSIZE "\n", size2[0], size2[1]); if (s->vds != vds_off) { - const hsize_t half_size[RANK] = {size[0] / 2, size[1] / 2}; + const hsize_t half_size[RANK2] = {size2[0] / 2, size2[1] / 2}; sources_t *const srcs = &s->sources[which]; if (H5Dset_extent(srcs->ul, half_size) < 0) { @@ -1687,10 +1892,18 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * fprintf(stderr, "H5Dset_extent failed\n"); TEST_ERROR; } - } - else if (H5Dset_extent(dset_id, size) < 0) { - fprintf(stderr, "H5Dset_extent failed\n"); - TEST_ERROR; + } else { + if (s->test_3d) { + if (H5Dset_extent(dset_id, size3) < 0) { + fprintf(stderr, "H5Dset_extent for 3D dataset failed\n"); + TEST_ERROR; + } + } else { + if (H5Dset_extent(dset_id, size2) < 0) { + fprintf(stderr, "H5Dset_extent for 2D dataset failed\n"); + TEST_ERROR; + } + } } if ((filespace = H5Dget_space(dset_id)) < 0) { @@ -1698,8 +1911,14 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * TEST_ERROR; } - if (s->two_dee) { + if (s->test_3d || !s->expand_2d) { + if (!init_and_write_chunk(s, filespace, mat, which, last)) { + fprintf(stderr, "init_and_write_chunk failed\n"); + TEST_ERROR; + } + } else if (s->expand_2d) { base.col = last.col; + base.depth = 0; for (base.row = 0; base.row <= last.row; base.row += s->chunk_dims[0]) { dbgf(1, "writing chunk %" PRIuHSIZE ", %" PRIuHSIZE "\n", base.row, base.col); if (!init_and_write_chunk(s, filespace, mat, which, base)) { @@ -1717,12 +1936,6 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * } } } - else { - if (!init_and_write_chunk(s, filespace, mat, which, last)) { - fprintf(stderr, "init_and_write_chunk failed\n"); - TEST_ERROR; - } - } if (H5Sclose(filespace) < 0) { fprintf(stderr, "H5Sclose failed\n"); @@ -1732,11 +1945,9 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * return true; error: - H5E_BEGIN_TRY - { + H5E_BEGIN_TRY { H5Sclose(filespace); - } - H5E_END_TRY; + } H5E_END_TRY; return false; } @@ -1744,15 +1955,24 @@ error: static bool write_dsets(state_t s, np_state_t *np, mat_t *mat) { - unsigned last_step, step, which; + unsigned last_step, step, which; unsigned long long old_tick_num; - H5F_t * f = NULL; + H5F_t *f = NULL; + struct timespec start_time, end_time; if (NULL == (f = (H5F_t *)H5VL_object(s.file[0]))) { fprintf(stderr, "H5VL_object failed\n"); TEST_ERROR; } + /* For checking the time spent in writing data. It's for running the writer alone */ + if (s.do_perf) { + if (HDclock_gettime(CLOCK_MONOTONIC, &start_time) == -1) { + fprintf(stderr, "HDclock_gettime failed"); + TEST_ERROR; + } + } + old_tick_num = f->shared->tick_num; /* Write as many as chunks within the same tick number before notifying @@ -1772,13 +1992,13 @@ write_dsets(state_t s, np_state_t *np, mat_t *mat) /* After finishing writing all the chunks, end the tick */ if (s.use_vfd_swmr && step == (s.nsteps - 1)) { - unsigned long i; + unsigned long i; - if (s.vds != vds_multi) - H5Fvfd_swmr_end_tick(s.file[0]); - else - for (i = 0; i < NELMTS(s.file); i++) - H5Fvfd_swmr_end_tick(s.file[i]); + if (s.vds != vds_multi) + H5Fvfd_swmr_end_tick(s.file[0]); + else + for (i = 0; i < NELMTS(s.file); i++) + H5Fvfd_swmr_end_tick(s.file[i]); } /* Notify the reader to start verification by @@ -1795,6 +2015,16 @@ write_dsets(state_t s, np_state_t *np, mat_t *mat) } } + /* For checking the time spent in writing data. It's for running the writer alone */ + if (s.do_perf) { + if (HDclock_gettime(CLOCK_MONOTONIC, &end_time) == -1) { + fprintf(stderr, "HDclock_gettime failed"); + TEST_ERROR; + } + + fprintf(stdout, "Dataset write time (for running the writer alone) = %lf\n", TIME_PASSED(start_time, end_time)); + } + return true; error: @@ -1804,7 +2034,7 @@ error: int main(int argc, char **argv) { - mat_t * mat; + mat_t *mat; hid_t fcpl = H5I_INVALID_HID; unsigned which; state_t s; @@ -1877,32 +2107,28 @@ main(int argc, char **argv) TEST_ERROR; } - /* Create NDATASETS datasets as the reader is doing verification. So no communication with - * the reader during the creation of datasets. - */ - for (which = 0; which < s.ndatasets; which++) - if (!create_extensible_dset(&s, which)) { - fprintf(stderr, "create_extensible_dset failed: number %u\n", which); - TEST_ERROR; - } + /* Creates multiple datasets */ + if (!create_dsets(s)) { + fprintf(stderr, "create_dsets failed"); + TEST_ERROR; + } /* Call H5Fvfd_swmr_end_tick to end the tick. No communication with the reader in this step */ if (s.use_vfd_swmr && s.use_named_pipe) { - unsigned long j; - - if (s.vds != vds_multi) { - if (H5Fvfd_swmr_end_tick(s.file[0]) < 0) { - fprintf(stderr, "H5Fvfd_swmr_end_tick failed\n"); - TEST_ERROR; - } - } - else { - for (j = 0; j < NELMTS(s.file); j++) - if (H5Fvfd_swmr_end_tick(s.file[j]) < 0) { - fprintf(stderr, "H5Fvfd_swmr_end_tick failed\n"); - TEST_ERROR; - } - } + unsigned long j; + + if (s.vds != vds_multi) { + if (H5Fvfd_swmr_end_tick(s.file[0]) < 0) { + fprintf(stderr, "H5Fvfd_swmr_end_tick failed\n"); + TEST_ERROR; + } + } else { + for (j = 0; j < NELMTS(s.file); j++) + if (H5Fvfd_swmr_end_tick(s.file[j]) < 0) { + fprintf(stderr, "H5Fvfd_swmr_end_tick failed\n"); + TEST_ERROR; + } + } } /* Notify the reader of finishing dataset creation by sending the timestamp @@ -1919,8 +2145,7 @@ main(int argc, char **argv) fprintf(stderr, "write_dsets failed"); TEST_ERROR; } - } - else { + } else { /* Wait for the writer's notice before starting the validation of dataset creation */ np.verify = 1; if (s.use_named_pipe && reader_verify(np, np.verify) < 0) { @@ -1981,14 +2206,12 @@ main(int argc, char **argv) return EXIT_SUCCESS; error: - H5E_BEGIN_TRY - { + H5E_BEGIN_TRY { H5Pclose(fcpl); for (i = 0; i < NELMTS(s.file); i++) H5Fclose(s.file[i]); - } - H5E_END_TRY; + } H5E_END_TRY; if (s.use_named_pipe && np.fd_writer_to_reader >= 0) HDclose(np.fd_writer_to_reader); -- cgit v0.12 From 082de1cb75edd1e63269c9c6e1953a2e418d770e Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 8 Jul 2021 16:20:33 +0000 Subject: Committing clang-format changes --- test/vfd_swmr_bigset_writer.c | 396 ++++++++++++++++++++++++------------------ 1 file changed, 230 insertions(+), 166 deletions(-) diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c index 48981d5..2f86d64 100644 --- a/test/vfd_swmr_bigset_writer.c +++ b/test/vfd_swmr_bigset_writer.c @@ -95,7 +95,7 @@ * X is the beginning time; Y is the ending time. * Expects X, Y to be struct timespec from the function call HDclock_gettime. */ -#define TIME_PASSED(X,Y) \ +#define TIME_PASSED(X, Y) \ ((double)((Y.tv_sec - X.tv_sec) * 1000000000 + (Y.tv_nsec - X.tv_nsec))) / 1000000000.0 typedef struct _base { @@ -122,13 +122,13 @@ typedef struct _sources { #define MANY_FILES 4 typedef struct { - hid_t *dataset; - sources_t *sources; - hid_t file[MANY_FILES]; - hid_t dapl, filetype, memspace, one_by_one_sid, quadrant_dcpl; - unsigned ndatasets; + hid_t * dataset; + sources_t * sources; + hid_t file[MANY_FILES]; + hid_t dapl, filetype, memspace, one_by_one_sid, quadrant_dcpl; + unsigned ndatasets; const char *filename[MANY_FILES]; - char progname[PATH_MAX]; + char progname[PATH_MAX]; struct { quadrant_t ul, ur, bl, br, src; } quadrants; @@ -138,25 +138,25 @@ typedef struct { bool expand_2d; bool test_3d; enum { vds_off, vds_single, vds_multi } vds; - bool use_vfd_swmr; - bool use_named_pipe; - bool do_perf; - bool cross_chunks; - bool writer; - bool fixed_array; - hsize_t chunk_dims[RANK2]; - hsize_t one_dee_max_dims[RANK2]; + bool use_vfd_swmr; + bool use_named_pipe; + bool do_perf; + bool cross_chunks; + bool writer; + bool fixed_array; + hsize_t chunk_dims[RANK2]; + hsize_t one_dee_max_dims[RANK2]; struct timespec ival; } state_t; /* Structure to hold info for named pipes */ typedef struct { - const char *fifo_writer_to_reader; /* Name of fifo for writer to reader */ - const char *fifo_reader_to_writer; /* Name of fifo for reader to writer */ - int fd_writer_to_reader; /* File ID for fifo from writer to reader */ - int fd_reader_to_writer; /* File ID for fifo from reader to writer */ - int notify; /* Value to notify between writer and reader */ - int verify; /* Value to verify between writer and reader */ + const char *fifo_writer_to_reader; /* Name of fifo for writer to reader */ + const char *fifo_reader_to_writer; /* Name of fifo for reader to writer */ + int fd_writer_to_reader; /* File ID for fifo from writer to reader */ + int fd_reader_to_writer; /* File ID for fifo from reader to writer */ + int notify; /* Value to notify between writer and reader */ + int verify; /* Value to verify between writer and reader */ } np_state_t; typedef struct { @@ -165,14 +165,13 @@ typedef struct { } exchange_info_t; /* Initializations for np_state_t */ -#define NP_INITIALIZER (np_state_t) { \ - .fifo_writer_to_reader = "./fifo_bigset_writer_to_reader", \ - .fifo_reader_to_writer = "./fifo_bigset_reader_to_writer", \ - .fd_writer_to_reader = -1, \ - .fd_reader_to_writer = -1, \ - .notify = 0, \ - .verify = 0 \ -} +#define NP_INITIALIZER \ + (np_state_t) \ + { \ + .fifo_writer_to_reader = "./fifo_bigset_writer_to_reader", \ + .fifo_reader_to_writer = "./fifo_bigset_reader_to_writer", .fd_writer_to_reader = -1, \ + .fd_reader_to_writer = -1, .notify = 0, .verify = 0 \ + } static inline state_t state_initializer(void) @@ -291,7 +290,7 @@ static bool make_quadrant_dataspace(state_t *s, quadrant_t *q) { if ((q->space = H5Screate_simple(NELMTS(s->chunk_dims), s->chunk_dims, - s->expand_2d ? two_dee_max_dims : s->one_dee_max_dims)) < 0) { + s->expand_2d ? two_dee_max_dims : s->one_dee_max_dims)) < 0) { fprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } @@ -304,9 +303,11 @@ make_quadrant_dataspace(state_t *s, quadrant_t *q) return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Sclose(q->space); - } H5E_END_TRY; + } + H5E_END_TRY; return false; } @@ -317,8 +318,8 @@ state_init(state_t *s, int argc, char **argv) unsigned long tmp; int ch; unsigned i; - const hsize_t dims = 1; - char *tfile = NULL; + const hsize_t dims = 1; + char * tfile = NULL; char * end; quadrant_t *const ul = &s->quadrants.ul, *const ur = &s->quadrants.ur, *const bl = &s->quadrants.bl, *const br = &s->quadrants.br, *const src = &s->quadrants.src; @@ -406,12 +407,13 @@ state_init(state_t *s, int argc, char **argv) else if (ch == 'l') { /* Translate the tick number to time represented by the timespec struct */ float time = (float)(((unsigned)tmp * TICK_LEN) / 10.0); - unsigned sec = (unsigned)time; + unsigned sec = (unsigned)time; unsigned nsec = (unsigned)((time - sec) * 10 * 1000 * 1000); - s->ival.tv_sec = sec; + s->ival.tv_sec = sec; s->ival.tv_nsec = nsec; - } else if (ch == 'n') + } + else if (ch == 'n') s->nsteps = (unsigned)tmp; else if (ch == 'r') s->rows = (unsigned)tmp; @@ -463,8 +465,8 @@ state_init(state_t *s, int argc, char **argv) } } - s->chunk_dims[0] = s->rows; - s->chunk_dims[1] = s->cols; + s->chunk_dims[0] = s->rows; + s->chunk_dims[1] = s->cols; s->one_dee_max_dims[0] = s->rows; if (s->fixed_array) { @@ -472,16 +474,17 @@ state_init(state_t *s, int argc, char **argv) two_dee_max_dims[0] = s->rows * s->nsteps; two_dee_max_dims[1] = s->cols * s->nsteps; - if(s->test_3d) { + if (s->test_3d) { three_dee_max_dims[0] = s->nsteps; three_dee_max_dims[1] = s->rows; three_dee_max_dims[2] = s->cols; } - } else { + } + else { s->one_dee_max_dims[1] = H5S_UNLIMITED; two_dee_max_dims[0] = two_dee_max_dims[1] = H5S_UNLIMITED; - if(s->test_3d) { + if (s->test_3d) { three_dee_max_dims[0] = H5S_UNLIMITED; three_dee_max_dims[1] = s->rows; three_dee_max_dims[2] = s->cols; @@ -561,7 +564,8 @@ state_init(state_t *s, int argc, char **argv) TEST_ERROR; } - if (H5Sselect_hyperslab(src->space, H5S_SELECT_SET, src->start, src->stride, src->count, src->block) < 0) { + if (H5Sselect_hyperslab(src->space, H5S_SELECT_SET, src->start, src->stride, src->count, src->block) < + 0) { fprintf(stderr, "H5Sselect_hyperslab failed\n"); TEST_ERROR; } @@ -609,13 +613,14 @@ state_init(state_t *s, int argc, char **argv) } if (s->test_3d) { - hsize_t dims3[RANK3] = {1, s->chunk_dims[0], s->chunk_dims[1]}; + hsize_t dims3[RANK3] = {1, s->chunk_dims[0], s->chunk_dims[1]}; - if ((s->memspace = H5Screate_simple(RANK3, dims3, NULL)) < 0) { + if ((s->memspace = H5Screate_simple(RANK3, dims3, NULL)) < 0) { fprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; - } - } else { + } + } + else { if ((s->memspace = H5Screate_simple(RANK2, s->chunk_dims, NULL)) < 0) { fprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; @@ -663,7 +668,8 @@ state_init(state_t *s, int argc, char **argv) return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Pclose(s->quadrant_dcpl); H5Sclose(ul->space); H5Sclose(ur->space); @@ -676,7 +682,8 @@ error: H5Sclose(src->space); H5Sclose(s->one_by_one_sid); H5Sclose(s->memspace); - } H5E_END_TRY; + } + H5E_END_TRY; HDfree(tfile); @@ -690,7 +697,7 @@ error: static bool state_destroy(state_t *s) { - size_t i; + size_t i; struct timespec start_time, end_time; if (H5Pclose(s->dapl) < 0) { @@ -759,7 +766,8 @@ state_destroy(state_t *s) TEST_ERROR; } - fprintf(stdout, "File close time (for running the writer alone) = %lf\n", TIME_PASSED(start_time, end_time)); + fprintf(stdout, "File close time (for running the writer alone) = %lf\n", + TIME_PASSED(start_time, end_time)); } HDfree(s->dataset); @@ -769,11 +777,13 @@ state_destroy(state_t *s) return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Pclose(s->quadrant_dcpl); H5Sclose(s->one_by_one_sid); H5Sclose(s->memspace); - } H5E_END_TRY; + } + H5E_END_TRY; HDfree(s->dataset); @@ -861,13 +871,13 @@ np_close(np_state_t *np, bool writer) } /* Reader finishes last and deletes the named pipes */ - if(!writer) { - if(HDremove(np->fifo_writer_to_reader) != 0) { + if (!writer) { + if (HDremove(np->fifo_writer_to_reader) != 0) { fprintf(stderr, "HDremove fifo_writer_to_reader failed\n"); TEST_ERROR; } - if(HDremove(np->fifo_reader_to_writer) != 0) { + if (HDremove(np->fifo_reader_to_writer) != 0) { fprintf(stderr, "HDremove fifo_reader_to_writer failed\n"); TEST_ERROR; } @@ -926,9 +936,11 @@ notify_and_wait_for_reader(state_t *s, np_state_t *np) for (i = 0; i < MAX_LAG + 1; i++) { decisleep(TICK_LEN); - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Aexists(s->file[0], "nonexistent"); - } H5E_END_TRY; + } + H5E_END_TRY; } /* Wait until the reader finishes validating creation */ @@ -1017,7 +1029,7 @@ create_extensible_dset(state_t *s, unsigned int which) char dname[sizeof("/dataset-9999999999")]; char ul_dname[sizeof("/ul-dataset-9999999999")], ur_dname[sizeof("/ur-dataset-9999999999")], bl_dname[sizeof("/bl-dataset-9999999999")], br_dname[sizeof("/br-dataset-9999999999")]; - hid_t dcpl = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, filespace = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, filespace = H5I_INVALID_HID; hsize_t dims3[3] = {1, s->chunk_dims[0], s->chunk_dims[1]}; esnprintf(dname, sizeof(dname), "/dataset-%d", which); @@ -1033,7 +1045,8 @@ create_extensible_dset(state_t *s, unsigned int which) fprintf(stderr, "H5Pset_chunk for 3D dataset failed\n"); TEST_ERROR; } - } else { + } + else { if (H5Pset_chunk(dcpl, RANK2, s->chunk_dims) < 0) { fprintf(stderr, "H5Pset_chunk for 2D dataset failed\n"); TEST_ERROR; @@ -1048,22 +1061,26 @@ create_extensible_dset(state_t *s, unsigned int which) esnprintf(bl_dname, sizeof(bl_dname), "/bl-dataset-%d", which); esnprintf(br_dname, sizeof(br_dname), "/br-dataset-%d", which); - if ((srcs->ul = H5Dcreate2(s->file[0], ul_dname, s->filetype, ul->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { + if ((srcs->ul = H5Dcreate2(s->file[0], ul_dname, s->filetype, ul->src_space, H5P_DEFAULT, + s->quadrant_dcpl, s->dapl)) < 0) { fprintf(stderr, "H5Dcreate2 failed\n"); TEST_ERROR; } - if ((srcs->ur = H5Dcreate2(s->file[1], ur_dname, s->filetype, ur->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { + if ((srcs->ur = H5Dcreate2(s->file[1], ur_dname, s->filetype, ur->src_space, H5P_DEFAULT, + s->quadrant_dcpl, s->dapl)) < 0) { fprintf(stderr, "H5Dcreate2 failed\n"); TEST_ERROR; } - if ((srcs->bl = H5Dcreate2(s->file[2], bl_dname, s->filetype, bl->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { + if ((srcs->bl = H5Dcreate2(s->file[2], bl_dname, s->filetype, bl->src_space, H5P_DEFAULT, + s->quadrant_dcpl, s->dapl)) < 0) { fprintf(stderr, "H5Dcreate2 failed\n"); TEST_ERROR; } - if ((srcs->br = H5Dcreate2(s->file[3], br_dname, s->filetype, br->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { + if ((srcs->br = H5Dcreate2(s->file[3], br_dname, s->filetype, br->src_space, H5P_DEFAULT, + s->quadrant_dcpl, s->dapl)) < 0) { fprintf(stderr, "H5Dcreate2 failed\n"); TEST_ERROR; } @@ -1094,9 +1111,10 @@ create_extensible_dset(state_t *s, unsigned int which) fprintf(stderr, "H5Screate_simple 3D dataspace failed\n"); TEST_ERROR; } - } else { + } + else { if ((filespace = H5Screate_simple(RANK2, s->chunk_dims, - s->expand_2d ? two_dee_max_dims : s->one_dee_max_dims)) < 0) { + s->expand_2d ? two_dee_max_dims : s->one_dee_max_dims)) < 0) { fprintf(stderr, "H5Screate_simple 2D dataspace failed\n"); TEST_ERROR; } @@ -1122,11 +1140,13 @@ create_extensible_dset(state_t *s, unsigned int which) return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Dclose(dset_id); H5Pclose(dcpl); H5Sclose(filespace); - } H5E_END_TRY; + } + H5E_END_TRY; return false; } @@ -1166,9 +1186,11 @@ close_extensible_dset(state_t *s, unsigned int which) return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Dclose(dset_id); - } H5E_END_TRY; + } + H5E_END_TRY; return false; } @@ -1190,9 +1212,11 @@ open_extensible_dset(state_t *s) * NUM_ATTEMPTS times without success, report it as a failure */ for (i = 0; i < NUM_ATTEMPTS; i++) { - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { dset_id = H5Dopen2(s->file[0], dname, s->dapl); - } H5E_END_TRY; + } + H5E_END_TRY; if (dset_id >= 0) break; @@ -1235,7 +1259,8 @@ open_extensible_dset(state_t *s) fprintf(stderr, "H5Sget_simple_extent_dims failed\n"); TEST_ERROR; } - } else { + } + else { if (H5Sget_simple_extent_dims(filespace, dims2, maxdims2) < 0) { fprintf(stderr, "H5Sget_simple_extent_dims failed\n"); TEST_ERROR; @@ -1256,22 +1281,25 @@ open_extensible_dset(state_t *s) if (maxdims3[0] != three_dee_max_dims[0] || maxdims3[1] != three_dee_max_dims[1] || maxdims3[2] != three_dee_max_dims[2]) { fprintf(stderr, "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE " x %" PRIuHSIZE, - maxdims3[0], maxdims3[1], maxdims3[2]); + maxdims3[0], maxdims3[1], maxdims3[2]); TEST_ERROR; } - } else { + } + else { if (s->expand_2d) { if (maxdims2[0] != two_dee_max_dims[0] || maxdims2[1] != two_dee_max_dims[1] || maxdims2[0] != maxdims2[1]) { fprintf(stderr, "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE, maxdims2[0], - maxdims2[1]); + maxdims2[1]); TEST_ERROR; } - } else if (maxdims2[0] != s->one_dee_max_dims[0] || maxdims2[1] != s->one_dee_max_dims[1] || - dims2[0] != s->chunk_dims[0]) { + } + else if (maxdims2[0] != s->one_dee_max_dims[0] || maxdims2[1] != s->one_dee_max_dims[1] || + dims2[0] != s->chunk_dims[0]) { fprintf(stderr, - "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE " or columns %" PRIuHSIZE, - maxdims2[0], maxdims2[1], dims2[1]); + "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE + " or columns %" PRIuHSIZE, + maxdims2[0], maxdims2[1], dims2[1]); } } @@ -1281,11 +1309,13 @@ open_extensible_dset(state_t *s) return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Dclose(dset_id); H5Tclose(dtype); H5Sclose(filespace); - } H5E_END_TRY; + } + H5E_END_TRY; return false; } @@ -1294,7 +1324,7 @@ static bool create_dsets(state_t s) { struct timespec start_time, end_time; - unsigned int which; + unsigned int which; /* For checking the time spent in dataset creation. It's for running the writer alone */ if (s.do_perf) { @@ -1320,7 +1350,8 @@ create_dsets(state_t s) TEST_ERROR; } - fprintf(stdout, "Dataset creation time (for running the writer alone) = %lf\n", TIME_PASSED(start_time, end_time)); + fprintf(stdout, "Dataset creation time (for running the writer alone) = %lf\n", + TIME_PASSED(start_time, end_time)); } return true; @@ -1378,7 +1409,8 @@ set_or_verify_matrix(mat_t *mat, unsigned int which, base_t base, bool do_set) ret = false; break; } - } else if (matget(mat, row, col) != v) { + } + else if (matget(mat, row, col) != v) { /* If the data doesn't match, simply return false and * let the caller repeat this step */ @@ -1408,7 +1440,7 @@ verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t bas { hsize_t offset2[RANK2] = {base.row, base.col}; hsize_t offset3[RANK3] = {base.depth, base.row, base.col}; - hsize_t count3[RANK3] = {1, s->chunk_dims[0], s->chunk_dims[1]}; + hsize_t count3[RANK3] = {1, s->chunk_dims[0], s->chunk_dims[1]}; herr_t status; hid_t dset_id; @@ -1418,7 +1450,8 @@ verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t bas } if (s->test_3d) - dbgf(1, "verifying chunk %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE "\n", base.row, base.col, base.depth); + dbgf(1, "verifying chunk %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE "\n", base.row, base.col, + base.depth); else dbgf(1, "verifying chunk %" PRIuHSIZE ", %" PRIuHSIZE "\n", base.row, base.col); @@ -1429,19 +1462,22 @@ verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t bas fprintf(stderr, "H5Sselect_hyperslab failed\n"); TEST_ERROR; } - } else { + } + else { if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset2, NULL, s->chunk_dims, NULL) < 0) { fprintf(stderr, "H5Sselect_hyperslab failed\n"); TEST_ERROR; } } - /* A failure to read the data may indicate the data isn't ready yet. Instead of displaying the error stack, - * simply return false and let the caller repeat this step. + /* A failure to read the data may indicate the data isn't ready yet. Instead of displaying the error + * stack, simply return false and let the caller repeat this step. */ - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { status = H5Dread(dset_id, H5T_NATIVE_UINT32, s->memspace, filespace, H5P_DEFAULT, mat->elt); - } H5E_END_TRY; + } + H5E_END_TRY; if (status < 0) TEST_ERROR; @@ -1490,7 +1526,7 @@ init_and_write_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, ba { hsize_t offset2[RANK2] = {base.row, base.col}; hsize_t offset3[RANK3] = {base.depth, base.row, base.col}; - hsize_t count3[RANK3] = {1, s->chunk_dims[0], s->chunk_dims[1]}; + hsize_t count3[RANK3] = {1, s->chunk_dims[0], s->chunk_dims[1]}; hid_t dset_id; dset_id = s->dataset[which]; @@ -1501,12 +1537,13 @@ init_and_write_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, ba } if (s->test_3d) { - /* The chunk dimensions are 1 x M x N. It grows along the first dimension */ + /* The chunk dimensions are 1 x M x N. It grows along the first dimension */ if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset3, NULL, count3, NULL) < 0) { fprintf(stderr, "H5Sselect_hyperslab for 2D dataset failed\n"); TEST_ERROR; } - } else { + } + else { if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset2, NULL, s->chunk_dims, NULL) < 0) { fprintf(stderr, "H5Sselect_hyperslab for 2D dataset failed\n"); TEST_ERROR; @@ -1558,9 +1595,11 @@ verify_dset_attribute(hid_t dset_id, unsigned int which, unsigned int step) return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Aclose(aid); - } H5E_END_TRY; + } + H5E_END_TRY; return false; } @@ -1600,7 +1639,8 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini } nchunks = (unsigned)size3[0]; - } else { + } + else { if (H5Sget_simple_extent_dims(filespace, size2, NULL) < 0) { fprintf(stderr, "H5Sget_simple_extent_dims failed\n"); TEST_ERROR; @@ -1632,34 +1672,40 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini ofs = 0; else ofs = step % 2; - } else + } + else ofs = 0; if (s->test_3d) { - size3[0] = 1 + step; - size3[1] = s->chunk_dims[0]; - size3[2] = s->chunk_dims[1]; + size3[0] = 1 + step; + size3[1] = s->chunk_dims[0]; + size3[2] = s->chunk_dims[1]; last.depth = step; - last.row = 0; - last.col = 0; - } else { + last.row = 0; + last.col = 0; + } + else { if (s->expand_2d) { - size2[0] = s->chunk_dims[0] * (1 + step); - size2[1] = s->chunk_dims[1] * (1 + step); + size2[0] = s->chunk_dims[0] * (1 + step); + size2[1] = s->chunk_dims[1] * (1 + step); last.row = s->chunk_dims[0] * step + ofs; last.col = s->chunk_dims[1] * step + ofs; - } else { - size2[0] = s->chunk_dims[0]; - size2[1] = s->chunk_dims[1] * (1 + step); + } + else { + size2[0] = s->chunk_dims[0]; + size2[1] = s->chunk_dims[1] * (1 + step); last.row = 0; last.col = s->chunk_dims[1] * step + ofs; } } if (s->test_3d) { - dbgf(1, "new size3 %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE "\n", size3[0], size3[1], size3[2]); - dbgf(1, "last row %" PRIuHSIZE " col %" PRIuHSIZE " depth %" PRIuHSIZE "\n", last.row, last.col, last.depth); - } else { + dbgf(1, "new size3 %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE "\n", size3[0], size3[1], + size3[2]); + dbgf(1, "last row %" PRIuHSIZE " col %" PRIuHSIZE " depth %" PRIuHSIZE "\n", last.row, last.col, + last.depth); + } + else { dbgf(1, "new size2 %" PRIuHSIZE ", %" PRIuHSIZE "\n", size2[0], size2[1]); dbgf(1, "last row %" PRIuHSIZE " col %" PRIuHSIZE "\n", last.row, last.col); } @@ -1669,7 +1715,8 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini fprintf(stderr, "chunk verification failed\n"); TEST_ERROR; } - } else { + } + else { /* Down the right side, intersecting the bottom row. */ base.col = last.col; for (base.row = 0; base.row <= last.row; base.row += s->chunk_dims[0]) { @@ -1702,9 +1749,11 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Sclose(filespace); - } H5E_END_TRY; + } + H5E_END_TRY; return false; } @@ -1714,7 +1763,7 @@ verify_dsets(state_t s, np_state_t *np, mat_t *mat) { unsigned finished_step = 0; unsigned which; - unsigned counter = 0; + unsigned counter = 0; double passed_time = 0.0, total_time = 0.0, min_time = 1000000.0, max_time = 0.0; exchange_info_t last; struct timespec end_time; @@ -1773,7 +1822,8 @@ verify_dsets(state_t s, np_state_t *np, mat_t *mat) /* Print out the performance information */ if (s.use_named_pipe && s.do_perf && counter) - fprintf(stdout, "Dataset verification: mean time = %lf, max time = %lf, min time = %lf\n", total_time / (double)counter, max_time, min_time); + fprintf(stdout, "Dataset verification: mean time = %lf, max time = %lf, min time = %lf\n", + total_time / (double)counter, max_time, min_time); return true; @@ -1809,9 +1859,11 @@ add_dset_attribute(const state_t *s, hid_t ds, hid_t sid, unsigned int which, un return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Aclose(aid); - } H5E_END_TRY; + } + H5E_END_TRY; return false; } @@ -1843,21 +1895,23 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * } if (s->test_3d) { - size3[0] = 1 + step; - size3[1] = s->chunk_dims[0]; - size3[2] = s->chunk_dims[1]; + size3[0] = 1 + step; + size3[1] = s->chunk_dims[0]; + size3[2] = s->chunk_dims[1]; last.depth = step; - last.row = 0; - last.col = 0; - } else { + last.row = 0; + last.col = 0; + } + else { if (s->expand_2d) { - size2[0] = s->chunk_dims[0] * (1 + step); - size2[1] = s->chunk_dims[1] * (1 + step); + size2[0] = s->chunk_dims[0] * (1 + step); + size2[1] = s->chunk_dims[1] * (1 + step); last.row = s->chunk_dims[0] * step; last.col = s->chunk_dims[1] * step; - } else { - size2[0] = s->chunk_dims[0]; - size2[1] = s->chunk_dims[1] * (1 + step); + } + else { + size2[0] = s->chunk_dims[0]; + size2[1] = s->chunk_dims[1] * (1 + step); last.row = 0; last.col = s->chunk_dims[1] * step; } @@ -1871,7 +1925,7 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * if (s->vds != vds_off) { const hsize_t half_size[RANK2] = {size2[0] / 2, size2[1] / 2}; - sources_t *const srcs = &s->sources[which]; + sources_t *const srcs = &s->sources[which]; if (H5Dset_extent(srcs->ul, half_size) < 0) { fprintf(stderr, "H5Dset_extent failed\n"); @@ -1892,13 +1946,15 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * fprintf(stderr, "H5Dset_extent failed\n"); TEST_ERROR; } - } else { + } + else { if (s->test_3d) { if (H5Dset_extent(dset_id, size3) < 0) { fprintf(stderr, "H5Dset_extent for 3D dataset failed\n"); TEST_ERROR; } - } else { + } + else { if (H5Dset_extent(dset_id, size2) < 0) { fprintf(stderr, "H5Dset_extent for 2D dataset failed\n"); TEST_ERROR; @@ -1916,8 +1972,9 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * fprintf(stderr, "init_and_write_chunk failed\n"); TEST_ERROR; } - } else if (s->expand_2d) { - base.col = last.col; + } + else if (s->expand_2d) { + base.col = last.col; base.depth = 0; for (base.row = 0; base.row <= last.row; base.row += s->chunk_dims[0]) { dbgf(1, "writing chunk %" PRIuHSIZE ", %" PRIuHSIZE "\n", base.row, base.col); @@ -1945,9 +2002,11 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * return true; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Sclose(filespace); - } H5E_END_TRY; + } + H5E_END_TRY; return false; } @@ -1955,10 +2014,10 @@ error: static bool write_dsets(state_t s, np_state_t *np, mat_t *mat) { - unsigned last_step, step, which; + unsigned last_step, step, which; unsigned long long old_tick_num; - H5F_t *f = NULL; - struct timespec start_time, end_time; + H5F_t * f = NULL; + struct timespec start_time, end_time; if (NULL == (f = (H5F_t *)H5VL_object(s.file[0]))) { fprintf(stderr, "H5VL_object failed\n"); @@ -1992,13 +2051,13 @@ write_dsets(state_t s, np_state_t *np, mat_t *mat) /* After finishing writing all the chunks, end the tick */ if (s.use_vfd_swmr && step == (s.nsteps - 1)) { - unsigned long i; + unsigned long i; - if (s.vds != vds_multi) - H5Fvfd_swmr_end_tick(s.file[0]); - else - for (i = 0; i < NELMTS(s.file); i++) - H5Fvfd_swmr_end_tick(s.file[i]); + if (s.vds != vds_multi) + H5Fvfd_swmr_end_tick(s.file[0]); + else + for (i = 0; i < NELMTS(s.file); i++) + H5Fvfd_swmr_end_tick(s.file[i]); } /* Notify the reader to start verification by @@ -2022,7 +2081,8 @@ write_dsets(state_t s, np_state_t *np, mat_t *mat) TEST_ERROR; } - fprintf(stdout, "Dataset write time (for running the writer alone) = %lf\n", TIME_PASSED(start_time, end_time)); + fprintf(stdout, "Dataset write time (for running the writer alone) = %lf\n", + TIME_PASSED(start_time, end_time)); } return true; @@ -2034,7 +2094,7 @@ error: int main(int argc, char **argv) { - mat_t *mat; + mat_t * mat; hid_t fcpl = H5I_INVALID_HID; unsigned which; state_t s; @@ -2115,20 +2175,21 @@ main(int argc, char **argv) /* Call H5Fvfd_swmr_end_tick to end the tick. No communication with the reader in this step */ if (s.use_vfd_swmr && s.use_named_pipe) { - unsigned long j; - - if (s.vds != vds_multi) { - if (H5Fvfd_swmr_end_tick(s.file[0]) < 0) { - fprintf(stderr, "H5Fvfd_swmr_end_tick failed\n"); - TEST_ERROR; - } - } else { - for (j = 0; j < NELMTS(s.file); j++) - if (H5Fvfd_swmr_end_tick(s.file[j]) < 0) { - fprintf(stderr, "H5Fvfd_swmr_end_tick failed\n"); - TEST_ERROR; - } - } + unsigned long j; + + if (s.vds != vds_multi) { + if (H5Fvfd_swmr_end_tick(s.file[0]) < 0) { + fprintf(stderr, "H5Fvfd_swmr_end_tick failed\n"); + TEST_ERROR; + } + } + else { + for (j = 0; j < NELMTS(s.file); j++) + if (H5Fvfd_swmr_end_tick(s.file[j]) < 0) { + fprintf(stderr, "H5Fvfd_swmr_end_tick failed\n"); + TEST_ERROR; + } + } } /* Notify the reader of finishing dataset creation by sending the timestamp @@ -2145,7 +2206,8 @@ main(int argc, char **argv) fprintf(stderr, "write_dsets failed"); TEST_ERROR; } - } else { + } + else { /* Wait for the writer's notice before starting the validation of dataset creation */ np.verify = 1; if (s.use_named_pipe && reader_verify(np, np.verify) < 0) { @@ -2206,12 +2268,14 @@ main(int argc, char **argv) return EXIT_SUCCESS; error: - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { H5Pclose(fcpl); for (i = 0; i < NELMTS(s.file); i++) H5Fclose(s.file[i]); - } H5E_END_TRY; + } + H5E_END_TRY; if (s.use_named_pipe && np.fd_writer_to_reader >= 0) HDclose(np.fd_writer_to_reader); -- cgit v0.12 From cab2b3fc5fa4b430a55b75ac36649b3a484f74d2 Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Thu, 8 Jul 2021 12:06:26 -0500 Subject: Some minor corrections and adjustments. --- test/testvfdswmr.sh.in | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in index d187d97..6095c03 100644 --- a/test/testvfdswmr.sh.in +++ b/test/testvfdswmr.sh.in @@ -1006,9 +1006,6 @@ for options in ${os_grp_op_list[*]}; do rm -f vfd_swmr_group_reader.*.{out,rc} done -# bigset test for smaller chunks -======= - ############################################################################### # # Setting for bigset (few_big and many_small) tests @@ -1026,8 +1023,8 @@ BIGSET_many_s=100 # -s option: # of datasets (for many_small t # Setting for exhaustive and quick runs # if [[ "$HDF5TestExpress" -eq 0 ]] ; then # exhaustive run - BIGSET_n=50 - BIGSET_few_s=40 + BIGSET_n=200 + BIGSET_few_s=100 BIGSET_many_s=1000 elif [[ "$HDF5TestExpress" -gt 1 ]]; then # quick run BIGSET_n=10 @@ -1036,15 +1033,14 @@ elif [[ "$HDF5TestExpress" -gt 1 ]]; then # quick run fi # # - -for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do +for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do if [ ${do_many_small:-no} = no ]; then continue fi # # Test many small datasets of one and two dimensions. # - # Perform 25 iterations on 200 extensible datasets configured with + # Perform 25 iterations on 100 extensible datasets configured with # 16x16 chunks of 32-bit unsigned integer elements, # expanding each dataset by a chunk in one dimension (up to 50x1 # 16x16 chunks) on each iteration. @@ -1061,8 +1057,8 @@ for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_many_s -r 16 -c 16 -q -l 6 & pid_reader=$! - # Wait for the reader to finish before the - # writer quits: the writer holds the file open so that the + # Wait for the reader to finish before signalling the + # writer to quit: the writer holds the file open so that the # reader will find the shadow file when it opens # the .h5 file. wait $pid_reader @@ -1086,11 +1082,11 @@ for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F done # bigset test for bigger chunks -for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do +for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do # # Test a few big datasets of one and two dimensions. # - # Perform 25 iterations on 20 extensible datasets configured with + # Perform 25 iterations on 10 extensible datasets configured with # 256x256 chunks of 32-bit unsigned integer elements, # expanding each dataset by a chunk in one dimension (up to 50x1 # 256x256 chunks) on each iteration. @@ -1110,8 +1106,8 @@ for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_few_s -r 256 -c 256 -q -l 3 & pid_reader=$! - # Wait for the reader to finish before the - # writer quits: the writer holds the file open so that the + # Wait for the reader to finish before signalling the + # writer to quit: the writer holds the file open so that the # reader will find the shadow file when it opens # the .h5 file. wait $pid_reader -- cgit v0.12 From 0bfc2b7f707f4f272e9a67c031ea96b9e7b63247 Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Thu, 15 Jul 2021 09:49:29 -0500 Subject: Minor change: move the end tick function to just before the file is closed. --- test/vfd_swmr_bigset_writer.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c index 23f2125..9c05fa6 100644 --- a/test/vfd_swmr_bigset_writer.c +++ b/test/vfd_swmr_bigset_writer.c @@ -737,6 +737,17 @@ state_destroy(state_t *s) TEST_ERROR; } + /* The writer ends the tick before closing the file to make the data visible to the reader */ + if (s->writer && s->use_vfd_swmr) { + unsigned long j; + + if (s->vds != vds_multi) + H5Fvfd_swmr_end_tick(s->file[0]); + else + for (j = 0; j < NELMTS(s->file); j++) + H5Fvfd_swmr_end_tick(s->file[j]); + } + /* For checking the time spent in file close. It's for running the writer alone */ if (s->do_perf) { if (HDclock_gettime(CLOCK_MONOTONIC, &start_time) == -1) { @@ -2049,17 +2060,6 @@ write_dsets(state_t s, np_state_t *np, mat_t *mat) } } - /* After finishing writing all the chunks, end the tick */ - if (s.use_vfd_swmr && step == (s.nsteps - 1)) { - unsigned long i; - - if (s.vds != vds_multi) - H5Fvfd_swmr_end_tick(s.file[0]); - else - for (i = 0; i < NELMTS(s.file); i++) - H5Fvfd_swmr_end_tick(s.file[i]); - } - /* Notify the reader to start verification by * sending the timestamp and the number of chunks written */ -- cgit v0.12 From 77321a55641a0520a887a6065cca9d335502f739 Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Wed, 1 Sep 2021 11:08:15 -0500 Subject: Some important updates to the test: added test for 3D datasets and several new command-line options. --- test/testvfdswmr.sh.in | 32 +- test/vfd_swmr_bigset_writer.c | 666 +++++++++++++++++++++++++++++------------- 2 files changed, 488 insertions(+), 210 deletions(-) diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in index ac54968..5a3b0f7 100644 --- a/test/testvfdswmr.sh.in +++ b/test/testvfdswmr.sh.in @@ -1063,7 +1063,7 @@ elif [[ "$HDF5TestExpress" -gt 1 ]]; then # quick run fi # # -for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do +for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -t -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do if [ ${do_many_small:-no} = no ]; then continue fi @@ -1071,20 +1071,24 @@ for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -V" "-d 1 -M" " # Test many small datasets of one and two dimensions. # # Perform 25 iterations on 100 extensible datasets configured with - # 16x16 chunks of 32-bit unsigned integer elements, - # expanding each dataset by a chunk in one dimension (up to 50x1 + # 2D 16x16 chunks or 3D 8x16x16 chunks of 32-bit unsigned integer elements, + # expanding each dataset by a chunk in one dimension (up to 25x1 # 16x16 chunks) on each iteration. # # Perform the test again, extending each dataset - # in *two* dimensions (up to 50x50 16x16 chunks). + # in *two* dimensions (up to 25x25 16x16 chunks). # + # If testing 3D datasets (-t option), extending each dataset along the + # first dimension (up to 25 8x16x16) + # + echo launch vfd_swmr_bigset_writer many small, options $options catch_out_err_and_rc vfd_swmr_bigset_writer \ - ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_many_s -r 16 -c 16 -q -l 6 & + ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_many_s -e 8 -r 16 -c 16 -q -l 6 & pid_writer=$! catch_out_err_and_rc vfd_swmr_bigset_reader \ - ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_many_s -r 16 -c 16 -q -l 6 & + ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_many_s -e 8 -r 16 -c 16 -q -l 6 & pid_reader=$! # Wait for the reader to finish before signalling the @@ -1112,28 +1116,32 @@ for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -V" "-d 1 -M" " done # bigset test for bigger chunks -for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do +for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -t -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do # # Test a few big datasets of one and two dimensions. # # Perform 25 iterations on 10 extensible datasets configured with - # 256x256 chunks of 32-bit unsigned integer elements, - # expanding each dataset by a chunk in one dimension (up to 50x1 + # 2D 256x256 chunks or 3D 64x256x256 of 32-bit unsigned integer elements, + # expanding each dataset by a chunk in one dimension (up to 25x1 # 256x256 chunks) on each iteration. # # Perform the test again, extending each dataset - # in *two* dimensions (up to 50x50 256x256 chunks). + # in *two* dimensions (up to 25x25 256x256 chunks). # + # If testing 3D datasets (-t option), extending each dataset along the + # first dimension (up to 25 64x256x256) + # + if [ ${do_few_big:-no} = no ]; then continue fi echo launch vfd_swmr_bigset_writer few big, options $options ......may take some time...... catch_out_err_and_rc vfd_swmr_bigset_writer \ - ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_few_s -r 256 -c 256 -q -l 3 & + ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_few_s -e 64 -r 256 -c 256 -q -l 3 & pid_writer=$! catch_out_err_and_rc vfd_swmr_bigset_reader \ - ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_few_s -r 256 -c 256 -q -l 3 & + ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_few_s -e 64 -r 256 -c 256 -q -l 3 & pid_reader=$! # Wait for the reader to finish before signalling the diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c index 61d552e..dc95cfb 100644 --- a/test/vfd_swmr_bigset_writer.c +++ b/test/vfd_swmr_bigset_writer.c @@ -38,9 +38,12 @@ * (or the width and height) of a chunk, and writes a test pattern to * the dataset on chunk boundaries. * + * For 3D dataset, the extension is always along the first dimension. + * It does not support VDS yet. + * * The reader should be started with the same user-selectable parameters * as the writer: iterations, number of datasets, chunk width and - * height, dimensions. + * height and depth, dimensions. * * The reader opens the same HDF5 file, reads and re-reads it until all * `n` datasets appear, and then reads and re-reads the datasets until @@ -72,7 +75,6 @@ #include "H5Cpkg.h" #include "H5Fpkg.h" -// #include "H5Iprivate.h" #include "H5HGprivate.h" #include "H5VLprivate.h" @@ -84,12 +86,15 @@ #define MAX_READ_LEN_IN_SECONDS 2 #define TICK_LEN 4 #define MAX_LAG 7 +#define FSP_SIZE 4096 +#define PAGE_BUF_SIZE 4096 #define ROWS 256 #define COLS 512 -#define DEPTHS 1 +#define DEPTH 1 #define RANK2 2 #define RANK3 3 #define NUM_ATTEMPTS 100 +#define SKIP_CHUNK 0 /* Calculate the time passed in seconds. * X is the beginning time; Y is the ending time. @@ -103,7 +108,7 @@ typedef struct _base { } base_t; typedef struct _mat { - unsigned rows, cols; + unsigned depth, rows, cols; uint32_t elt[1]; } mat_t; @@ -132,20 +137,31 @@ typedef struct { struct { quadrant_t ul, ur, bl, br, src; } quadrants; - unsigned int cols, rows; + unsigned int depth, cols, rows; unsigned int asteps; unsigned int nsteps; + unsigned int part_chunk; + unsigned int skip_chunk; + unsigned int over_extend; bool expand_2d; bool test_3d; enum { vds_off, vds_single, vds_multi } vds; bool use_vfd_swmr; bool use_named_pipe; bool do_perf; - bool cross_chunks; + bool cross_chunk_read; bool writer; bool fixed_array; + bool flush_raw_data; hsize_t chunk_dims[RANK2]; hsize_t one_dee_max_dims[RANK2]; + hsize_t fsp_size; + size_t page_buf_size; + uint32_t tick_len; + uint32_t max_lag; + unsigned mdc_init_size; + size_t chunk_cache_size; + unsigned int deflate_level; struct timespec ival; } state_t; @@ -182,11 +198,15 @@ state_initializer(void) .filetype = H5T_NATIVE_UINT32, .one_by_one_sid = H5I_INVALID_HID, .quadrant_dcpl = H5I_INVALID_HID, + .depth = DEPTH, .rows = ROWS, .cols = COLS, .ndatasets = 5, .asteps = 10, .nsteps = 100, + .part_chunk = 0, + .skip_chunk = SKIP_CHUNK, + .over_extend = 1, .filename = {"", "", "", ""}, .expand_2d = false, .test_3d = false, @@ -194,11 +214,19 @@ state_initializer(void) .use_vfd_swmr = true, .use_named_pipe = true, .do_perf = false, - .cross_chunks = false, + .cross_chunk_read = false, .writer = true, .fixed_array = false, .one_dee_max_dims = {ROWS, H5S_UNLIMITED}, .chunk_dims = {ROWS, COLS}, + .fsp_size = FSP_SIZE, + .page_buf_size = PAGE_BUF_SIZE, + .tick_len = TICK_LEN, + .max_lag = MAX_LAG, + .flush_raw_data = false, + .mdc_init_size = 0, + .chunk_cache_size = 0, + .deflate_level = 0, .ival = (struct timespec){.tv_sec = MAX_READ_LEN_IN_SECONDS, .tv_nsec = 0}}; } @@ -208,79 +236,48 @@ static const hid_t badhid = H5I_INVALID_HID; static hsize_t two_dee_max_dims[RANK2], three_dee_max_dims[RANK3]; -static uint32_t -matget(const mat_t *mat, unsigned i, unsigned j) -{ - return mat->elt[i * mat->cols + j]; -} - -static bool -matset(mat_t *mat, unsigned i, unsigned j, uint32_t v) -{ - if (i >= mat->rows || j >= mat->cols) { - fprintf(stderr, "index out of boundary\n"); - TEST_ERROR; - } - - mat->elt[i * mat->cols + j] = v; - - return true; - -error: - return false; -} - -static mat_t * -newmat(unsigned rows, unsigned cols) -{ - mat_t *mat; - - mat = HDmalloc(sizeof(*mat) + (rows * cols - 1) * sizeof(mat->elt[0])); - - if (mat == NULL) { - fprintf(stderr, "HDmalloc failed\n"); - TEST_ERROR; - } - - mat->rows = rows; - mat->cols = cols; - - return mat; - -error: - return NULL; -} - static void usage(const char *progname) { fprintf(stderr, - "usage: %s [-C] [-F] [-M] [-P] [-S] [-V] [-W] [-a steps] [-b] [-c cols]\n" - " [-d dims]\n" - " [-l tick_num] [-n iterations] [-r rows] [-s datasets]\n" - " [-t] [-u milliseconds]\n" + "usage: %s [-C] [-F] [-M] [-P] [-R] [-S] [-V] [-W] [-a steps] [-b] [-c cols]\n" + " [-d dims] [-e depth] [-f tick_len] [-g max_lag] [-j skip_chunk] [-k part_chunk]\n" + " [-l tick_num] [-n iterations] [-o page_buf_size] [-p fsp_size] [-r rows]\n" + " [-s datasets] [-t] [-u over_extend] [-v chunk_cache_size] [-w deflate_level]\n" "\n" - "-C: cross-over chunks during chunk verification\n" + "-C: cross-over chunk read during chunk verification\n" "-F: fixed maximal dimension for the chunked datasets\n" "-M: use virtual datasets and many source\n" " files\n" - "-P: do the performance measurement" + "-N: do not use named pipes\n" + "-P: do the performance measurement\n" + "-R: flush raw data\n" "-S: do not use VFD SWMR\n" "-V: use virtual datasets and a single\n" " source file\n" "-a steps: `steps` between adding attributes\n" "-b: write data in big-endian byte order\n" - "-c cols: `cols` columns per chunk\n" + "-c cols: `cols` columns of the chunk\n" "-d 1|one|2|two|both: select dataset expansion in one or\n" " both dimensions\n" + "-e depth: the first dimension of the 3D chunk\n" + "-f tick_len: tick length\n" + "-g max_lag: maximal lag\n" + "-j skip_chunk: skip the Nth (skip_chunk) chunks during chunk writing\n" + "-k part_chunk: the size for partial chunk write\n" "-l tick_num: expected maximal number of ticks from\n" " the writer's finishing creation to the reader's finishing validation\n" - "-N: do not use named pipes\n" + "-m mdc_init_size: the initial size of metadata cache in megabytes (must be between 1 and 32MB)\n" "-n iterations: how many times to expand each dataset\n" - "-r rows: `rows` rows per chunk\n" + "-o page_buf_size: page buffer size\n" + "-p fsp_size: file space page size\n" + "-r rows: `rows` rows of the chunk\n" "-s datasets: number of datasets to create\n" "-t: enable test for 3D datasets (dataset expansion is along one dimension)\n" " currently, 3D datasets isn't tested with VDS\n" + "-u over_extend: extend the size of the dataset in multiple chunks or partial chunks\n" + "-v chunk_cache_size: the size of raw data chunk cache in bytes\n" + "-w deflate_level: the level (0 - 9) of gzip compression\n" "\n", progname); exit(EXIT_FAILURE); @@ -321,6 +318,8 @@ state_init(state_t *s, int argc, char **argv) const hsize_t dims = 1; char * tfile = NULL; char * end; + size_t rdcc_nslots, rdcc_nbytes; + double rdcc_w0; quadrant_t *const ul = &s->quadrants.ul, *const ur = &s->quadrants.ur, *const bl = &s->quadrants.bl, *const br = &s->quadrants.br, *const src = &s->quadrants.src; const char *personality; @@ -336,11 +335,11 @@ state_init(state_t *s, int argc, char **argv) HDfree(tfile); - while ((ch = getopt(argc, argv, "CFMNPSVa:bc:d:l:n:qr:s:t")) != -1) { + while ((ch = getopt(argc, argv, "CFMNPRSVa:bc:d:e:f:g:j:k:l:m:n:o:p:qr:s:tu:v:w:")) != -1) { switch (ch) { case 'C': /* This flag indicates cross-over chunk read during data validation */ - s->cross_chunks = true; + s->cross_chunk_read = true; break; case 'F': /* The flag to indicate whether the maximal dimension of the chunked datasets is fixed or @@ -353,6 +352,9 @@ state_init(state_t *s, int argc, char **argv) case 'P': s->do_perf = true; break; + case 'R': + s->flush_raw_data = true; + break; case 'S': s->use_vfd_swmr = false; break; @@ -376,10 +378,21 @@ state_init(state_t *s, int argc, char **argv) break; case 'a': case 'c': + case 'e': + case 'f': + case 'g': + case 'j': + case 'k': case 'l': + case 'm': case 'n': + case 'o': + case 'p': case 'r': case 's': + case 'u': + case 'v': + case 'w': errno = 0; tmp = HDstrtoul(optarg, &end, 0); if (end == optarg || *end != '\0') { @@ -404,6 +417,16 @@ state_init(state_t *s, int argc, char **argv) s->asteps = (unsigned)tmp; else if (ch == 'c') s->cols = (unsigned)tmp; + else if (ch == 'e') + s->depth = (unsigned)tmp; + else if (ch == 'f') + s->tick_len = (unsigned)tmp; + else if (ch == 'g') + s->max_lag = (unsigned)tmp; + else if (ch == 'j') + s->skip_chunk = (unsigned)tmp; + else if (ch == 'k') + s->part_chunk = (unsigned)tmp; else if (ch == 'l') { /* Translate the tick number to time represented by the timespec struct */ float time = (float)(((unsigned)tmp * TICK_LEN) / 10.0); @@ -413,10 +436,22 @@ state_init(state_t *s, int argc, char **argv) s->ival.tv_sec = sec; s->ival.tv_nsec = nsec; } + else if (ch == 'm') + s->mdc_init_size = (unsigned)tmp; else if (ch == 'n') s->nsteps = (unsigned)tmp; + else if (ch == 'o') + s->page_buf_size = (unsigned)tmp; + else if (ch == 'p') + s->fsp_size = (unsigned)tmp; else if (ch == 'r') s->rows = (unsigned)tmp; + else if (ch == 'u') + s->over_extend = (unsigned)tmp; + else if (ch == 'v') + s->chunk_cache_size = (unsigned)tmp; + else if (ch == 'w') + s->deflate_level = (unsigned)tmp; else s->ndatasets = (unsigned)tmp; break; @@ -449,13 +484,13 @@ state_init(state_t *s, int argc, char **argv) } if (s->test_3d) { - if (s->expand_2d) { - fprintf(stderr, "3D dataset test doesn't support 2D expansion\n"); + if (s->depth < 1) { + fprintf(stderr, "The depth of 3D dataset can't be less than 1\n"); TEST_ERROR; } - if (s->cross_chunks) { - fprintf(stderr, "3D dataset test doesn't support cross-over chunks during verification\n"); + if (s->expand_2d) { + fprintf(stderr, "3D dataset test doesn't support 2D expansion\n"); TEST_ERROR; } @@ -475,7 +510,7 @@ state_init(state_t *s, int argc, char **argv) two_dee_max_dims[1] = s->cols * s->nsteps; if (s->test_3d) { - three_dee_max_dims[0] = s->nsteps; + three_dee_max_dims[0] = s->depth * s->nsteps; three_dee_max_dims[1] = s->rows; three_dee_max_dims[2] = s->cols; } @@ -597,7 +632,7 @@ state_init(state_t *s, int argc, char **argv) TEST_ERROR; } - if ((s->dataset = HDmalloc(sizeof(*s->dataset) * s->ndatasets)) == NULL) { + if ((s->dataset = HDmalloc(sizeof(hid_t) * s->ndatasets)) == NULL) { fprintf(stderr, "HDmalloc failed\n"); TEST_ERROR; } @@ -613,20 +648,46 @@ state_init(state_t *s, int argc, char **argv) } if (s->test_3d) { - hsize_t dims3[RANK3] = {1, s->chunk_dims[0], s->chunk_dims[1]}; + hsize_t dims3[RANK3] = {s->depth, s->chunk_dims[0], s->chunk_dims[1]}; + + if (s->part_chunk) + dims3[0] = s->part_chunk; if ((s->memspace = H5Screate_simple(RANK3, dims3, NULL)) < 0) { fprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } - } - else { - if ((s->memspace = H5Screate_simple(RANK2, s->chunk_dims, NULL)) < 0) { + } else { + hsize_t dims2[RANK2]; + + if (s->expand_2d) { + dims2[0] = s->chunk_dims[0]; + dims2[1] = s->chunk_dims[1]; + } else { + dims2[0] = s->chunk_dims[0]; + + if (s->part_chunk) + dims2[1] = s->part_chunk; + else + dims2[1] = s->chunk_dims[1]; + } + + if ((s->memspace = H5Screate_simple(RANK2, dims2, NULL)) < 0) { fprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } } + if (s->skip_chunk == 1) { + fprintf(stderr, "can't skip every chunk\n"); + TEST_ERROR; + } + + if (s->over_extend == 0) { + fprintf(stderr, "Extension of the dataset can't be zero\n"); + TEST_ERROR; + } + s->filename[0] = "vfd_swmr_bigset.h5"; if (s->vds == vds_multi) { s->filename[1] = "vfd_swmr_bigset-ur.h5"; @@ -655,8 +716,20 @@ state_init(state_t *s, int argc, char **argv) TEST_ERROR; } - if (H5Pset_chunk_cache(s->dapl, 0, 0, H5D_CHUNK_CACHE_W0_DEFAULT) < 0) { - fprintf(stderr, "H5Pset_chunk_cache failed\n"); + if (s->chunk_cache_size) { + if (H5Pget_chunk_cache(s->dapl, &rdcc_nslots, &rdcc_nbytes, &rdcc_w0) < 0) { + fprintf(stderr, "H5Pget_chunk_cache failed\n"); + TEST_ERROR; + } + + if (H5Pset_chunk_cache(s->dapl, rdcc_nslots, s->chunk_cache_size, rdcc_w0) < 0) { + fprintf(stderr, "H5Pset_chunk_cache failed\n"); + TEST_ERROR; + } + } + + if (s->deflate_level > 9) { + fprintf(stderr, "deflation level must be between 0 and 9\n"); TEST_ERROR; } @@ -1010,22 +1083,24 @@ error: static int notify_reader(np_state_t *np, unsigned step) { - exchange_info_t last; + exchange_info_t *last = HDcalloc(1, sizeof(exchange_info_t)); /* Get the time */ - if (HDclock_gettime(CLOCK_MONOTONIC, &(last.time)) < 0) { + if (HDclock_gettime(CLOCK_MONOTONIC, &(last->time)) < 0) { fprintf(stderr, "HDclock_gettime failed\n"); TEST_ERROR; } - last.step = step; + last->step = step; /* Notify the reader by sending the timestamp and the number of chunks written */ - if (HDwrite(np->fd_writer_to_reader, &last, sizeof(last)) < 0) { + if (HDwrite(np->fd_writer_to_reader, last, sizeof(exchange_info_t)) < 0) { fprintf(stderr, "HDwrite failed"); TEST_ERROR; } + HDfree(last); + return 0; error: @@ -1041,7 +1116,7 @@ create_extensible_dset(state_t *s, unsigned int which) char ul_dname[sizeof("/ul-dataset-9999999999")], ur_dname[sizeof("/ur-dataset-9999999999")], bl_dname[sizeof("/bl-dataset-9999999999")], br_dname[sizeof("/br-dataset-9999999999")]; hid_t dcpl = H5I_INVALID_HID, dset_id = H5I_INVALID_HID, filespace = H5I_INVALID_HID; - hsize_t dims3[3] = {1, s->chunk_dims[0], s->chunk_dims[1]}; + hsize_t dims3[3] = {s->depth, s->chunk_dims[0], s->chunk_dims[1]}; esnprintf(dname, sizeof(dname), "/dataset-%d", which); @@ -1051,7 +1126,7 @@ create_extensible_dset(state_t *s, unsigned int which) } if (s->test_3d) { - /* The chunk is 1 x M x N and grows along the first dimension */ + /* The chunk is L x M x N and grows along the first dimension */ if (H5Pset_chunk(dcpl, RANK3, dims3) < 0) { fprintf(stderr, "H5Pset_chunk for 3D dataset failed\n"); TEST_ERROR; @@ -1064,6 +1139,18 @@ create_extensible_dset(state_t *s, unsigned int which) } } + /* Never write fill value when new chunks are allocated */ + if (H5Pset_fill_time(dcpl, H5D_FILL_TIME_NEVER) < 0) { + fprintf(stderr, "H5Pset_fill_time failed\n"); + TEST_ERROR; + } + + /* GZIP compression */ + if (s->deflate_level && H5Pset_deflate(dcpl, s->deflate_level) < 0) { + fprintf(stderr, "H5Pset_deflate failed\n"); + TEST_ERROR; + } + if (s->vds != vds_off) { sources_t *const srcs = &s->sources[which]; @@ -1122,8 +1209,7 @@ create_extensible_dset(state_t *s, unsigned int which) fprintf(stderr, "H5Screate_simple 3D dataspace failed\n"); TEST_ERROR; } - } - else { + } else { if ((filespace = H5Screate_simple(RANK2, s->chunk_dims, s->expand_2d ? two_dee_max_dims : s->one_dee_max_dims)) < 0) { fprintf(stderr, "H5Screate_simple 2D dataspace failed\n"); @@ -1371,13 +1457,80 @@ error: return false; } +static uint32_t +matget(const mat_t *mat, unsigned k, unsigned i, unsigned j) +{ + return mat->elt[k * mat->rows * mat->cols + i * mat->cols + j]; +} + +static bool +matset(mat_t *mat, unsigned k, unsigned i, unsigned j, uint32_t v) +{ + if (k >= mat->depth || i >= mat->rows || j >= mat->cols) { + fprintf(stderr, "index out of boundary\n"); + TEST_ERROR; + } + + mat->elt[k * mat->rows * mat->cols + i * mat->cols + j] = v; + + return true; + +error: + return false; +} + +static mat_t * +newmat(state_t s) +{ + mat_t *mat; + + /* + * If partial chunk is enabled, the chunk size along the growing dimension + * is replaced with the partial size + */ + if (s.test_3d) { + if (s.part_chunk) { + mat = HDmalloc(sizeof(*mat) + (s.part_chunk * s.rows * s.cols - 1) * sizeof(mat->elt[0])); + mat->depth = s.part_chunk; + } else { + mat = HDmalloc(sizeof(*mat) + (s.depth * s.rows * s.cols - 1) * sizeof(mat->elt[0])); + mat->depth = s.depth; + } + + mat->rows = s.rows; + mat->cols = s.cols; + } else { + if (s.part_chunk && !s.expand_2d) { + mat = HDmalloc(sizeof(*mat) + (s.rows * s.part_chunk - 1) * sizeof(mat->elt[0])); + mat->depth = 1; + mat->rows = s.rows; + mat->cols = s.part_chunk; + } else { + mat = HDmalloc(sizeof(*mat) + (s.rows * s.cols - 1) * sizeof(mat->elt[0])); + mat->depth = 1; + mat->rows = s.rows; + mat->cols = s.cols; + } + } + + if (mat == NULL) { + fprintf(stderr, "HDmalloc failed\n"); + TEST_ERROR; + } + + return mat; + +error: + return NULL; +} + /* Write or verify the dataset test pattern in the matrix `mat`. * `mat` is a "subview" of the `which`th dataset with origin * `(base.row, base.col)`. * * If `do_set` is true, write the pattern; otherwise, verify. * - * The basic test pattern consists of increasing + * For 2D datasets, the basic test pattern consists of increasing * integers written in nested corners of the dataset * starting at element (0, 0): * @@ -1396,38 +1549,43 @@ error: * 15 14 13 12 * * In an actual pattern, the dataset number, `which`, is added to each integer. + * + * For 3D datasets, the increment of chunks is along the first dimension. */ static bool set_or_verify_matrix(mat_t *mat, unsigned int which, base_t base, bool do_set) { - unsigned row, col; + unsigned depth, row, col; bool ret = true; - for (row = 0; row < mat->rows; row++) { - for (col = 0; col < mat->cols; col++) { - uint32_t v; - hsize_t i = base.row + row, j = base.col + col, u; + /* For 2D datasets, `depth` is one */ + for (depth = 0; depth < mat->depth; depth++) { + for (row = 0; row < mat->rows; row++) { + for (col = 0; col < mat->cols; col++) { + uint32_t v; + hsize_t k = base.depth + depth, i = base.row + row, j = base.col + col, u; - if (j <= i) - u = (i + 1) * (i + 1) - 1 - j; - else - u = j * j + i; + if (j <= i) + u = k * 10 + (i + 1) * (i + 1) - 1 - j; + else + u = k * 10 + j * j + i; + + v = (uint32_t)(u + which); - v = (uint32_t)(u + which); - if (do_set) { - if (!matset(mat, row, col, v)) { - fprintf(stderr, "data initialization failed\n"); + if (do_set) { + if (!matset(mat, depth, row, col, v)) { + fprintf(stderr, "data initialization failed\n"); + ret = false; + break; + } + } else if (matget(mat, depth, row, col) != v) { + /* If the data doesn't match, simply return false and + * let the caller repeat this step + */ ret = false; break; } } - else if (matget(mat, row, col) != v) { - /* If the data doesn't match, simply return false and - * let the caller repeat this step - */ - ret = false; - break; - } } } @@ -1446,12 +1604,35 @@ verify_matrix(mat_t *mat, unsigned int which, base_t base) return set_or_verify_matrix(mat, which, base, false); } +static unsigned int +calc_total_steps(state_t s) { + unsigned int total_steps = 0; + + /* Calculate the number of steps depending on if partial chunk is enabled. + * e.g. the original number of steps is 10 and the size of the chunk along + * the growing dimension is 6. When the size of the partial chunk along the + * growing dimension is 5, the number of steps become 12. + */ + if (s.test_3d) { + if (s.part_chunk) + total_steps = s.nsteps * s.depth / s.part_chunk; + else + total_steps = s.nsteps; + } else if (s.expand_2d) { + total_steps = s.nsteps; + } else { + if (s.part_chunk) + total_steps = s.nsteps * s.cols / s.part_chunk; + else + total_steps = s.nsteps; + } + + return total_steps; +} + static bool verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t base) { - hsize_t offset2[RANK2] = {base.row, base.col}; - hsize_t offset3[RANK3] = {base.depth, base.row, base.col}; - hsize_t count3[RANK3] = {1, s->chunk_dims[0], s->chunk_dims[1]}; herr_t status; hid_t dset_id; @@ -1460,22 +1641,36 @@ verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t bas TEST_ERROR; } - if (s->test_3d) - dbgf(1, "verifying chunk %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE "\n", base.row, base.col, - base.depth); - else - dbgf(1, "verifying chunk %" PRIuHSIZE ", %" PRIuHSIZE "\n", base.row, base.col); - dset_id = s->dataset[which]; if (s->test_3d) { + hsize_t offset3[RANK3] = {base.depth, base.row, base.col}; + hsize_t count3[RANK3] = {s->depth, s->chunk_dims[0], s->chunk_dims[1]}; + + if (s->part_chunk) + count3[0] = s->part_chunk; + if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset3, NULL, count3, NULL) < 0) { fprintf(stderr, "H5Sselect_hyperslab failed\n"); TEST_ERROR; } - } - else { - if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset2, NULL, s->chunk_dims, NULL) < 0) { + } else { + hsize_t offset2[RANK2] = {base.row, base.col}; + hsize_t count2[RANK2]; + + if (s->expand_2d) { + count2[0] = s->chunk_dims[0]; + count2[1] = s->chunk_dims[1]; + } else { + count2[0] = s->chunk_dims[0]; + + if (s->part_chunk) + count2[1] = s->part_chunk; + else + count2[1] = s->chunk_dims[1]; + } + + if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset2, NULL, count2, NULL) < 0) { fprintf(stderr, "H5Sselect_hyperslab failed\n"); TEST_ERROR; } @@ -1484,8 +1679,7 @@ verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t bas /* A failure to read the data may indicate the data isn't ready yet. Instead of displaying the error * stack, simply return false and let the caller repeat this step. */ - H5E_BEGIN_TRY - { + H5E_BEGIN_TRY { status = H5Dread(dset_id, H5T_NATIVE_UINT32, s->memspace, filespace, H5P_DEFAULT, mat->elt); } H5E_END_TRY; @@ -1535,9 +1729,6 @@ error: static bool init_and_write_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t base) { - hsize_t offset2[RANK2] = {base.row, base.col}; - hsize_t offset3[RANK3] = {base.depth, base.row, base.col}; - hsize_t count3[RANK3] = {1, s->chunk_dims[0], s->chunk_dims[1]}; hid_t dset_id; dset_id = s->dataset[which]; @@ -1548,14 +1739,37 @@ init_and_write_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, ba } if (s->test_3d) { - /* The chunk dimensions are 1 x M x N. It grows along the first dimension */ + hsize_t offset3[RANK3] = {base.depth, base.row, base.col}; + hsize_t count3[RANK3] = {s->depth, s->chunk_dims[0], s->chunk_dims[1]}; + + /* Handling partial chunk */ + if (s->part_chunk) + count3[0] = s->part_chunk; + + /* The chunk dimensions are L x M x N. It grows along the first dimension */ if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset3, NULL, count3, NULL) < 0) { fprintf(stderr, "H5Sselect_hyperslab for 2D dataset failed\n"); TEST_ERROR; } } else { - if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset2, NULL, s->chunk_dims, NULL) < 0) { + hsize_t offset2[RANK2] = {base.row, base.col}; + hsize_t count2[RANK2]; + + if (s->expand_2d) { + count2[0] = s->chunk_dims[0]; + count2[1] = s->chunk_dims[1]; + } else { + count2[0] = s->chunk_dims[0]; + + /* Handling partial chunk */ + if (s->part_chunk) + count2[1] = s->part_chunk; + else + count2[1] = s->chunk_dims[1]; + } + + if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset2, NULL, count2, NULL) < 0) { fprintf(stderr, "H5Sselect_hyperslab for 2D dataset failed\n"); TEST_ERROR; } @@ -1631,7 +1845,8 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini dset_id = s->dataset[which]; - /* Attempt to check the availablity of the chunks for a number time before reporting it as a failure */ + /* Attempt to check the availablity of the chunks for a number times + * (NUM_ATTEMPTS) before reporting it as a failure */ for (i = 0; i < NUM_ATTEMPTS; i++) { if (H5Drefresh(dset_id) < 0) { fprintf(stderr, "H5Drefresh failed\n"); @@ -1649,15 +1864,22 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini TEST_ERROR; } - nchunks = (unsigned)size3[0]; - } - else { + /* Handling partial chunks */ + if (s->part_chunk) + nchunks = (unsigned)size3[0] / s->part_chunk; + else + nchunks = (unsigned)size3[0] / s->depth; + } else { if (H5Sget_simple_extent_dims(filespace, size2, NULL) < 0) { fprintf(stderr, "H5Sget_simple_extent_dims failed\n"); TEST_ERROR; } - nchunks = (unsigned)(size2[1] / s->chunk_dims[1]); + /* Handling partial chunks */ + if (s->part_chunk) + nchunks = (unsigned)(size2[1] / s->part_chunk); + else + nchunks = (unsigned)(size2[1] / s->chunk_dims[1]); } /* Make sure the chunks show up on the reader side. Otherwise sleep a while and try again */ @@ -1675,10 +1897,13 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini for (step = finished_step; step < last_step; step++) { dbgf(1, "%s: which %u step %u\n", __func__, which, step); + if (s->skip_chunk && step % s->skip_chunk == 0) + continue; + /* Read data that randomly crosses over chunks. But it should not happen to * the last chunk being written */ - if (s->cross_chunks) { + if (s->cross_chunk_read) { if (step == last_step - 1) ofs = 0; else @@ -1688,48 +1913,46 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini ofs = 0; if (s->test_3d) { - size3[0] = 1 + step; - size3[1] = s->chunk_dims[0]; - size3[2] = s->chunk_dims[1]; - last.depth = step; + if (s->part_chunk) { + last.depth = s->part_chunk * step + ofs; + } else { + last.depth = s->depth * step + ofs; + } + last.row = 0; last.col = 0; - } - else { + } else { + last.depth = 0; + if (s->expand_2d) { - size2[0] = s->chunk_dims[0] * (1 + step); - size2[1] = s->chunk_dims[1] * (1 + step); last.row = s->chunk_dims[0] * step + ofs; last.col = s->chunk_dims[1] * step + ofs; - } - else { - size2[0] = s->chunk_dims[0]; - size2[1] = s->chunk_dims[1] * (1 + step); + } else { last.row = 0; - last.col = s->chunk_dims[1] * step + ofs; + + if (s->part_chunk) { + last.col = s->part_chunk * step + ofs; + } else { + last.col = s->chunk_dims[1] * step + ofs; + } } } - if (s->test_3d) { - dbgf(1, "new size3 %" PRIuHSIZE ", %" PRIuHSIZE ", %" PRIuHSIZE "\n", size3[0], size3[1], - size3[2]); + if (s->test_3d) dbgf(1, "last row %" PRIuHSIZE " col %" PRIuHSIZE " depth %" PRIuHSIZE "\n", last.row, last.col, last.depth); - } - else { - dbgf(1, "new size2 %" PRIuHSIZE ", %" PRIuHSIZE "\n", size2[0], size2[1]); + else dbgf(1, "last row %" PRIuHSIZE " col %" PRIuHSIZE "\n", last.row, last.col); - } if (s->test_3d || !s->expand_2d) { if (!repeat_verify_chunk(s, filespace, mat, which, last)) { fprintf(stderr, "chunk verification failed\n"); TEST_ERROR; } - } - else { + } else { /* Down the right side, intersecting the bottom row. */ base.col = last.col; + base.depth = 0; for (base.row = 0; base.row <= last.row; base.row += s->chunk_dims[0]) { if (!repeat_verify_chunk(s, filespace, mat, which, base)) { fprintf(stderr, "chunk verification failed\n"); @@ -1775,10 +1998,13 @@ verify_dsets(state_t s, np_state_t *np, mat_t *mat) unsigned finished_step = 0; unsigned which; unsigned counter = 0; + unsigned total_steps = 0; double passed_time = 0.0, total_time = 0.0, min_time = 1000000.0, max_time = 0.0; exchange_info_t last; struct timespec end_time; + total_steps = calc_total_steps(s); + do { /* Receive the notice of the writer finishing creation, * including the number of chunks finished and the timestamp @@ -1829,7 +2055,7 @@ verify_dsets(state_t s, np_state_t *np, mat_t *mat) if (passed_time < min_time) min_time = passed_time; } - } while (finished_step < s.nsteps); + } while (finished_step < total_steps); /* Print out the performance information */ if (s.use_named_pipe && s.do_perf && counter) @@ -1898,33 +2124,50 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * dset_id = s->dataset[which]; - if (s->asteps != 0 && step % s->asteps == 0) { + if (s->asteps != 0 && step % s->asteps == 0) { if (!add_dset_attribute(s, dset_id, s->one_by_one_sid, which, step)) { fprintf(stderr, "add_dset_attribute failed\n"); TEST_ERROR; } } + /* Handling both over extension of the datasets and partial chunks. Datasets + * can be extended multiple chunks instead of one chunk at a time. + * e.g. if the over extension is set to 10 chunks, the datasets are extended + * 10 chunks along the growing dimension after every 10 chunks are written. + */ if (s->test_3d) { - size3[0] = 1 + step; + if (s->part_chunk) { + size3[0] = s->over_extend * s->part_chunk * (1 + step / s->over_extend); + last.depth = s->part_chunk * step; + } else { + size3[0] = s->over_extend * s->depth * (1 + step / s->over_extend); + last.depth = s->depth * step; + } + size3[1] = s->chunk_dims[0]; size3[2] = s->chunk_dims[1]; - last.depth = step; + last.row = 0; last.col = 0; - } - else { + } else { if (s->expand_2d) { - size2[0] = s->chunk_dims[0] * (1 + step); - size2[1] = s->chunk_dims[1] * (1 + step); + size2[0] = s->over_extend * s->chunk_dims[0] * (1 + step / s->over_extend); + size2[1] = s->over_extend * s->chunk_dims[1] * (1 + step / s->over_extend); + last.row = s->chunk_dims[0] * step; last.col = s->chunk_dims[1] * step; - } - else { + } else { size2[0] = s->chunk_dims[0]; - size2[1] = s->chunk_dims[1] * (1 + step); last.row = 0; - last.col = s->chunk_dims[1] * step; + + if (s->part_chunk) { + size2[1] = s->over_extend * s->part_chunk * (1 + step / s->over_extend); + last.col = s->part_chunk * step; + } else { + size2[1] = s->over_extend * s->chunk_dims[1] * (1 + step / s->over_extend); + last.col = s->chunk_dims[1] * step; + } } last.depth = 0; } @@ -1957,18 +2200,22 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * fprintf(stderr, "H5Dset_extent failed\n"); TEST_ERROR; } - } - else { - if (s->test_3d) { - if (H5Dset_extent(dset_id, size3) < 0) { - fprintf(stderr, "H5Dset_extent for 3D dataset failed\n"); - TEST_ERROR; - } - } - else { - if (H5Dset_extent(dset_id, size2) < 0) { - fprintf(stderr, "H5Dset_extent for 2D dataset failed\n"); - TEST_ERROR; + } else { + /* Handling over extension. Making sure the dataset size doesn't exceed the fixed maximal size */ + if (step % s->over_extend == 0) { + if (s->test_3d) { + if (size3[0] <= three_dee_max_dims[0] && H5Dset_extent(dset_id, size3) < 0) { + fprintf(stderr, "H5Dset_extent for 3D dataset failed\n"); + TEST_ERROR; + } + } else { + if ((s->expand_2d && size2[0] <= two_dee_max_dims[0] && size2[0] <= two_dee_max_dims[0]) + || (!s->expand_2d && size2[1] <= two_dee_max_dims[1])) { + if (H5Dset_extent(dset_id, size2) < 0) { + fprintf(stderr, "H5Dset_extent for 2D dataset failed\n"); + TEST_ERROR; + } + } } } } @@ -1983,8 +2230,7 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * fprintf(stderr, "init_and_write_chunk failed\n"); TEST_ERROR; } - } - else if (s->expand_2d) { + } else if (s->expand_2d) { base.col = last.col; base.depth = 0; for (base.row = 0; base.row <= last.row; base.row += s->chunk_dims[0]) { @@ -2025,7 +2271,7 @@ error: static bool write_dsets(state_t s, np_state_t *np, mat_t *mat) { - unsigned last_step, step, which; + unsigned last_step, step, total_steps, which; unsigned long long old_tick_num; H5F_t * f = NULL; struct timespec start_time, end_time; @@ -2046,16 +2292,22 @@ write_dsets(state_t s, np_state_t *np, mat_t *mat) old_tick_num = f->shared->tick_num; /* Write as many as chunks within the same tick number before notifying - * the reader to verify them. + * the reader to verify them. Take account of partial chunk write + * here by multiplying the dividing factor for partial chunk. Treat each + * partial chunk as if it's a chunk. */ - for (step = 0; step < s.nsteps; step++) { + total_steps = calc_total_steps(s); + + for (step = 0; step < total_steps; step++) { /* Write as many as chunks before the tick number changes */ if (f->shared->tick_num == old_tick_num) { - for (which = 0; which < s.ndatasets; which++) { - dbgf(2, "step %d which %d\n", step, which); - if (!write_extensible_dset(&s, which, step, mat)) { - fprintf(stderr, "write_extensible_dset failed\n"); - TEST_ERROR; + if (!s.skip_chunk || (s.skip_chunk && step % s.skip_chunk != 0)) { + for (which = 0; which < s.ndatasets; which++) { + dbgf(2, "step %d which %d\n", step, which); + if (!write_extensible_dset(&s, which, step, mat)) { + fprintf(stderr, "write_extensible_dset failed\n"); + TEST_ERROR; + } } } } @@ -2063,7 +2315,7 @@ write_dsets(state_t s, np_state_t *np, mat_t *mat) /* Notify the reader to start verification by * sending the timestamp and the number of chunks written */ - if (f->shared->tick_num > old_tick_num || step == (s.nsteps - 1)) { + if (f->shared->tick_num > old_tick_num || step == (total_steps - 1)) { last_step = step + 1; if (s.use_named_pipe && notify_reader(np, last_step) < 0) { fprintf(stderr, "notify_reader failed\n"); @@ -2106,18 +2358,21 @@ main(int argc, char **argv) TEST_ERROR; } - if ((mat = newmat(s.rows, s.cols)) == NULL) { + if ((mat = newmat(s)) == NULL) { fprintf(stderr, "could not allocate matrix\n"); TEST_ERROR; } /* Set fs_strategy (file space strategy) and fs_page_size (file space page size) */ - if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) - errx(EXIT_FAILURE, "H5Pcreate"); + if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, s.fsp_size)) < 0) { + fprintf(stderr, "vfd_swmr_create_fcpl failed\n"); + TEST_ERROR; + } for (i = 0; i < NELMTS(s.file); i++) { hid_t fapl; H5F_vfd_swmr_config_t config; + H5AC_cache_config_t mdc_config; if (s.vds != vds_multi && i > 0) { s.file[i] = s.file[0]; @@ -2125,13 +2380,34 @@ main(int argc, char **argv) } /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(&config, TICK_LEN, MAX_LAG, s.writer, FALSE, 128, "./bigset-shadow-%zu", i); + init_vfd_swmr_config(&config, s.tick_len, s.max_lag, s.writer, s.flush_raw_data, 128, "./bigset-shadow-%zu", i); /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ - fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, 4096, &config); + if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, s.page_buf_size, &config)) < 0) { + fprintf(stderr, "vfd_swmr_create_fapl failed"); + TEST_ERROR; + } - if (fapl < 0) - errx(EXIT_FAILURE, "vfd_swmr_create_fapl"); + /* Set the initial size for the metadata cache between 1 and 32 in megabytes. + * Zero means using the default value, which is no-op. + */ + if (s.mdc_init_size) { + mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION; + + if (H5Pget_mdc_config(fapl, &mdc_config) < 0) { + fprintf(stderr, "H5Pget_mdc_config failed"); + TEST_ERROR; + } + + /* Convert the value to megabytes */ + mdc_config.set_initial_size = TRUE; + mdc_config.initial_size = s.mdc_init_size * 1024 * 1024; + + if (H5Pset_mdc_config(fapl, &mdc_config) < 0) { + fprintf(stderr, "H5Pset_mdc_config failed"); + TEST_ERROR; + } + } s.file[i] = s.writer ? H5Fcreate(s.filename[i], H5F_ACC_TRUNC, fcpl, fapl) : H5Fopen(s.filename[i], H5F_ACC_RDONLY, fapl); @@ -2200,8 +2476,7 @@ main(int argc, char **argv) fprintf(stderr, "write_dsets failed"); TEST_ERROR; } - } - else { + } else { /* Wait for the writer's notice before starting the validation of dataset creation */ np.verify = 1; if (s.use_named_pipe && reader_verify(np, np.verify) < 0) { @@ -2259,11 +2534,6 @@ main(int argc, char **argv) HDfree(mat); - if (s.dataset) - HDfree(s.dataset); - if (s.sources) - HDfree(s.sources); - return EXIT_SUCCESS; error: -- cgit v0.12 From a95bf70967d8075263a86f11ef6047817ff3476a Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 1 Sep 2021 16:12:48 +0000 Subject: Committing clang-format changes --- test/vfd_swmr_bigset_writer.c | 212 ++++++++++++++++++++++++------------------ 1 file changed, 121 insertions(+), 91 deletions(-) diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c index dc95cfb..891bfc9 100644 --- a/test/vfd_swmr_bigset_writer.c +++ b/test/vfd_swmr_bigset_writer.c @@ -239,47 +239,48 @@ static hsize_t two_dee_max_dims[RANK2], three_dee_max_dims[RANK3]; static void usage(const char *progname) { - fprintf(stderr, - "usage: %s [-C] [-F] [-M] [-P] [-R] [-S] [-V] [-W] [-a steps] [-b] [-c cols]\n" - " [-d dims] [-e depth] [-f tick_len] [-g max_lag] [-j skip_chunk] [-k part_chunk]\n" - " [-l tick_num] [-n iterations] [-o page_buf_size] [-p fsp_size] [-r rows]\n" - " [-s datasets] [-t] [-u over_extend] [-v chunk_cache_size] [-w deflate_level]\n" - "\n" - "-C: cross-over chunk read during chunk verification\n" - "-F: fixed maximal dimension for the chunked datasets\n" - "-M: use virtual datasets and many source\n" - " files\n" - "-N: do not use named pipes\n" - "-P: do the performance measurement\n" - "-R: flush raw data\n" - "-S: do not use VFD SWMR\n" - "-V: use virtual datasets and a single\n" - " source file\n" - "-a steps: `steps` between adding attributes\n" - "-b: write data in big-endian byte order\n" - "-c cols: `cols` columns of the chunk\n" - "-d 1|one|2|two|both: select dataset expansion in one or\n" - " both dimensions\n" - "-e depth: the first dimension of the 3D chunk\n" - "-f tick_len: tick length\n" - "-g max_lag: maximal lag\n" - "-j skip_chunk: skip the Nth (skip_chunk) chunks during chunk writing\n" - "-k part_chunk: the size for partial chunk write\n" - "-l tick_num: expected maximal number of ticks from\n" - " the writer's finishing creation to the reader's finishing validation\n" - "-m mdc_init_size: the initial size of metadata cache in megabytes (must be between 1 and 32MB)\n" - "-n iterations: how many times to expand each dataset\n" - "-o page_buf_size: page buffer size\n" - "-p fsp_size: file space page size\n" - "-r rows: `rows` rows of the chunk\n" - "-s datasets: number of datasets to create\n" - "-t: enable test for 3D datasets (dataset expansion is along one dimension)\n" - " currently, 3D datasets isn't tested with VDS\n" - "-u over_extend: extend the size of the dataset in multiple chunks or partial chunks\n" - "-v chunk_cache_size: the size of raw data chunk cache in bytes\n" - "-w deflate_level: the level (0 - 9) of gzip compression\n" - "\n", - progname); + fprintf( + stderr, + "usage: %s [-C] [-F] [-M] [-P] [-R] [-S] [-V] [-W] [-a steps] [-b] [-c cols]\n" + " [-d dims] [-e depth] [-f tick_len] [-g max_lag] [-j skip_chunk] [-k part_chunk]\n" + " [-l tick_num] [-n iterations] [-o page_buf_size] [-p fsp_size] [-r rows]\n" + " [-s datasets] [-t] [-u over_extend] [-v chunk_cache_size] [-w deflate_level]\n" + "\n" + "-C: cross-over chunk read during chunk verification\n" + "-F: fixed maximal dimension for the chunked datasets\n" + "-M: use virtual datasets and many source\n" + " files\n" + "-N: do not use named pipes\n" + "-P: do the performance measurement\n" + "-R: flush raw data\n" + "-S: do not use VFD SWMR\n" + "-V: use virtual datasets and a single\n" + " source file\n" + "-a steps: `steps` between adding attributes\n" + "-b: write data in big-endian byte order\n" + "-c cols: `cols` columns of the chunk\n" + "-d 1|one|2|two|both: select dataset expansion in one or\n" + " both dimensions\n" + "-e depth: the first dimension of the 3D chunk\n" + "-f tick_len: tick length\n" + "-g max_lag: maximal lag\n" + "-j skip_chunk: skip the Nth (skip_chunk) chunks during chunk writing\n" + "-k part_chunk: the size for partial chunk write\n" + "-l tick_num: expected maximal number of ticks from\n" + " the writer's finishing creation to the reader's finishing validation\n" + "-m mdc_init_size: the initial size of metadata cache in megabytes (must be between 1 and 32MB)\n" + "-n iterations: how many times to expand each dataset\n" + "-o page_buf_size: page buffer size\n" + "-p fsp_size: file space page size\n" + "-r rows: `rows` rows of the chunk\n" + "-s datasets: number of datasets to create\n" + "-t: enable test for 3D datasets (dataset expansion is along one dimension)\n" + " currently, 3D datasets isn't tested with VDS\n" + "-u over_extend: extend the size of the dataset in multiple chunks or partial chunks\n" + "-v chunk_cache_size: the size of raw data chunk cache in bytes\n" + "-w deflate_level: the level (0 - 9) of gzip compression\n" + "\n", + progname); exit(EXIT_FAILURE); } @@ -657,13 +658,15 @@ state_init(state_t *s, int argc, char **argv) fprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } - } else { + } + else { hsize_t dims2[RANK2]; if (s->expand_2d) { dims2[0] = s->chunk_dims[0]; dims2[1] = s->chunk_dims[1]; - } else { + } + else { dims2[0] = s->chunk_dims[0]; if (s->part_chunk) @@ -1209,7 +1212,8 @@ create_extensible_dset(state_t *s, unsigned int which) fprintf(stderr, "H5Screate_simple 3D dataspace failed\n"); TEST_ERROR; } - } else { + } + else { if ((filespace = H5Screate_simple(RANK2, s->chunk_dims, s->expand_2d ? two_dee_max_dims : s->one_dee_max_dims)) < 0) { fprintf(stderr, "H5Screate_simple 2D dataspace failed\n"); @@ -1490,26 +1494,29 @@ newmat(state_t s) */ if (s.test_3d) { if (s.part_chunk) { - mat = HDmalloc(sizeof(*mat) + (s.part_chunk * s.rows * s.cols - 1) * sizeof(mat->elt[0])); + mat = HDmalloc(sizeof(*mat) + (s.part_chunk * s.rows * s.cols - 1) * sizeof(mat->elt[0])); mat->depth = s.part_chunk; - } else { - mat = HDmalloc(sizeof(*mat) + (s.depth * s.rows * s.cols - 1) * sizeof(mat->elt[0])); + } + else { + mat = HDmalloc(sizeof(*mat) + (s.depth * s.rows * s.cols - 1) * sizeof(mat->elt[0])); mat->depth = s.depth; } mat->rows = s.rows; mat->cols = s.cols; - } else { + } + else { if (s.part_chunk && !s.expand_2d) { - mat = HDmalloc(sizeof(*mat) + (s.rows * s.part_chunk - 1) * sizeof(mat->elt[0])); + mat = HDmalloc(sizeof(*mat) + (s.rows * s.part_chunk - 1) * sizeof(mat->elt[0])); mat->depth = 1; - mat->rows = s.rows; - mat->cols = s.part_chunk; - } else { - mat = HDmalloc(sizeof(*mat) + (s.rows * s.cols - 1) * sizeof(mat->elt[0])); + mat->rows = s.rows; + mat->cols = s.part_chunk; + } + else { + mat = HDmalloc(sizeof(*mat) + (s.rows * s.cols - 1) * sizeof(mat->elt[0])); mat->depth = 1; - mat->rows = s.rows; - mat->cols = s.cols; + mat->rows = s.rows; + mat->cols = s.cols; } } @@ -1578,7 +1585,8 @@ set_or_verify_matrix(mat_t *mat, unsigned int which, base_t base, bool do_set) ret = false; break; } - } else if (matget(mat, depth, row, col) != v) { + } + else if (matget(mat, depth, row, col) != v) { /* If the data doesn't match, simply return false and * let the caller repeat this step */ @@ -1605,7 +1613,8 @@ verify_matrix(mat_t *mat, unsigned int which, base_t base) } static unsigned int -calc_total_steps(state_t s) { +calc_total_steps(state_t s) +{ unsigned int total_steps = 0; /* Calculate the number of steps depending on if partial chunk is enabled. @@ -1618,9 +1627,11 @@ calc_total_steps(state_t s) { total_steps = s.nsteps * s.depth / s.part_chunk; else total_steps = s.nsteps; - } else if (s.expand_2d) { + } + else if (s.expand_2d) { total_steps = s.nsteps; - } else { + } + else { if (s.part_chunk) total_steps = s.nsteps * s.cols / s.part_chunk; else @@ -1633,8 +1644,8 @@ calc_total_steps(state_t s) { static bool verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t base) { - herr_t status; - hid_t dset_id; + herr_t status; + hid_t dset_id; if (which >= s->ndatasets) { fprintf(stderr, "the dataset order is bigger than the number of datasets"); @@ -1654,14 +1665,16 @@ verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t bas fprintf(stderr, "H5Sselect_hyperslab failed\n"); TEST_ERROR; } - } else { + } + else { hsize_t offset2[RANK2] = {base.row, base.col}; hsize_t count2[RANK2]; if (s->expand_2d) { count2[0] = s->chunk_dims[0]; count2[1] = s->chunk_dims[1]; - } else { + } + else { count2[0] = s->chunk_dims[0]; if (s->part_chunk) @@ -1679,7 +1692,8 @@ verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t bas /* A failure to read the data may indicate the data isn't ready yet. Instead of displaying the error * stack, simply return false and let the caller repeat this step. */ - H5E_BEGIN_TRY { + H5E_BEGIN_TRY + { status = H5Dread(dset_id, H5T_NATIVE_UINT32, s->memspace, filespace, H5P_DEFAULT, mat->elt); } H5E_END_TRY; @@ -1729,7 +1743,7 @@ error: static bool init_and_write_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t base) { - hid_t dset_id; + hid_t dset_id; dset_id = s->dataset[which]; @@ -1759,7 +1773,8 @@ init_and_write_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, ba if (s->expand_2d) { count2[0] = s->chunk_dims[0]; count2[1] = s->chunk_dims[1]; - } else { + } + else { count2[0] = s->chunk_dims[0]; /* Handling partial chunk */ @@ -1869,7 +1884,8 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini nchunks = (unsigned)size3[0] / s->part_chunk; else nchunks = (unsigned)size3[0] / s->depth; - } else { + } + else { if (H5Sget_simple_extent_dims(filespace, size2, NULL) < 0) { fprintf(stderr, "H5Sget_simple_extent_dims failed\n"); TEST_ERROR; @@ -1915,24 +1931,28 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini if (s->test_3d) { if (s->part_chunk) { last.depth = s->part_chunk * step + ofs; - } else { + } + else { last.depth = s->depth * step + ofs; } - last.row = 0; - last.col = 0; - } else { + last.row = 0; + last.col = 0; + } + else { last.depth = 0; if (s->expand_2d) { last.row = s->chunk_dims[0] * step + ofs; last.col = s->chunk_dims[1] * step + ofs; - } else { + } + else { last.row = 0; if (s->part_chunk) { last.col = s->part_chunk * step + ofs; - } else { + } + else { last.col = s->chunk_dims[1] * step + ofs; } } @@ -1949,9 +1969,10 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini fprintf(stderr, "chunk verification failed\n"); TEST_ERROR; } - } else { + } + else { /* Down the right side, intersecting the bottom row. */ - base.col = last.col; + base.col = last.col; base.depth = 0; for (base.row = 0; base.row <= last.row; base.row += s->chunk_dims[0]) { if (!repeat_verify_chunk(s, filespace, mat, which, base)) { @@ -2124,7 +2145,7 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * dset_id = s->dataset[which]; - if (s->asteps != 0 && step % s->asteps == 0) { + if (s->asteps != 0 && step % s->asteps == 0) { if (!add_dset_attribute(s, dset_id, s->one_by_one_sid, which, step)) { fprintf(stderr, "add_dset_attribute failed\n"); TEST_ERROR; @@ -2140,31 +2161,35 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * if (s->part_chunk) { size3[0] = s->over_extend * s->part_chunk * (1 + step / s->over_extend); last.depth = s->part_chunk * step; - } else { + } + else { size3[0] = s->over_extend * s->depth * (1 + step / s->over_extend); last.depth = s->depth * step; } - size3[1] = s->chunk_dims[0]; - size3[2] = s->chunk_dims[1]; + size3[1] = s->chunk_dims[0]; + size3[2] = s->chunk_dims[1]; - last.row = 0; - last.col = 0; - } else { + last.row = 0; + last.col = 0; + } + else { if (s->expand_2d) { size2[0] = s->over_extend * s->chunk_dims[0] * (1 + step / s->over_extend); size2[1] = s->over_extend * s->chunk_dims[1] * (1 + step / s->over_extend); last.row = s->chunk_dims[0] * step; last.col = s->chunk_dims[1] * step; - } else { + } + else { size2[0] = s->chunk_dims[0]; last.row = 0; if (s->part_chunk) { size2[1] = s->over_extend * s->part_chunk * (1 + step / s->over_extend); last.col = s->part_chunk * step; - } else { + } + else { size2[1] = s->over_extend * s->chunk_dims[1] * (1 + step / s->over_extend); last.col = s->chunk_dims[1] * step; } @@ -2200,7 +2225,8 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * fprintf(stderr, "H5Dset_extent failed\n"); TEST_ERROR; } - } else { + } + else { /* Handling over extension. Making sure the dataset size doesn't exceed the fixed maximal size */ if (step % s->over_extend == 0) { if (s->test_3d) { @@ -2208,9 +2234,10 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * fprintf(stderr, "H5Dset_extent for 3D dataset failed\n"); TEST_ERROR; } - } else { - if ((s->expand_2d && size2[0] <= two_dee_max_dims[0] && size2[0] <= two_dee_max_dims[0]) - || (!s->expand_2d && size2[1] <= two_dee_max_dims[1])) { + } + else { + if ((s->expand_2d && size2[0] <= two_dee_max_dims[0] && size2[0] <= two_dee_max_dims[0]) || + (!s->expand_2d && size2[1] <= two_dee_max_dims[1])) { if (H5Dset_extent(dset_id, size2) < 0) { fprintf(stderr, "H5Dset_extent for 2D dataset failed\n"); TEST_ERROR; @@ -2230,7 +2257,8 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * fprintf(stderr, "init_and_write_chunk failed\n"); TEST_ERROR; } - } else if (s->expand_2d) { + } + else if (s->expand_2d) { base.col = last.col; base.depth = 0; for (base.row = 0; base.row <= last.row; base.row += s->chunk_dims[0]) { @@ -2380,7 +2408,8 @@ main(int argc, char **argv) } /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(&config, s.tick_len, s.max_lag, s.writer, s.flush_raw_data, 128, "./bigset-shadow-%zu", i); + init_vfd_swmr_config(&config, s.tick_len, s.max_lag, s.writer, s.flush_raw_data, 128, + "./bigset-shadow-%zu", i); /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, s.page_buf_size, &config)) < 0) { @@ -2476,7 +2505,8 @@ main(int argc, char **argv) fprintf(stderr, "write_dsets failed"); TEST_ERROR; } - } else { + } + else { /* Wait for the writer's notice before starting the validation of dataset creation */ np.verify = 1; if (s.use_named_pipe && reader_verify(np, np.verify) < 0) { -- cgit v0.12 From 9a9e4983a3f6569785356ebdd4ba496a91fe6ecd Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Wed, 1 Sep 2021 11:44:26 -0500 Subject: Minor adjustment to a parameter. --- test/testvfdswmr.sh.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in index 5a3b0f7..2852f24 100644 --- a/test/testvfdswmr.sh.in +++ b/test/testvfdswmr.sh.in @@ -1084,11 +1084,11 @@ for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -t -F" "-d 1 -V echo launch vfd_swmr_bigset_writer many small, options $options catch_out_err_and_rc vfd_swmr_bigset_writer \ - ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_many_s -e 8 -r 16 -c 16 -q -l 6 & + ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_many_s -e 8 -r 16 -c 16 -q -l 3 & pid_writer=$! catch_out_err_and_rc vfd_swmr_bigset_reader \ - ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_many_s -e 8 -r 16 -c 16 -q -l 6 & + ../vfd_swmr_bigset_reader -n $BIGSET_n $options -s $BIGSET_many_s -e 8 -r 16 -c 16 -q -l 3 & pid_reader=$! # Wait for the reader to finish before signalling the -- cgit v0.12 From 6685a58b8d31675ac8d36c82d79fffa56bd52a0c Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Fri, 3 Sep 2021 15:06:06 -0500 Subject: Add a function call to allocate space early during H5Dcreate and H5Dextend. --- test/vfd_swmr_bigset_writer.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c index 891bfc9..649cf38 100644 --- a/test/vfd_swmr_bigset_writer.c +++ b/test/vfd_swmr_bigset_writer.c @@ -248,20 +248,20 @@ usage(const char *progname) "\n" "-C: cross-over chunk read during chunk verification\n" "-F: fixed maximal dimension for the chunked datasets\n" - "-M: use virtual datasets and many source\n" + "-M: use virtual datasets and many source\n" " files\n" "-N: do not use named pipes\n" "-P: do the performance measurement\n" "-R: flush raw data\n" - "-S: do not use VFD SWMR\n" - "-V: use virtual datasets and a single\n" + "-S: do not use VFD SWMR\n" + "-V: use virtual datasets and a single\n" " source file\n" - "-a steps: `steps` between adding attributes\n" - "-b: write data in big-endian byte order\n" - "-c cols: `cols` columns of the chunk\n" + "-a steps: `steps` between adding attributes\n" + "-b: write data in big-endian byte order\n" + "-c cols: `cols` columns of the chunk\n" "-d 1|one|2|two|both: select dataset expansion in one or\n" " both dimensions\n" - "-e depth: the first dimension of the 3D chunk\n" + "-e depth: the first dimension of the 3D chunk\n" "-f tick_len: tick length\n" "-g max_lag: maximal lag\n" "-j skip_chunk: skip the Nth (skip_chunk) chunks during chunk writing\n" @@ -272,7 +272,7 @@ usage(const char *progname) "-n iterations: how many times to expand each dataset\n" "-o page_buf_size: page buffer size\n" "-p fsp_size: file space page size\n" - "-r rows: `rows` rows of the chunk\n" + "-r rows: `rows` rows of the chunk\n" "-s datasets: number of datasets to create\n" "-t: enable test for 3D datasets (dataset expansion is along one dimension)\n" " currently, 3D datasets isn't tested with VDS\n" @@ -1148,6 +1148,12 @@ create_extensible_dset(state_t *s, unsigned int which) TEST_ERROR; } + /* Early space allocation */ + if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0) { + fprintf(stderr, "H5Pset_alloc_time failed\n"); + TEST_ERROR; + } + /* GZIP compression */ if (s->deflate_level && H5Pset_deflate(dcpl, s->deflate_level) < 0) { fprintf(stderr, "H5Pset_deflate failed\n"); -- cgit v0.12 From 54f414bd564bb7d9cab85b98f144b187ad77feaa Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Mon, 13 Sep 2021 15:22:57 -0500 Subject: Some minor changes and more comments. --- test/testvfdswmr.sh.in | 4 ++-- test/vfd_swmr_bigset_writer.c | 55 ++++++++++++++++++++++++++++++------------- 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in index 2852f24..4dc1e82 100644 --- a/test/testvfdswmr.sh.in +++ b/test/testvfdswmr.sh.in @@ -1063,7 +1063,7 @@ elif [[ "$HDF5TestExpress" -gt 1 ]]; then # quick run fi # # -for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -t -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do +for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -t -F" "-d 1 -t -R" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do if [ ${do_many_small:-no} = no ]; then continue fi @@ -1116,7 +1116,7 @@ for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -t -F" "-d 1 -V done # bigset test for bigger chunks -for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -t -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do +for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -t" "-d 1 -t -F" "-d 1 -t -R" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do # # Test a few big datasets of one and two dimensions. # diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c index 649cf38..bcb3d24 100644 --- a/test/vfd_swmr_bigset_writer.c +++ b/test/vfd_swmr_bigset_writer.c @@ -39,7 +39,9 @@ * the dataset on chunk boundaries. * * For 3D dataset, the extension is always along the first dimension. - * It does not support VDS yet. + * e.g. the chunk size is `l` x `m` x `n`, after `i` iterations, the + * dataset size becomes `i x l` x `m` x `n`. + * It does not test VDS for 3D dataset. * * The reader should be started with the same user-selectable parameters * as the writer: iterations, number of datasets, chunk width and @@ -265,7 +267,7 @@ usage(const char *progname) "-f tick_len: tick length\n" "-g max_lag: maximal lag\n" "-j skip_chunk: skip the Nth (skip_chunk) chunks during chunk writing\n" - "-k part_chunk: the size for partial chunk write\n" + "-k part_chunk: the size for partial chunk write (only along the first dimension)\n" "-l tick_num: expected maximal number of ticks from\n" " the writer's finishing creation to the reader's finishing validation\n" "-m mdc_init_size: the initial size of metadata cache in megabytes (must be between 1 and 32MB)\n" @@ -334,7 +336,8 @@ state_init(state_t *s, int argc, char **argv) esnprintf(s->progname, sizeof(s->progname), "%s", tfile); - HDfree(tfile); + if (tfile) + HDfree(tfile); while ((ch = getopt(argc, argv, "CFMNPRSVa:bc:d:e:f:g:j:k:l:m:n:o:p:qr:s:tu:v:w:")) != -1) { switch (ch) { @@ -681,6 +684,7 @@ state_init(state_t *s, int argc, char **argv) } } + /* The default is zero, meaning no skip */ if (s->skip_chunk == 1) { fprintf(stderr, "can't skip every chunk\n"); TEST_ERROR; @@ -761,11 +765,14 @@ error: } H5E_END_TRY; - HDfree(tfile); + if (tfile) + HDfree(tfile); - HDfree(s->dataset); + if (s->dataset) + HDfree(s->dataset); - HDfree(s->sources); + if (s->sources) + HDfree(s->sources); return false; } @@ -818,10 +825,16 @@ state_destroy(state_t *s) unsigned long j; if (s->vds != vds_multi) - H5Fvfd_swmr_end_tick(s->file[0]); + if (H5Fvfd_swmr_end_tick(s->file[0]) < 0) { + fprintf(stderr, "H5Fclose failed\n"); + TEST_ERROR; + } else for (j = 0; j < NELMTS(s->file); j++) - H5Fvfd_swmr_end_tick(s->file[j]); + if (H5Fvfd_swmr_end_tick(s->file[j]) < 0) { + fprintf(stderr, "H5Fclose failed\n"); + TEST_ERROR; + } } /* For checking the time spent in file close. It's for running the writer alone */ @@ -857,9 +870,11 @@ state_destroy(state_t *s) TIME_PASSED(start_time, end_time)); } - HDfree(s->dataset); + if (s->dataset) + HDfree(s->dataset); - HDfree(s->sources); + if (s->sources) + HDfree(s->sources); return true; @@ -872,9 +887,11 @@ error: } H5E_END_TRY; - HDfree(s->dataset); + if (s->dataset) + HDfree(s->dataset); - HDfree(s->sources); + if (s->sources) + HDfree(s->sources); return false; } @@ -1102,7 +1119,8 @@ notify_reader(np_state_t *np, unsigned step) TEST_ERROR; } - HDfree(last); + if (last) + HDfree(last); return 0; @@ -1625,8 +1643,9 @@ calc_total_steps(state_t s) /* Calculate the number of steps depending on if partial chunk is enabled. * e.g. the original number of steps is 10 and the size of the chunk along - * the growing dimension is 6. When the size of the partial chunk along the - * growing dimension is 5, the number of steps become 12. + * the growing dimension is 6. The number of elements for this dimension is + * 60. When the size of the partial chunk along the growing dimension is 5 + * (treated as the new chunk size), the number of steps becomes 12. */ if (s.test_3d) { if (s.part_chunk) @@ -2568,7 +2587,8 @@ main(int argc, char **argv) TEST_ERROR; } - HDfree(mat); + if (mat) + HDfree(mat); return EXIT_SUCCESS; @@ -2593,6 +2613,9 @@ error: HDremove(np.fifo_reader_to_writer); } + if (mat) + HDfree(mat); + return EXIT_FAILURE; } #else /* H5_HAVE_WIN32_API */ -- cgit v0.12 From 8d3cf7eb46da53b08cdf694ee291fefacb6c15e3 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 13 Sep 2021 20:26:40 +0000 Subject: Committing clang-format changes --- test/vfd_swmr_bigset_writer.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c index bcb3d24..011440c 100644 --- a/test/vfd_swmr_bigset_writer.c +++ b/test/vfd_swmr_bigset_writer.c @@ -829,12 +829,12 @@ state_destroy(state_t *s) fprintf(stderr, "H5Fclose failed\n"); TEST_ERROR; } - else - for (j = 0; j < NELMTS(s->file); j++) - if (H5Fvfd_swmr_end_tick(s->file[j]) < 0) { - fprintf(stderr, "H5Fclose failed\n"); - TEST_ERROR; - } + else + for (j = 0; j < NELMTS(s->file); j++) + if (H5Fvfd_swmr_end_tick(s->file[j]) < 0) { + fprintf(stderr, "H5Fclose failed\n"); + TEST_ERROR; + } } /* For checking the time spent in file close. It's for running the writer alone */ @@ -1644,7 +1644,7 @@ calc_total_steps(state_t s) /* Calculate the number of steps depending on if partial chunk is enabled. * e.g. the original number of steps is 10 and the size of the chunk along * the growing dimension is 6. The number of elements for this dimension is - * 60. When the size of the partial chunk along the growing dimension is 5 + * 60. When the size of the partial chunk along the growing dimension is 5 * (treated as the new chunk size), the number of steps becomes 12. */ if (s.test_3d) { -- cgit v0.12 From db4a1755b3e9fb8a14018ec8a44cc9a92d830aac Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Tue, 14 Sep 2021 11:46:40 -0500 Subject: Changed fprintf to HDfprintf and also printed out the write speed for performance information. --- test/vfd_swmr_bigset_writer.c | 400 ++++++++++++++++++++++-------------------- 1 file changed, 207 insertions(+), 193 deletions(-) diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c index 011440c..22ec3fb 100644 --- a/test/vfd_swmr_bigset_writer.c +++ b/test/vfd_swmr_bigset_writer.c @@ -241,7 +241,7 @@ static hsize_t two_dee_max_dims[RANK2], three_dee_max_dims[RANK3]; static void usage(const char *progname) { - fprintf( + HDfprintf( stderr, "usage: %s [-C] [-F] [-M] [-P] [-R] [-S] [-V] [-W] [-a steps] [-b] [-c cols]\n" " [-d dims] [-e depth] [-f tick_len] [-g max_lag] [-j skip_chunk] [-k part_chunk]\n" @@ -291,12 +291,12 @@ make_quadrant_dataspace(state_t *s, quadrant_t *q) { if ((q->space = H5Screate_simple(NELMTS(s->chunk_dims), s->chunk_dims, s->expand_2d ? two_dee_max_dims : s->one_dee_max_dims)) < 0) { - fprintf(stderr, "H5Screate_simple failed\n"); + HDfprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } if (H5Sselect_hyperslab(q->space, H5S_SELECT_SET, q->start, q->stride, q->count, q->block) < 0) { - fprintf(stderr, "H5Sselect_hyperslab failed\n"); + HDfprintf(stderr, "H5Sselect_hyperslab failed\n"); TEST_ERROR; } @@ -330,7 +330,7 @@ state_init(state_t *s, int argc, char **argv) *s = state_initializer(); if (H5_basename(argv[0], &tfile) < 0) { - fprintf(stderr, "H5_basename failed\n"); + HDfprintf(stderr, "H5_basename failed\n"); TEST_ERROR; } @@ -376,7 +376,7 @@ state_init(state_t *s, int argc, char **argv) strcmp(optarg, "both") == 0) s->expand_2d = true; else { - fprintf(stderr, "bad -d argument %s\n", optarg); + HDfprintf(stderr, "bad -d argument %s\n", optarg); TEST_ERROR; } break; @@ -400,20 +400,20 @@ state_init(state_t *s, int argc, char **argv) errno = 0; tmp = HDstrtoul(optarg, &end, 0); if (end == optarg || *end != '\0') { - fprintf(stderr, "couldn't parse -%c argument %s\n", ch, optarg); + HDfprintf(stderr, "couldn't parse -%c argument %s\n", ch, optarg); TEST_ERROR; } else if (errno != 0) { - fprintf(stderr, "couldn't parse -%c argument %s\n", ch, optarg); + HDfprintf(stderr, "couldn't parse -%c argument %s\n", ch, optarg); TEST_ERROR; } else if (tmp > UINT_MAX) { - fprintf(stderr, "-%c argument %lu too large", ch, tmp); + HDfprintf(stderr, "-%c argument %lu too large", ch, tmp); TEST_ERROR; } if ((ch == 'c' || ch == 'r') && tmp == 0) { - fprintf(stderr, "-%c argument %lu must be >= 1", ch, tmp); + HDfprintf(stderr, "-%c argument %lu must be >= 1", ch, tmp); TEST_ERROR; } @@ -478,28 +478,28 @@ state_init(state_t *s, int argc, char **argv) argv += optind; if (argc > 0) { - fprintf(stderr, "unexpected command-line arguments\n"); + HDfprintf(stderr, "unexpected command-line arguments\n"); TEST_ERROR; } if (s->vds != vds_off && s->expand_2d) { - fprintf(stderr, "virtual datasets and 2D datasets are mutually exclusive\n"); + HDfprintf(stderr, "virtual datasets and 2D datasets are mutually exclusive\n"); TEST_ERROR; } if (s->test_3d) { if (s->depth < 1) { - fprintf(stderr, "The depth of 3D dataset can't be less than 1\n"); + HDfprintf(stderr, "The depth of 3D dataset can't be less than 1\n"); TEST_ERROR; } if (s->expand_2d) { - fprintf(stderr, "3D dataset test doesn't support 2D expansion\n"); + HDfprintf(stderr, "3D dataset test doesn't support 2D expansion\n"); TEST_ERROR; } if (s->vds != vds_off) { - fprintf(stderr, "3D dataset test doesn't support VDS\n"); + HDfprintf(stderr, "3D dataset test doesn't support VDS\n"); TEST_ERROR; } } @@ -544,12 +544,12 @@ state_init(state_t *s, int argc, char **argv) } if ((s->quadrant_dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) { - fprintf(stderr, "H5Pcreate failed\n"); + HDfprintf(stderr, "H5Pcreate failed\n"); TEST_ERROR; } if (H5Pset_chunk(s->quadrant_dcpl, RANK2, half_chunk_dims) < 0) { - fprintf(stderr, "H5Pset_chunk failed\n"); + HDfprintf(stderr, "H5Pset_chunk failed\n"); TEST_ERROR; } @@ -574,22 +574,22 @@ state_init(state_t *s, int argc, char **argv) .count = {1, H5S_UNLIMITED}}; if (!make_quadrant_dataspace(s, ul)) { - fprintf(stderr, "make_quadrant_dataspace failed\n"); + HDfprintf(stderr, "make_quadrant_dataspace failed\n"); TEST_ERROR; } if (!make_quadrant_dataspace(s, ur)) { - fprintf(stderr, "make_quadrant_dataspace failed\n"); + HDfprintf(stderr, "make_quadrant_dataspace failed\n"); TEST_ERROR; } if (!make_quadrant_dataspace(s, bl)) { - fprintf(stderr, "make_quadrant_dataspace failed\n"); + HDfprintf(stderr, "make_quadrant_dataspace failed\n"); TEST_ERROR; } if (!make_quadrant_dataspace(s, br)) { - fprintf(stderr, "make_quadrant_dataspace failed\n"); + HDfprintf(stderr, "make_quadrant_dataspace failed\n"); TEST_ERROR; } @@ -599,50 +599,50 @@ state_init(state_t *s, int argc, char **argv) .count = {1, H5S_UNLIMITED}}; if ((src->space = H5Screate_simple(RANK2, half_chunk_dims, half_max_dims)) < 0) { - fprintf(stderr, "H5Screate_simple failed\n"); + HDfprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } if (H5Sselect_hyperslab(src->space, H5S_SELECT_SET, src->start, src->stride, src->count, src->block) < 0) { - fprintf(stderr, "H5Sselect_hyperslab failed\n"); + HDfprintf(stderr, "H5Sselect_hyperslab failed\n"); TEST_ERROR; } if ((ul->src_space = H5Screate_simple(RANK2, half_chunk_dims, half_max_dims)) < 0) { - fprintf(stderr, "H5Screate_simple failed\n"); + HDfprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } if ((ur->src_space = H5Screate_simple(RANK2, half_chunk_dims, half_max_dims)) < 0) { - fprintf(stderr, "H5Screate_simple failed\n"); + HDfprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } if ((bl->src_space = H5Screate_simple(RANK2, half_chunk_dims, half_max_dims)) < 0) { - fprintf(stderr, "H5Screate_simple failed\n"); + HDfprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } if ((br->src_space = H5Screate_simple(RANK2, half_chunk_dims, half_max_dims)) < 0) { - fprintf(stderr, "H5Screate_simple failed\n"); + HDfprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } } /* space for attributes */ if ((s->one_by_one_sid = H5Screate_simple(1, &dims, &dims)) < 0) { - fprintf(stderr, "H5Screate_simple failed\n"); + HDfprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } if ((s->dataset = HDmalloc(sizeof(hid_t) * s->ndatasets)) == NULL) { - fprintf(stderr, "HDmalloc failed\n"); + HDfprintf(stderr, "HDmalloc failed\n"); TEST_ERROR; } if ((s->sources = HDmalloc(sizeof(*s->sources) * s->ndatasets)) == NULL) { - fprintf(stderr, "HDmalloc failed\n"); + HDfprintf(stderr, "HDmalloc failed\n"); TEST_ERROR; } @@ -658,7 +658,7 @@ state_init(state_t *s, int argc, char **argv) dims3[0] = s->part_chunk; if ((s->memspace = H5Screate_simple(RANK3, dims3, NULL)) < 0) { - fprintf(stderr, "H5Screate_simple failed\n"); + HDfprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } } @@ -679,19 +679,19 @@ state_init(state_t *s, int argc, char **argv) } if ((s->memspace = H5Screate_simple(RANK2, dims2, NULL)) < 0) { - fprintf(stderr, "H5Screate_simple failed\n"); + HDfprintf(stderr, "H5Screate_simple failed\n"); TEST_ERROR; } } /* The default is zero, meaning no skip */ if (s->skip_chunk == 1) { - fprintf(stderr, "can't skip every chunk\n"); + HDfprintf(stderr, "can't skip every chunk\n"); TEST_ERROR; } if (s->over_extend == 0) { - fprintf(stderr, "Extension of the dataset can't be zero\n"); + HDfprintf(stderr, "Extension of the dataset can't be zero\n"); TEST_ERROR; } @@ -714,34 +714,34 @@ state_init(state_t *s, int argc, char **argv) else if (personality != NULL && HDstrcmp(personality, "vfd_swmr_bigset_reader") == 0) s->writer = false; else { - fprintf(stderr, "unknown personality, expected vfd_swmr_bigset_{reader,writer}\n"); + HDfprintf(stderr, "unknown personality, expected vfd_swmr_bigset_{reader,writer}\n"); TEST_ERROR; } if ((s->dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) { - fprintf(stderr, "H5Pcreate failed\n"); + HDfprintf(stderr, "H5Pcreate failed\n"); TEST_ERROR; } if (s->chunk_cache_size) { if (H5Pget_chunk_cache(s->dapl, &rdcc_nslots, &rdcc_nbytes, &rdcc_w0) < 0) { - fprintf(stderr, "H5Pget_chunk_cache failed\n"); + HDfprintf(stderr, "H5Pget_chunk_cache failed\n"); TEST_ERROR; } if (H5Pset_chunk_cache(s->dapl, rdcc_nslots, s->chunk_cache_size, rdcc_w0) < 0) { - fprintf(stderr, "H5Pset_chunk_cache failed\n"); + HDfprintf(stderr, "H5Pset_chunk_cache failed\n"); TEST_ERROR; } } if (s->deflate_level > 9) { - fprintf(stderr, "deflation level must be between 0 and 9\n"); + HDfprintf(stderr, "deflation level must be between 0 and 9\n"); TEST_ERROR; } if (s->vds != vds_off && H5Pset_virtual_view(s->dapl, H5D_VDS_FIRST_MISSING) < 0) { - fprintf(stderr, "H5Pset_virtual_view failed\n"); + HDfprintf(stderr, "H5Pset_virtual_view failed\n"); TEST_ERROR; } @@ -784,7 +784,7 @@ state_destroy(state_t *s) struct timespec start_time, end_time; if (H5Pclose(s->dapl) < 0) { - fprintf(stderr, "H5Pclose failed\n"); + HDfprintf(stderr, "H5Pclose failed\n"); TEST_ERROR; } @@ -794,29 +794,29 @@ state_destroy(state_t *s) if (H5Sclose(ul->src_space) < 0 || H5Sclose(ur->src_space) < 0 || H5Sclose(bl->src_space) < 0 || H5Sclose(br->src_space) < 0) { - fprintf(stderr, "H5Sclose failed\n"); + HDfprintf(stderr, "H5Sclose failed\n"); TEST_ERROR; } if (H5Sclose(ul->space) < 0 || H5Sclose(ur->space) < 0 || H5Sclose(bl->space) < 0 || H5Sclose(br->space) < 0) { - fprintf(stderr, "H5Sclose failed\n"); + HDfprintf(stderr, "H5Sclose failed\n"); TEST_ERROR; } if (H5Pclose(s->quadrant_dcpl) < 0) { - fprintf(stderr, "H5Pclose failed\n"); + HDfprintf(stderr, "H5Pclose failed\n"); TEST_ERROR; } } if (H5Sclose(s->one_by_one_sid) < 0) { - fprintf(stderr, "H5Sclose failed\n"); + HDfprintf(stderr, "H5Sclose failed\n"); TEST_ERROR; } if (H5Sclose(s->memspace) < 0) { - fprintf(stderr, "H5Sclose failed\n"); + HDfprintf(stderr, "H5Sclose failed\n"); TEST_ERROR; } @@ -824,23 +824,25 @@ state_destroy(state_t *s) if (s->writer && s->use_vfd_swmr) { unsigned long j; - if (s->vds != vds_multi) + if (s->vds != vds_multi) { if (H5Fvfd_swmr_end_tick(s->file[0]) < 0) { - fprintf(stderr, "H5Fclose failed\n"); + HDfprintf(stderr, "H5Fclose failed\n"); TEST_ERROR; } - else - for (j = 0; j < NELMTS(s->file); j++) - if (H5Fvfd_swmr_end_tick(s->file[j]) < 0) { - fprintf(stderr, "H5Fclose failed\n"); - TEST_ERROR; - } + } + else { + for (j = 0; j < NELMTS(s->file); j++) + if (H5Fvfd_swmr_end_tick(s->file[j]) < 0) { + HDfprintf(stderr, "H5Fclose failed\n"); + TEST_ERROR; + } + } } /* For checking the time spent in file close. It's for running the writer alone */ if (s->do_perf) { if (HDclock_gettime(CLOCK_MONOTONIC, &start_time) == -1) { - fprintf(stderr, "HDclock_gettime failed"); + HDfprintf(stderr, "HDclock_gettime failed"); TEST_ERROR; } } @@ -854,7 +856,7 @@ state_destroy(state_t *s) continue; if (H5Fclose(fid) < 0) { - fprintf(stderr, "H5Fclose failed\n"); + HDfprintf(stderr, "H5Fclose failed\n"); TEST_ERROR; } } @@ -862,11 +864,11 @@ state_destroy(state_t *s) /* For checking the time spent in file close. It's for running the writer alone */ if (s->do_perf) { if (HDclock_gettime(CLOCK_MONOTONIC, &end_time) == -1) { - fprintf(stderr, "HDclock_gettime failed"); + HDfprintf(stderr, "HDclock_gettime failed"); TEST_ERROR; } - fprintf(stdout, "File close time (for running the writer alone) = %lf\n", + HDfprintf(stdout, "File close time (for running the writer alone) = %lf seconds\n", TIME_PASSED(start_time, end_time)); } @@ -916,36 +918,36 @@ np_init(np_state_t *np, bool writer) /* If the named pipes are present at the start of the test, remove them */ if (HDaccess(np->fifo_writer_to_reader, F_OK) == 0) if (HDremove(np->fifo_writer_to_reader) != 0) { - fprintf(stderr, "HDremove fifo_writer_to_reader failed\n"); + HDfprintf(stderr, "HDremove fifo_writer_to_reader failed\n"); TEST_ERROR; } if (HDaccess(np->fifo_reader_to_writer, F_OK) == 0) if (HDremove(np->fifo_reader_to_writer) != 0) { - fprintf(stderr, "HDremove fifo_reader_to_writer failed\n"); + HDfprintf(stderr, "HDremove fifo_reader_to_writer failed\n"); TEST_ERROR; } /* Writer creates two named pipes(FIFO) */ if (HDmkfifo(np->fifo_writer_to_reader, 0600) < 0) { - fprintf(stderr, "HDmkfifo fifo_writer_to_reader failed\n"); + HDfprintf(stderr, "HDmkfifo fifo_writer_to_reader failed\n"); TEST_ERROR; } if (HDmkfifo(np->fifo_reader_to_writer, 0600) < 0) { - fprintf(stderr, "HDmkfifo fifo_reader_to_writer failed\n"); + HDfprintf(stderr, "HDmkfifo fifo_reader_to_writer failed\n"); TEST_ERROR; } } /* Both the writer and reader open the pipes */ if ((np->fd_writer_to_reader = HDopen(np->fifo_writer_to_reader, O_RDWR)) < 0) { - fprintf(stderr, "HDopen fifo_writer_to_reader failed\n"); + HDfprintf(stderr, "HDopen fifo_writer_to_reader failed\n"); TEST_ERROR; } if ((np->fd_reader_to_writer = HDopen(np->fifo_reader_to_writer, O_RDWR)) < 0) { - fprintf(stderr, "HDopen fifo_reader_to_writer failed\n"); + HDfprintf(stderr, "HDopen fifo_reader_to_writer failed\n"); TEST_ERROR; } @@ -965,24 +967,24 @@ np_close(np_state_t *np, bool writer) { /* Both the writer and reader close the named pipes */ if (HDclose(np->fd_writer_to_reader) < 0) { - fprintf(stderr, "HDclose fd_writer_to_reader failed\n"); + HDfprintf(stderr, "HDclose fd_writer_to_reader failed\n"); TEST_ERROR; } if (HDclose(np->fd_reader_to_writer) < 0) { - fprintf(stderr, "HDclose fd_reader_to_writer failed\n"); + HDfprintf(stderr, "HDclose fd_reader_to_writer failed\n"); TEST_ERROR; } /* Reader finishes last and deletes the named pipes */ if (!writer) { if (HDremove(np->fifo_writer_to_reader) != 0) { - fprintf(stderr, "HDremove fifo_writer_to_reader failed\n"); + HDfprintf(stderr, "HDremove fifo_writer_to_reader failed\n"); TEST_ERROR; } if (HDremove(np->fifo_reader_to_writer) != 0) { - fprintf(stderr, "HDremove fifo_reader_to_writer failed\n"); + HDfprintf(stderr, "HDremove fifo_reader_to_writer failed\n"); TEST_ERROR; } } @@ -999,12 +1001,12 @@ reader_verify(np_state_t np, int verify) int notify; if (HDread(np.fd_writer_to_reader, ¬ify, sizeof(int)) < 0) { - fprintf(stderr, "HDread failed\n"); + HDfprintf(stderr, "HDread failed\n"); TEST_ERROR; } if (notify != verify) { - fprintf(stderr, "expected %d but read %d\n", verify, notify); + HDfprintf(stderr, "expected %d but read %d\n", verify, notify); TEST_ERROR; } @@ -1025,13 +1027,13 @@ notify_and_wait_for_reader(state_t *s, np_state_t *np) /* Get the time when finishing creation */ if (HDclock_gettime(CLOCK_MONOTONIC, &last) < 0) { - fprintf(stderr, "HDclock_gettime failed\n"); + HDfprintf(stderr, "HDclock_gettime failed\n"); TEST_ERROR; } /* Notify the reader of finishing creation by sending the timestamp */ if (HDwrite(np->fd_writer_to_reader, &last, sizeof(last)) < 0) { - fprintf(stderr, "HDwrite failed\n"); + HDfprintf(stderr, "HDwrite failed\n"); TEST_ERROR; } @@ -1049,12 +1051,12 @@ notify_and_wait_for_reader(state_t *s, np_state_t *np) /* Wait until the reader finishes validating creation */ if (HDread(np->fd_reader_to_writer, ¬ify, sizeof(int)) < 0) { - fprintf(stderr, "HDread failed\n"); + HDfprintf(stderr, "HDread failed\n"); TEST_ERROR; } if (notify != np->verify) { - fprintf(stderr, "expected %d but read %d\n", np->verify, notify); + HDfprintf(stderr, "expected %d but read %d\n", np->verify, notify); TEST_ERROR; } @@ -1075,7 +1077,7 @@ reader_check_time_and_notify_writer(np_state_t *np, state_t s) /* Receive the notice of the writer finishing creation (timestamp) */ if (HDread(np->fd_writer_to_reader, &last, sizeof(last)) < 0) { - fprintf(stderr, "HDread failed\n"); + HDfprintf(stderr, "HDread failed\n"); TEST_ERROR; } @@ -1084,12 +1086,12 @@ reader_check_time_and_notify_writer(np_state_t *np, state_t s) * the validation of dataset creation */ if (below_speed_limit(&last, &(s.ival))) { AT(); - fprintf(stderr, "Warning: dataset validation took too long to finish\n"); + HDfprintf(stderr, "Warning: dataset validation took too long to finish\n"); } /* Notify the writer that dataset validation is finished */ if (HDwrite(np->fd_reader_to_writer, &(np->notify), sizeof(int)) < 0) { - fprintf(stderr, "HDwrite failed\n"); + HDfprintf(stderr, "HDwrite failed\n"); TEST_ERROR; } @@ -1107,7 +1109,7 @@ notify_reader(np_state_t *np, unsigned step) /* Get the time */ if (HDclock_gettime(CLOCK_MONOTONIC, &(last->time)) < 0) { - fprintf(stderr, "HDclock_gettime failed\n"); + HDfprintf(stderr, "HDclock_gettime failed\n"); TEST_ERROR; } @@ -1115,7 +1117,7 @@ notify_reader(np_state_t *np, unsigned step) /* Notify the reader by sending the timestamp and the number of chunks written */ if (HDwrite(np->fd_writer_to_reader, last, sizeof(exchange_info_t)) < 0) { - fprintf(stderr, "HDwrite failed"); + HDfprintf(stderr, "HDwrite failed"); TEST_ERROR; } @@ -1142,39 +1144,39 @@ create_extensible_dset(state_t *s, unsigned int which) esnprintf(dname, sizeof(dname), "/dataset-%d", which); if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) { - fprintf(stderr, "H5Pcreate failed\n"); + HDfprintf(stderr, "H5Pcreate failed\n"); TEST_ERROR; } if (s->test_3d) { /* The chunk is L x M x N and grows along the first dimension */ if (H5Pset_chunk(dcpl, RANK3, dims3) < 0) { - fprintf(stderr, "H5Pset_chunk for 3D dataset failed\n"); + HDfprintf(stderr, "H5Pset_chunk for 3D dataset failed\n"); TEST_ERROR; } } else { if (H5Pset_chunk(dcpl, RANK2, s->chunk_dims) < 0) { - fprintf(stderr, "H5Pset_chunk for 2D dataset failed\n"); + HDfprintf(stderr, "H5Pset_chunk for 2D dataset failed\n"); TEST_ERROR; } } /* Never write fill value when new chunks are allocated */ if (H5Pset_fill_time(dcpl, H5D_FILL_TIME_NEVER) < 0) { - fprintf(stderr, "H5Pset_fill_time failed\n"); + HDfprintf(stderr, "H5Pset_fill_time failed\n"); TEST_ERROR; } /* Early space allocation */ if (H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0) { - fprintf(stderr, "H5Pset_alloc_time failed\n"); + HDfprintf(stderr, "H5Pset_alloc_time failed\n"); TEST_ERROR; } /* GZIP compression */ if (s->deflate_level && H5Pset_deflate(dcpl, s->deflate_level) < 0) { - fprintf(stderr, "H5Pset_deflate failed\n"); + HDfprintf(stderr, "H5Pset_deflate failed\n"); TEST_ERROR; } @@ -1188,75 +1190,75 @@ create_extensible_dset(state_t *s, unsigned int which) if ((srcs->ul = H5Dcreate2(s->file[0], ul_dname, s->filetype, ul->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { - fprintf(stderr, "H5Dcreate2 failed\n"); + HDfprintf(stderr, "H5Dcreate2 failed\n"); TEST_ERROR; } if ((srcs->ur = H5Dcreate2(s->file[1], ur_dname, s->filetype, ur->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { - fprintf(stderr, "H5Dcreate2 failed\n"); + HDfprintf(stderr, "H5Dcreate2 failed\n"); TEST_ERROR; } if ((srcs->bl = H5Dcreate2(s->file[2], bl_dname, s->filetype, bl->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { - fprintf(stderr, "H5Dcreate2 failed\n"); + HDfprintf(stderr, "H5Dcreate2 failed\n"); TEST_ERROR; } if ((srcs->br = H5Dcreate2(s->file[3], br_dname, s->filetype, br->src_space, H5P_DEFAULT, s->quadrant_dcpl, s->dapl)) < 0) { - fprintf(stderr, "H5Dcreate2 failed\n"); + HDfprintf(stderr, "H5Dcreate2 failed\n"); TEST_ERROR; } if (H5Pset_virtual(dcpl, ul->space, s->filename[0], ul_dname, src->space) < 0) { - fprintf(stderr, "H5Pset_virtual failed\n"); + HDfprintf(stderr, "H5Pset_virtual failed\n"); TEST_ERROR; } if (H5Pset_virtual(dcpl, ur->space, s->filename[1], ur_dname, src->space) < 0) { - fprintf(stderr, "H5Pset_virtual failed\n"); + HDfprintf(stderr, "H5Pset_virtual failed\n"); TEST_ERROR; } if (H5Pset_virtual(dcpl, bl->space, s->filename[2], bl_dname, src->space) < 0) { - fprintf(stderr, "H5Pset_virtual failed\n"); + HDfprintf(stderr, "H5Pset_virtual failed\n"); TEST_ERROR; } if (H5Pset_virtual(dcpl, br->space, s->filename[3], br_dname, src->space) < 0) { - fprintf(stderr, "H5Pset_virtual failed\n"); + HDfprintf(stderr, "H5Pset_virtual failed\n"); TEST_ERROR; } } if (s->test_3d) { if ((filespace = H5Screate_simple(RANK3, dims3, three_dee_max_dims)) < 0) { - fprintf(stderr, "H5Screate_simple 3D dataspace failed\n"); + HDfprintf(stderr, "H5Screate_simple 3D dataspace failed\n"); TEST_ERROR; } } else { if ((filespace = H5Screate_simple(RANK2, s->chunk_dims, s->expand_2d ? two_dee_max_dims : s->one_dee_max_dims)) < 0) { - fprintf(stderr, "H5Screate_simple 2D dataspace failed\n"); + HDfprintf(stderr, "H5Screate_simple 2D dataspace failed\n"); TEST_ERROR; } } if ((dset_id = H5Dcreate2(s->file[0], dname, s->filetype, filespace, H5P_DEFAULT, dcpl, s->dapl)) < 0) { - fprintf(stderr, "H5Dcreate2 failed\n"); + HDfprintf(stderr, "H5Dcreate2 failed\n"); TEST_ERROR; } if (H5Sclose(filespace) < 0) { - fprintf(stderr, "H5Sclose failed\n"); + HDfprintf(stderr, "H5Sclose failed\n"); TEST_ERROR; } if (H5Pclose(dcpl) < 0) { - fprintf(stderr, "H5Pclose failed\n"); + HDfprintf(stderr, "H5Pclose failed\n"); TEST_ERROR; } @@ -1283,7 +1285,7 @@ close_extensible_dset(state_t *s, unsigned int which) hid_t dset_id = H5I_INVALID_HID; if (which >= s->ndatasets) { - fprintf(stderr, "index is out of range\n"); + HDfprintf(stderr, "index is out of range\n"); TEST_ERROR; } @@ -1292,7 +1294,7 @@ close_extensible_dset(state_t *s, unsigned int which) dset_id = s->dataset[which]; if (H5Dclose(dset_id) < 0) { - fprintf(stderr, "H5Dclose failed\n"); + HDfprintf(stderr, "H5Dclose failed\n"); TEST_ERROR; } @@ -1303,7 +1305,7 @@ close_extensible_dset(state_t *s, unsigned int which) if (H5Dclose(srcs->ul) < 0 || H5Dclose(srcs->ur) < 0 || H5Dclose(srcs->bl) < 0 || H5Dclose(srcs->br) < 0) { - fprintf(stderr, "H5Dclose failed\n"); + HDfprintf(stderr, "H5Dclose failed\n"); TEST_ERROR; } } @@ -1350,62 +1352,62 @@ open_extensible_dset(state_t *s) } if (i == NUM_ATTEMPTS) { - fprintf(stderr, "chunk verification reached the maximal number of attempts\n"); + HDfprintf(stderr, "chunk verification reached the maximal number of attempts\n"); TEST_ERROR; } if ((dtype = H5Dget_type(dset_id)) < 0) { - fprintf(stderr, "H5Dget_type failed\n"); + HDfprintf(stderr, "H5Dget_type failed\n"); TEST_ERROR; } if (H5Tequal(dtype, s->filetype) <= 0) { - fprintf(stderr, "Unexpected data type\n"); + HDfprintf(stderr, "Unexpected data type\n"); TEST_ERROR; } if ((filespace = H5Dget_space(dset_id)) < 0) { - fprintf(stderr, "H5Dget_space failed\n"); + HDfprintf(stderr, "H5Dget_space failed\n"); TEST_ERROR; } if ((rank = H5Sget_simple_extent_ndims(filespace)) < 0) { - fprintf(stderr, "H5Sget_simple_extent_ndims failed\n"); + HDfprintf(stderr, "H5Sget_simple_extent_ndims failed\n"); TEST_ERROR; } if ((s->test_3d && rank != RANK3) || (!s->test_3d && rank != RANK2)) { - fprintf(stderr, "Unexpected data rank: %d\n", rank); + HDfprintf(stderr, "Unexpected data rank: %d\n", rank); TEST_ERROR; } if (s->test_3d) { if (H5Sget_simple_extent_dims(filespace, dims3, maxdims3) < 0) { - fprintf(stderr, "H5Sget_simple_extent_dims failed\n"); + HDfprintf(stderr, "H5Sget_simple_extent_dims failed\n"); TEST_ERROR; } } else { if (H5Sget_simple_extent_dims(filespace, dims2, maxdims2) < 0) { - fprintf(stderr, "H5Sget_simple_extent_dims failed\n"); + HDfprintf(stderr, "H5Sget_simple_extent_dims failed\n"); TEST_ERROR; } } if (H5Sclose(filespace) < 0) { - fprintf(stderr, "H5Sclose failed\n"); + HDfprintf(stderr, "H5Sclose failed\n"); TEST_ERROR; } if (H5Tclose(dtype) < 0) { - fprintf(stderr, "H5Tclose failed\n"); + HDfprintf(stderr, "H5Tclose failed\n"); TEST_ERROR; } if (s->test_3d) { if (maxdims3[0] != three_dee_max_dims[0] || maxdims3[1] != three_dee_max_dims[1] || maxdims3[2] != three_dee_max_dims[2]) { - fprintf(stderr, "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE " x %" PRIuHSIZE, + HDfprintf(stderr, "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE " x %" PRIuHSIZE, maxdims3[0], maxdims3[1], maxdims3[2]); TEST_ERROR; } @@ -1414,14 +1416,14 @@ open_extensible_dset(state_t *s) if (s->expand_2d) { if (maxdims2[0] != two_dee_max_dims[0] || maxdims2[1] != two_dee_max_dims[1] || maxdims2[0] != maxdims2[1]) { - fprintf(stderr, "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE, maxdims2[0], + HDfprintf(stderr, "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE, maxdims2[0], maxdims2[1]); TEST_ERROR; } } else if (maxdims2[0] != s->one_dee_max_dims[0] || maxdims2[1] != s->one_dee_max_dims[1] || dims2[0] != s->chunk_dims[0]) { - fprintf(stderr, + HDfprintf(stderr, "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE " or columns %" PRIuHSIZE, maxdims2[0], maxdims2[1], dims2[1]); @@ -1454,7 +1456,7 @@ create_dsets(state_t s) /* For checking the time spent in dataset creation. It's for running the writer alone */ if (s.do_perf) { if (HDclock_gettime(CLOCK_MONOTONIC, &start_time) == -1) { - fprintf(stderr, "HDclock_gettime failed"); + HDfprintf(stderr, "HDclock_gettime failed"); TEST_ERROR; } } @@ -1464,18 +1466,18 @@ create_dsets(state_t s) */ for (which = 0; which < s.ndatasets; which++) if (!create_extensible_dset(&s, which)) { - fprintf(stderr, "create_extensible_dset failed: number %u\n", which); + HDfprintf(stderr, "create_extensible_dset failed: number %u\n", which); TEST_ERROR; } /* For checking the time spent in dataset creation. It's for running the writer alone */ if (s.do_perf) { if (HDclock_gettime(CLOCK_MONOTONIC, &end_time) == -1) { - fprintf(stderr, "HDclock_gettime failed"); + HDfprintf(stderr, "HDclock_gettime failed"); TEST_ERROR; } - fprintf(stdout, "Dataset creation time (for running the writer alone) = %lf\n", + HDfprintf(stdout, "Dataset creation time (for running the writer alone) = %lf seconds\n", TIME_PASSED(start_time, end_time)); } @@ -1495,7 +1497,7 @@ static bool matset(mat_t *mat, unsigned k, unsigned i, unsigned j, uint32_t v) { if (k >= mat->depth || i >= mat->rows || j >= mat->cols) { - fprintf(stderr, "index out of boundary\n"); + HDfprintf(stderr, "index out of boundary\n"); TEST_ERROR; } @@ -1545,7 +1547,7 @@ newmat(state_t s) } if (mat == NULL) { - fprintf(stderr, "HDmalloc failed\n"); + HDfprintf(stderr, "HDmalloc failed\n"); TEST_ERROR; } @@ -1605,7 +1607,7 @@ set_or_verify_matrix(mat_t *mat, unsigned int which, base_t base, bool do_set) if (do_set) { if (!matset(mat, depth, row, col, v)) { - fprintf(stderr, "data initialization failed\n"); + HDfprintf(stderr, "data initialization failed\n"); ret = false; break; } @@ -1673,7 +1675,7 @@ verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t bas hid_t dset_id; if (which >= s->ndatasets) { - fprintf(stderr, "the dataset order is bigger than the number of datasets"); + HDfprintf(stderr, "the dataset order is bigger than the number of datasets"); TEST_ERROR; } @@ -1687,7 +1689,7 @@ verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t bas count3[0] = s->part_chunk; if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset3, NULL, count3, NULL) < 0) { - fprintf(stderr, "H5Sselect_hyperslab failed\n"); + HDfprintf(stderr, "H5Sselect_hyperslab failed\n"); TEST_ERROR; } } @@ -1709,7 +1711,7 @@ verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, base_t bas } if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset2, NULL, count2, NULL) < 0) { - fprintf(stderr, "H5Sselect_hyperslab failed\n"); + HDfprintf(stderr, "H5Sselect_hyperslab failed\n"); TEST_ERROR; } } @@ -1748,14 +1750,14 @@ repeat_verify_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, bas /* Refresh the dataset and try it again */ if (H5Drefresh(dset_id) < 0) { - fprintf(stderr, "H5Drefresh failed\n"); + HDfprintf(stderr, "H5Drefresh failed\n"); TEST_ERROR; } } } if (i == NUM_ATTEMPTS) { - fprintf(stderr, "chunk verification reached the maximal number of attempts\n"); + HDfprintf(stderr, "chunk verification reached the maximal number of attempts\n"); TEST_ERROR; } @@ -1773,7 +1775,7 @@ init_and_write_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, ba dset_id = s->dataset[which]; if (!init_matrix(mat, which, base)) { - fprintf(stderr, "data initialization failed\n"); + HDfprintf(stderr, "data initialization failed\n"); TEST_ERROR; } @@ -1787,7 +1789,7 @@ init_and_write_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, ba /* The chunk dimensions are L x M x N. It grows along the first dimension */ if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset3, NULL, count3, NULL) < 0) { - fprintf(stderr, "H5Sselect_hyperslab for 2D dataset failed\n"); + HDfprintf(stderr, "H5Sselect_hyperslab for 2D dataset failed\n"); TEST_ERROR; } } @@ -1810,13 +1812,13 @@ init_and_write_chunk(state_t *s, hid_t filespace, mat_t *mat, unsigned which, ba } if (H5Sselect_hyperslab(filespace, H5S_SELECT_SET, offset2, NULL, count2, NULL) < 0) { - fprintf(stderr, "H5Sselect_hyperslab for 2D dataset failed\n"); + HDfprintf(stderr, "H5Sselect_hyperslab for 2D dataset failed\n"); TEST_ERROR; } } if (H5Dwrite(dset_id, H5T_NATIVE_UINT32, s->memspace, filespace, H5P_DEFAULT, mat->elt) < 0) { - fprintf(stderr, "H5Dwrite failed\n"); + HDfprintf(stderr, "H5Dwrite failed\n"); TEST_ERROR; } @@ -1838,22 +1840,22 @@ verify_dset_attribute(hid_t dset_id, unsigned int which, unsigned int step) dbgf(1, "verifying attribute %s on dataset %u equals %u\n", name, which, step); if ((aid = H5Aopen(dset_id, name, H5P_DEFAULT)) < 0) { - fprintf(stderr, "H5Aopen failed\n"); + HDfprintf(stderr, "H5Aopen failed\n"); TEST_ERROR; } if (H5Aread(aid, H5T_NATIVE_UINT, &read_step) < 0) { - fprintf(stderr, "H5Aread failed\n"); + HDfprintf(stderr, "H5Aread failed\n"); TEST_ERROR; } if (H5Aclose(aid) < 0) { - fprintf(stderr, "H5Aclose failed\n"); + HDfprintf(stderr, "H5Aclose failed\n"); TEST_ERROR; } if (read_step != step) { - fprintf(stderr, "expected %u read %u\n", step, read_step); + HDfprintf(stderr, "expected %u read %u\n", step, read_step); TEST_ERROR; } @@ -1879,7 +1881,7 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini int i; if (which >= s->ndatasets) { - fprintf(stderr, "the dataset order is bigger than the number of datasets"); + HDfprintf(stderr, "the dataset order is bigger than the number of datasets"); TEST_ERROR; } @@ -1889,18 +1891,18 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini * (NUM_ATTEMPTS) before reporting it as a failure */ for (i = 0; i < NUM_ATTEMPTS; i++) { if (H5Drefresh(dset_id) < 0) { - fprintf(stderr, "H5Drefresh failed\n"); + HDfprintf(stderr, "H5Drefresh failed\n"); TEST_ERROR; } if ((filespace = H5Dget_space(dset_id)) < 0) { - fprintf(stderr, "H5Dget_space failed\n"); + HDfprintf(stderr, "H5Dget_space failed\n"); TEST_ERROR; } if (s->test_3d) { if (H5Sget_simple_extent_dims(filespace, size3, NULL) < 0) { - fprintf(stderr, "H5Sget_simple_extent_dims failed\n"); + HDfprintf(stderr, "H5Sget_simple_extent_dims failed\n"); TEST_ERROR; } @@ -1912,7 +1914,7 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini } else { if (H5Sget_simple_extent_dims(filespace, size2, NULL) < 0) { - fprintf(stderr, "H5Sget_simple_extent_dims failed\n"); + HDfprintf(stderr, "H5Sget_simple_extent_dims failed\n"); TEST_ERROR; } @@ -1931,7 +1933,7 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini } if (i == NUM_ATTEMPTS) { - fprintf(stderr, "chunk verification reached the maximal number of attempts"); + HDfprintf(stderr, "chunk verification reached the maximal number of attempts"); TEST_ERROR; } @@ -1991,7 +1993,7 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini if (s->test_3d || !s->expand_2d) { if (!repeat_verify_chunk(s, filespace, mat, which, last)) { - fprintf(stderr, "chunk verification failed\n"); + HDfprintf(stderr, "chunk verification failed\n"); TEST_ERROR; } } @@ -2001,7 +2003,7 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini base.depth = 0; for (base.row = 0; base.row <= last.row; base.row += s->chunk_dims[0]) { if (!repeat_verify_chunk(s, filespace, mat, which, base)) { - fprintf(stderr, "chunk verification failed\n"); + HDfprintf(stderr, "chunk verification failed\n"); TEST_ERROR; } } @@ -2012,7 +2014,7 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini base.row = last.row; for (base.col = 0; base.col < last.col; base.col += s->chunk_dims[1]) { if (!repeat_verify_chunk(s, filespace, mat, which, base)) { - fprintf(stderr, "chunk verification failed\n"); + HDfprintf(stderr, "chunk verification failed\n"); TEST_ERROR; } } @@ -2020,7 +2022,7 @@ verify_extensible_dset(state_t *s, unsigned int which, mat_t *mat, unsigned fini if (s->asteps != 0 && step % s->asteps == 0) { if (!verify_dset_attribute(dset_id, which, step)) { - fprintf(stderr, "verify_dset_attribute failed\n"); + HDfprintf(stderr, "verify_dset_attribute failed\n"); TEST_ERROR; } } @@ -2056,7 +2058,7 @@ verify_dsets(state_t s, np_state_t *np, mat_t *mat) * including the number of chunks finished and the timestamp */ if (s.use_named_pipe && HDread(np->fd_writer_to_reader, &last, sizeof(last)) < 0) { - fprintf(stderr, "HDread failed\n"); + HDfprintf(stderr, "HDread failed\n"); TEST_ERROR; } @@ -2065,7 +2067,7 @@ verify_dsets(state_t s, np_state_t *np, mat_t *mat) * to the ones written in this round */ if (!verify_extensible_dset(&s, which, mat, finished_step, last.step)) { - fprintf(stderr, "verify_extensible_dset failed\n"); + HDfprintf(stderr, "verify_extensible_dset failed\n"); TEST_ERROR; } @@ -2078,7 +2080,7 @@ verify_dsets(state_t s, np_state_t *np, mat_t *mat) * the validation of the chunks */ if (s.use_named_pipe && below_speed_limit(&(last.time), &(s.ival))) { AT(); - fprintf(stderr, "verify_extensible_dset took too long to finish\n"); + HDfprintf(stderr, "verify_extensible_dset took too long to finish\n"); } /* For checking the time lapse between the writer's finishing writing a batch of chunks @@ -2086,7 +2088,7 @@ verify_dsets(state_t s, np_state_t *np, mat_t *mat) */ if (s.use_named_pipe && s.do_perf) { if (HDclock_gettime(CLOCK_MONOTONIC, &end_time) == -1) { - fprintf(stderr, "HDclock_gettime failed"); + HDfprintf(stderr, "HDclock_gettime failed"); TEST_ERROR; } @@ -2105,7 +2107,7 @@ verify_dsets(state_t s, np_state_t *np, mat_t *mat) /* Print out the performance information */ if (s.use_named_pipe && s.do_perf && counter) - fprintf(stdout, "Dataset verification: mean time = %lf, max time = %lf, min time = %lf\n", + HDfprintf(stdout, "Dataset verification: mean time = %lf, max time = %lf, min time = %lf\n", total_time / (double)counter, max_time, min_time); return true; @@ -2125,17 +2127,17 @@ add_dset_attribute(const state_t *s, hid_t ds, hid_t sid, unsigned int which, un dbgf(1, "setting attribute %s on dataset %u to %u\n", name, which, step); if ((aid = H5Acreate2(ds, name, s->filetype, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) { - fprintf(stderr, "H5Acreate2 failed\n"); + HDfprintf(stderr, "H5Acreate2 failed\n"); TEST_ERROR; } if (H5Awrite(aid, H5T_NATIVE_UINT, &step) < 0) { - fprintf(stderr, "H5Awrite failed\n"); + HDfprintf(stderr, "H5Awrite failed\n"); TEST_ERROR; } if (H5Aclose(aid) < 0) { - fprintf(stderr, "H5Aclose failed\n"); + HDfprintf(stderr, "H5Aclose failed\n"); TEST_ERROR; } @@ -2164,7 +2166,7 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * dbgf(1, "%s: which %u step %u\n", __func__, which, step); if (which >= s->ndatasets) { - fprintf(stderr, "index is out of range\n"); + HDfprintf(stderr, "index is out of range\n"); TEST_ERROR; } @@ -2172,7 +2174,7 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * if (s->asteps != 0 && step % s->asteps == 0) { if (!add_dset_attribute(s, dset_id, s->one_by_one_sid, which, step)) { - fprintf(stderr, "add_dset_attribute failed\n"); + HDfprintf(stderr, "add_dset_attribute failed\n"); TEST_ERROR; } } @@ -2232,22 +2234,22 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * sources_t *const srcs = &s->sources[which]; if (H5Dset_extent(srcs->ul, half_size) < 0) { - fprintf(stderr, "H5Dset_extent failed\n"); + HDfprintf(stderr, "H5Dset_extent failed\n"); TEST_ERROR; } if (H5Dset_extent(srcs->ur, half_size) < 0) { - fprintf(stderr, "H5Dset_extent failed\n"); + HDfprintf(stderr, "H5Dset_extent failed\n"); TEST_ERROR; } if (H5Dset_extent(srcs->bl, half_size) < 0) { - fprintf(stderr, "H5Dset_extent failed\n"); + HDfprintf(stderr, "H5Dset_extent failed\n"); TEST_ERROR; } if (H5Dset_extent(srcs->br, half_size) < 0) { - fprintf(stderr, "H5Dset_extent failed\n"); + HDfprintf(stderr, "H5Dset_extent failed\n"); TEST_ERROR; } } @@ -2256,7 +2258,7 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * if (step % s->over_extend == 0) { if (s->test_3d) { if (size3[0] <= three_dee_max_dims[0] && H5Dset_extent(dset_id, size3) < 0) { - fprintf(stderr, "H5Dset_extent for 3D dataset failed\n"); + HDfprintf(stderr, "H5Dset_extent for 3D dataset failed\n"); TEST_ERROR; } } @@ -2264,7 +2266,7 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * if ((s->expand_2d && size2[0] <= two_dee_max_dims[0] && size2[0] <= two_dee_max_dims[0]) || (!s->expand_2d && size2[1] <= two_dee_max_dims[1])) { if (H5Dset_extent(dset_id, size2) < 0) { - fprintf(stderr, "H5Dset_extent for 2D dataset failed\n"); + HDfprintf(stderr, "H5Dset_extent for 2D dataset failed\n"); TEST_ERROR; } } @@ -2273,13 +2275,13 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * } if ((filespace = H5Dget_space(dset_id)) < 0) { - fprintf(stderr, "H5Dget_space failed\n"); + HDfprintf(stderr, "H5Dget_space failed\n"); TEST_ERROR; } if (s->test_3d || !s->expand_2d) { if (!init_and_write_chunk(s, filespace, mat, which, last)) { - fprintf(stderr, "init_and_write_chunk failed\n"); + HDfprintf(stderr, "init_and_write_chunk failed\n"); TEST_ERROR; } } @@ -2289,7 +2291,7 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * for (base.row = 0; base.row <= last.row; base.row += s->chunk_dims[0]) { dbgf(1, "writing chunk %" PRIuHSIZE ", %" PRIuHSIZE "\n", base.row, base.col); if (!init_and_write_chunk(s, filespace, mat, which, base)) { - fprintf(stderr, "init_and_write_chunk failed\n"); + HDfprintf(stderr, "init_and_write_chunk failed\n"); TEST_ERROR; } } @@ -2298,14 +2300,14 @@ write_extensible_dset(state_t *s, unsigned int which, unsigned int step, mat_t * for (base.col = 0; base.col < last.col; base.col += s->chunk_dims[1]) { dbgf(1, "writing chunk %" PRIuHSIZE ", %" PRIuHSIZE "\n", base.row, base.col); if (!init_and_write_chunk(s, filespace, mat, which, base)) { - fprintf(stderr, "init_and_write_chunk failed\n"); + HDfprintf(stderr, "init_and_write_chunk failed\n"); TEST_ERROR; } } } if (H5Sclose(filespace) < 0) { - fprintf(stderr, "H5Sclose failed\n"); + HDfprintf(stderr, "H5Sclose failed\n"); TEST_ERROR; } @@ -2330,14 +2332,14 @@ write_dsets(state_t s, np_state_t *np, mat_t *mat) struct timespec start_time, end_time; if (NULL == (f = (H5F_t *)H5VL_object(s.file[0]))) { - fprintf(stderr, "H5VL_object failed\n"); + HDfprintf(stderr, "H5VL_object failed\n"); TEST_ERROR; } /* For checking the time spent in writing data. It's for running the writer alone */ if (s.do_perf) { if (HDclock_gettime(CLOCK_MONOTONIC, &start_time) == -1) { - fprintf(stderr, "HDclock_gettime failed"); + HDfprintf(stderr, "HDclock_gettime failed"); TEST_ERROR; } } @@ -2358,7 +2360,7 @@ write_dsets(state_t s, np_state_t *np, mat_t *mat) for (which = 0; which < s.ndatasets; which++) { dbgf(2, "step %d which %d\n", step, which); if (!write_extensible_dset(&s, which, step, mat)) { - fprintf(stderr, "write_extensible_dset failed\n"); + HDfprintf(stderr, "write_extensible_dset failed\n"); TEST_ERROR; } } @@ -2371,7 +2373,7 @@ write_dsets(state_t s, np_state_t *np, mat_t *mat) if (f->shared->tick_num > old_tick_num || step == (total_steps - 1)) { last_step = step + 1; if (s.use_named_pipe && notify_reader(np, last_step) < 0) { - fprintf(stderr, "notify_reader failed\n"); + HDfprintf(stderr, "notify_reader failed\n"); TEST_ERROR; } @@ -2381,13 +2383,25 @@ write_dsets(state_t s, np_state_t *np, mat_t *mat) /* For checking the time spent in writing data. It's for running the writer alone */ if (s.do_perf) { + double throughput; + double time_passed; + if (HDclock_gettime(CLOCK_MONOTONIC, &end_time) == -1) { - fprintf(stderr, "HDclock_gettime failed"); + HDfprintf(stderr, "HDclock_gettime failed"); TEST_ERROR; } - fprintf(stdout, "Dataset write time (for running the writer alone) = %lf\n", - TIME_PASSED(start_time, end_time)); + time_passed = TIME_PASSED(start_time, end_time); + + /* Calculate the write speed */ + if (s.test_3d) + throughput = ((double)(sizeof(unsigned int) * s.depth * s.rows * s.cols * s.nsteps * s.ndatasets)) / time_passed; + else + throughput = ((double)(sizeof(unsigned int) * s.rows * s.cols * s.nsteps * s.ndatasets)) / time_passed; + + /* Print out the performance information */ + HDfprintf(stdout, "Dataset write time (for running the writer alone) = %lf seconds, write speed = %.2lf bytes/second\n", + time_passed, throughput); } return true; @@ -2407,18 +2421,18 @@ main(int argc, char **argv) size_t i; if (!state_init(&s, argc, argv)) { - fprintf(stderr, "state_init failed\n"); + HDfprintf(stderr, "state_init failed\n"); TEST_ERROR; } if ((mat = newmat(s)) == NULL) { - fprintf(stderr, "could not allocate matrix\n"); + HDfprintf(stderr, "could not allocate matrix\n"); TEST_ERROR; } /* Set fs_strategy (file space strategy) and fs_page_size (file space page size) */ if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, s.fsp_size)) < 0) { - fprintf(stderr, "vfd_swmr_create_fcpl failed\n"); + HDfprintf(stderr, "vfd_swmr_create_fcpl failed\n"); TEST_ERROR; } @@ -2438,7 +2452,7 @@ main(int argc, char **argv) /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, s.page_buf_size, &config)) < 0) { - fprintf(stderr, "vfd_swmr_create_fapl failed"); + HDfprintf(stderr, "vfd_swmr_create_fapl failed"); TEST_ERROR; } @@ -2449,7 +2463,7 @@ main(int argc, char **argv) mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION; if (H5Pget_mdc_config(fapl, &mdc_config) < 0) { - fprintf(stderr, "H5Pget_mdc_config failed"); + HDfprintf(stderr, "H5Pget_mdc_config failed"); TEST_ERROR; } @@ -2458,7 +2472,7 @@ main(int argc, char **argv) mdc_config.initial_size = s.mdc_init_size * 1024 * 1024; if (H5Pset_mdc_config(fapl, &mdc_config) < 0) { - fprintf(stderr, "H5Pset_mdc_config failed"); + HDfprintf(stderr, "H5Pset_mdc_config failed"); TEST_ERROR; } } @@ -2467,19 +2481,19 @@ main(int argc, char **argv) : H5Fopen(s.filename[i], H5F_ACC_RDONLY, fapl); if (s.file[i] == badhid) { - fprintf(stderr, s.writer ? "H5Fcreate failed" : "H5Fopen failed"); + HDfprintf(stderr, s.writer ? "H5Fcreate failed" : "H5Fopen failed"); TEST_ERROR; } if (H5Pclose(fapl) < 0) { - fprintf(stderr, "H5Pclose failed\n"); + HDfprintf(stderr, "H5Pclose failed\n"); TEST_ERROR; } } /* Initiailze named pipes */ if (s.use_named_pipe && !np_init(&np, s.writer)) { - fprintf(stderr, "np_init() failed\n"); + HDfprintf(stderr, "np_init() failed\n"); TEST_ERROR; } @@ -2487,13 +2501,13 @@ main(int argc, char **argv) /* Writer tells reader to start */ np.notify = 1; if (s.use_named_pipe && HDwrite(np.fd_writer_to_reader, &np.notify, sizeof(int)) < 0) { - fprintf(stderr, "HDwrite failed\n"); + HDfprintf(stderr, "HDwrite failed\n"); TEST_ERROR; } /* Creates multiple datasets */ if (!create_dsets(s)) { - fprintf(stderr, "create_dsets failed"); + HDfprintf(stderr, "create_dsets failed"); TEST_ERROR; } @@ -2503,14 +2517,14 @@ main(int argc, char **argv) if (s.vds != vds_multi) { if (H5Fvfd_swmr_end_tick(s.file[0]) < 0) { - fprintf(stderr, "H5Fvfd_swmr_end_tick failed\n"); + HDfprintf(stderr, "H5Fvfd_swmr_end_tick failed\n"); TEST_ERROR; } } else { for (j = 0; j < NELMTS(s.file); j++) if (H5Fvfd_swmr_end_tick(s.file[j]) < 0) { - fprintf(stderr, "H5Fvfd_swmr_end_tick failed\n"); + HDfprintf(stderr, "H5Fvfd_swmr_end_tick failed\n"); TEST_ERROR; } } @@ -2520,7 +2534,7 @@ main(int argc, char **argv) * and wait for the reader to finish validation before proceeding */ np.verify = 2; if (s.use_named_pipe && notify_and_wait_for_reader(&s, &np) < 0) { - fprintf(stderr, "notify_and_wait_for_reader failed\n"); + HDfprintf(stderr, "notify_and_wait_for_reader failed\n"); TEST_ERROR; } @@ -2535,7 +2549,7 @@ main(int argc, char **argv) /* Wait for the writer's notice before starting the validation of dataset creation */ np.verify = 1; if (s.use_named_pipe && reader_verify(np, np.verify) < 0) { - fprintf(stderr, "reader_verify failed\n"); + HDfprintf(stderr, "reader_verify failed\n"); TEST_ERROR; } @@ -2543,7 +2557,7 @@ main(int argc, char **argv) * the writer during this step. */ if (!open_extensible_dset(&s)) { - fprintf(stderr, "open_extensible_dset failed\n"); + HDfprintf(stderr, "open_extensible_dset failed\n"); TEST_ERROR; } @@ -2553,7 +2567,7 @@ main(int argc, char **argv) * the validation of dataset creation */ np.notify = 2; if (s.use_named_pipe && reader_check_time_and_notify_writer(&np, s) < 0) { - fprintf(stderr, "reader_check_time_and_notify_writer failed\n"); + HDfprintf(stderr, "reader_check_time_and_notify_writer failed\n"); TEST_ERROR; } @@ -2561,29 +2575,29 @@ main(int argc, char **argv) * Both the reader and writer finish by themselves. */ if (!verify_dsets(s, &np, mat)) { - fprintf(stderr, "verify_dsets failed\n"); + HDfprintf(stderr, "verify_dsets failed\n"); TEST_ERROR; } } for (which = 0; which < s.ndatasets; which++) if (!close_extensible_dset(&s, which)) { - fprintf(stderr, "close_extensible_dset failed\n"); + HDfprintf(stderr, "close_extensible_dset failed\n"); TEST_ERROR; } if (H5Pclose(fcpl) < 0) { - fprintf(stderr, "H5Pclose failed\n"); + HDfprintf(stderr, "H5Pclose failed\n"); TEST_ERROR; } if (s.use_named_pipe && !np_close(&np, s.writer)) { - fprintf(stderr, "np_close() failed\n"); + HDfprintf(stderr, "np_close() failed\n"); TEST_ERROR; } if (!state_destroy(&s)) { - fprintf(stderr, "state_destroy failed\n"); + HDfprintf(stderr, "state_destroy failed\n"); TEST_ERROR; } -- cgit v0.12 From 03f90ee196d02ef45e3a570e483d65c407133bd4 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 14 Sep 2021 16:49:15 +0000 Subject: Committing clang-format changes --- test/vfd_swmr_bigset_writer.c | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c index 22ec3fb..956973a 100644 --- a/test/vfd_swmr_bigset_writer.c +++ b/test/vfd_swmr_bigset_writer.c @@ -829,7 +829,7 @@ state_destroy(state_t *s) HDfprintf(stderr, "H5Fclose failed\n"); TEST_ERROR; } - } + } else { for (j = 0; j < NELMTS(s->file); j++) if (H5Fvfd_swmr_end_tick(s->file[j]) < 0) { @@ -869,7 +869,7 @@ state_destroy(state_t *s) } HDfprintf(stdout, "File close time (for running the writer alone) = %lf seconds\n", - TIME_PASSED(start_time, end_time)); + TIME_PASSED(start_time, end_time)); } if (s->dataset) @@ -1407,8 +1407,9 @@ open_extensible_dset(state_t *s) if (s->test_3d) { if (maxdims3[0] != three_dee_max_dims[0] || maxdims3[1] != three_dee_max_dims[1] || maxdims3[2] != three_dee_max_dims[2]) { - HDfprintf(stderr, "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE " x %" PRIuHSIZE, - maxdims3[0], maxdims3[1], maxdims3[2]); + HDfprintf(stderr, + "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE " x %" PRIuHSIZE, + maxdims3[0], maxdims3[1], maxdims3[2]); TEST_ERROR; } } @@ -1416,17 +1417,17 @@ open_extensible_dset(state_t *s) if (s->expand_2d) { if (maxdims2[0] != two_dee_max_dims[0] || maxdims2[1] != two_dee_max_dims[1] || maxdims2[0] != maxdims2[1]) { - HDfprintf(stderr, "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE, maxdims2[0], - maxdims2[1]); + HDfprintf(stderr, "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE, + maxdims2[0], maxdims2[1]); TEST_ERROR; } } else if (maxdims2[0] != s->one_dee_max_dims[0] || maxdims2[1] != s->one_dee_max_dims[1] || dims2[0] != s->chunk_dims[0]) { HDfprintf(stderr, - "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE - " or columns %" PRIuHSIZE, - maxdims2[0], maxdims2[1], dims2[1]); + "Unexpected maximum dimensions %" PRIuHSIZE " x %" PRIuHSIZE + " or columns %" PRIuHSIZE, + maxdims2[0], maxdims2[1], dims2[1]); } } @@ -1478,7 +1479,7 @@ create_dsets(state_t s) } HDfprintf(stdout, "Dataset creation time (for running the writer alone) = %lf seconds\n", - TIME_PASSED(start_time, end_time)); + TIME_PASSED(start_time, end_time)); } return true; @@ -2108,7 +2109,7 @@ verify_dsets(state_t s, np_state_t *np, mat_t *mat) /* Print out the performance information */ if (s.use_named_pipe && s.do_perf && counter) HDfprintf(stdout, "Dataset verification: mean time = %lf, max time = %lf, min time = %lf\n", - total_time / (double)counter, max_time, min_time); + total_time / (double)counter, max_time, min_time); return true; @@ -2395,13 +2396,18 @@ write_dsets(state_t s, np_state_t *np, mat_t *mat) /* Calculate the write speed */ if (s.test_3d) - throughput = ((double)(sizeof(unsigned int) * s.depth * s.rows * s.cols * s.nsteps * s.ndatasets)) / time_passed; + throughput = + ((double)(sizeof(unsigned int) * s.depth * s.rows * s.cols * s.nsteps * s.ndatasets)) / + time_passed; else - throughput = ((double)(sizeof(unsigned int) * s.rows * s.cols * s.nsteps * s.ndatasets)) / time_passed; + throughput = + ((double)(sizeof(unsigned int) * s.rows * s.cols * s.nsteps * s.ndatasets)) / time_passed; /* Print out the performance information */ - HDfprintf(stdout, "Dataset write time (for running the writer alone) = %lf seconds, write speed = %.2lf bytes/second\n", - time_passed, throughput); + HDfprintf(stdout, + "Dataset write time (for running the writer alone) = %lf seconds, write speed = %.2lf " + "bytes/second\n", + time_passed, throughput); } return true; -- cgit v0.12