diff options
| -rw-r--r-- | test/Makefile.am | 4 | ||||
| -rw-r--r-- | test/genall5.c | 17 | ||||
| -rw-r--r-- | test/testvfdswmr.sh.in | 230 | ||||
| -rw-r--r-- | test/vfd_swmr.c | 9 | ||||
| -rw-r--r-- | test/vfd_swmr_attrdset_writer.c | 2017 | ||||
| -rw-r--r-- | test/vfd_swmr_common.c | 13 | ||||
| -rw-r--r-- | test/vfd_swmr_common.h | 3 | ||||
| -rw-r--r-- | test/vfd_swmr_group_writer.c | 2854 | ||||
| -rw-r--r-- | test/vfd_swmr_zoo_writer.c | 731 |
9 files changed, 5450 insertions, 428 deletions
diff --git a/test/Makefile.am b/test/Makefile.am index 092645f..d6e030b 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -107,6 +107,7 @@ check_PROGRAMS=$(TEST_PROG) error_test err_compat tcheck_version \ vfd_swmr_group_reader vfd_swmr_group_writer \ vfd_swmr_vlstr_reader vfd_swmr_vlstr_writer \ vfd_swmr_zoo_reader vfd_swmr_zoo_writer \ + vfd_swmr_attrdset_reader vfd_swmr_attrdset_writer \ swmr_check_compat_vfd vds_env vds_swmr_gen vds_swmr_reader vds_swmr_writer \ mirror_vfd if HAVE_SHARED_CONDITIONAL @@ -173,6 +174,9 @@ vfd_swmr_zoo_reader_SOURCES=vfd_swmr_zoo_writer.c genall5.c vfd_swmr_bigset_reader_SOURCES=vfd_swmr_bigset_writer.c vfd_swmr_group_reader_SOURCES=vfd_swmr_group_writer.c +vfd_swmr_attrdset_writer_SOURCES=vfd_swmr_attrdset_writer.c +vfd_swmr_attrdset_reader_SOURCES=vfd_swmr_attrdset_writer.c + # Additional target for running timing test timings _timings: testmeta @for timing in $(TIMINGS) dummy; do \ diff --git a/test/genall5.c b/test/genall5.c index 39e25ac..70190c6 100644 --- a/test/genall5.c +++ b/test/genall5.c @@ -19,6 +19,7 @@ * of the same name. */ +#include <err.h> #include "cache_common.h" #include "vfd_swmr_common.h" /* for below_speed_limit() */ #include "genall5.h" @@ -412,7 +413,7 @@ vrfy_ns_grp_c(hid_t fid, const char *group_name, unsigned nlinks) gid = H5Gopen2(fid, group_name, H5P_DEFAULT); if (gid <= 0) { - failure_mssg = "vrfy_ns_grp_c: H5Gopen2() failed"; + failure_mssg = "vrfy_ns_grp_c: H5Gopen2 failed"; return false; } @@ -2753,12 +2754,14 @@ tend_zoo(hid_t fid, const char *base_path, struct timespec *lastmsgtime, zoo_con } out: if (!ok) { - if (HDstrcmp(failure_mssg, last_failure_mssg) != 0) - *lastmsgtime = (struct timespec){.tv_sec = 0, .tv_nsec = 0}; - - if (below_speed_limit(lastmsgtime, &config.msgival)) { - last_failure_mssg = failure_mssg; - HDfprintf(stderr, "%s: %s", __func__, failure_mssg); + /* Currently not used: this step makes sure the operation doesn't take too long. + * Any test that sets config.msgival or lastmsgtime to 0 will skip this step */ + if (strcmp(failure_mssg, last_failure_mssg) != 0 && ((config.msgival.tv_sec || config.msgival.tv_nsec)) + && (lastmsgtime->tv_sec || lastmsgtime->tv_nsec)) { + if (below_speed_limit(lastmsgtime, &config.msgival)) { + last_failure_mssg = failure_mssg; + warnx("%s: %s", __func__, failure_mssg); + } } } return ok; diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in index e8ead12..9c0f272 100644 --- a/test/testvfdswmr.sh.in +++ b/test/testvfdswmr.sh.in @@ -1,4 +1,4 @@ -#! /bin/bash +#!/usr/bin/env bash # # Copyright by The HDF Group. # Copyright by the Board of Trustees of the University of Illinois. @@ -46,7 +46,7 @@ nsofterrors=0 # soft errors are expected to occur some of the time ## 1: Default run. ## 2+: Quick run ############################################################################### -if [ -z $HDF5TestExpress ]; then # Set to default when not set +if [[ -z $HDF5TestExpress ]]; then # Set to default when not set HDF5TestExpress=1 fi ## @@ -54,17 +54,20 @@ fi BIGSET_n=25 # -n option: # of iterations BIGSET_few_s=20 # -s option: # of datasets (for few_big test) BIGSET_many_s=500 # -s option: # of datasets (for many_small test) -GROUP_n=100 # -n option: # of groups (for vfd_swmr_group_writer.c) +GROUP_n=40 # -n option: # of groups (for group test) +GROUP_attr_n=1 # -n option: # of groups (for group attribute test) + if [[ "$HDF5TestExpress" -eq 0 ]] ; then # Setting for exhaustive run BIGSET_n=50 BIGSET_few_s=40 BIGSET_many_s=1000 GROUP_n=400 + GROUP_attr_n=4 elif [[ "$HDF5TestExpress" -gt 1 ]]; then # Setting for quick run BIGSET_n=10 BIGSET_few_s=10 BIGSET_many_s=100 - GROUP_n=40 + GROUP_n=20 fi ############################################################################### @@ -74,6 +77,12 @@ WRITER_MESSAGE=VFD_SWMR_WRITER_MESSAGE # The message file created by writer tha # This should be the same as the define in "./swmr_common.h" MESSAGE_TIMEOUT=300 # Message timeout length in secs # This should be the same as the define in "./h5test.h" + +############################################################################### +## For attrdset test: definitions for fifo files to coordinate test runs +############################################################################### +FIFO_WRITER_TO_READER=fifo_attrdset_writer_to_reader +FIFO_READER_TO_WRITER=fifo_attrdset_reader_to_writer ############################################################################### ## short hands and function definitions @@ -146,8 +155,9 @@ if [ $rc -ne 0 ] ; then exit 0 fi +#all_tests="generator expand shrink expand_shrink sparse vlstr_null vlstr_oob zoo groups attrdset" all_tests="generator expand shrink expand_shrink sparse vlstr_null vlstr_oob zoo groups" -all_tests="${all_tests} few_big many_small" +all_tests="${all_tests} groups_attrs os_groups_attrs few_big many_small" tests=${all_tests} if [ $# -gt 0 ]; then @@ -191,6 +201,7 @@ mkdir vfd_swmr_test cd vfd_swmr_test + # Loop over index types for index_type in "-i ea" "-i b2" do @@ -575,16 +586,16 @@ done # read and written by VFD SWMR. # if [ ${do_zoo:-no} = yes ]; then - [ -e ./fifo ] && rm -f ./fifo - mkfifo -m 0600 ./fifo rm -f ./shared_tick_num echo launch vfd_swmr_zoo_writer - STDIN_PATH="./fifo" catch_out_err_and_rc vfd_swmr_zoo_writer \ - ../vfd_swmr_zoo_writer -m 1000 -q & + catch_out_err_and_rc vfd_swmr_zoo_writer \ + ../vfd_swmr_zoo_writer -q & pid_writer=$! - STDOUT_PATH="./fifo" catch_out_err_and_rc vfd_swmr_zoo_reader \ - ../vfd_swmr_zoo_reader -q -W & + # -l is the expected maximal number of ticks from the writer's finishing zoo creation or deletion + # to the reader's finishing validation of zoo creation or deletion + catch_out_err_and_rc vfd_swmr_zoo_reader \ + ../vfd_swmr_zoo_reader -l 4 -q & pid_reader=$! # Wait for the reader to finish before signalling the @@ -592,7 +603,6 @@ if [ ${do_zoo:-no} = yes ]; then # reader will find the shadow file when it opens # the .h5 file. wait $pid_reader - kill -USR1 $(cat vfd_swmr_zoo_writer.pid) wait $pid_writer # Collect exit code of the reader @@ -608,13 +618,61 @@ if [ ${do_zoo:-no} = yes ]; then fi # Clean up output files - rm -f ./fifo rm -f vfd_swmr_zoo_writer.{out,rc} rm -f vfd_swmr_zoo_reader.*.{out,rc} fi +# attrdset test +#for options in "-p -g -a 10 -v -m -d 10 -c 3 -u 5" "-k -a 20 -v -m -d 5"; do +# # +# # Test a few big datasets of one and two dimensions. +# # +# if [ ${do_attrdset:-no} = no ]; then +# continue +# fi +# # Clean up any existing fifo files from previous runs +# if [ -e ./$FIFO_WRITER_TO_READER ]; then # If writer fifo file is found +# rm -f ./$FIFO_WRITER_TO_READER +# fi +# if [ -e ./$FIFO_READER_TO_WRITER ]; then # If reader fifo file is found +# rm -f ./$FIFO_READER_TO_WRITER +# fi +# # +# echo launch vfd_swmr_attrdset_writer attrdset, options $options +# catch_out_err_and_rc vfd_swmr_attrdset_writer \ +# ../vfd_swmr_attrdset_writer $options & +# pid_writer=$! +# +# catch_out_err_and_rc vfd_swmr_attrdset_reader \ +# ../vfd_swmr_attrdset_reader $options & +# pid_reader=$! +# +# # Wait for the reader to finish before signaling the +# # writer to quit: the writer holds the file open so that the +# # reader will find the shadow file when it opens +# # the .h5 file. +# wait $pid_reader +# wait $pid_writer # -# Make sure that we can create GROUP_n groups (40, 100, or 400 depending on the HDF5TestExpress level) +# # Collect exit code of the reader +# if [ $(cat vfd_swmr_attrdset_reader.rc) -ne 0 ]; then +# echo reader had error +# nerrors=$((nerrors + 1)) +# fi +# +# # Collect exit code of the writer +# if [ $(cat vfd_swmr_attrdset_writer.rc) -ne 0 ]; then +# echo writer had error +# nerrors=$((nerrors + 1)) +# fi +# +# # Clean up output files +# rm -f vfd_swmr_attrdset_writer.{out,rc} +# rm -f vfd_swmr_attrdset_reader.*.{out,rc} +#done + +# +# Make sure that we can create GROUP_n groups (20, 40, or 400 depending on the HDF5TestExpress level) # while a reader waits for each to appear. # if [ ${do_groups:-no} = yes ]; then @@ -651,6 +709,150 @@ if [ ${do_groups:-no} = yes ]; then rm -f vfd_swmr_group_reader.*.{out,rc} fi +# The group attribute test takes longer. +# So for standard run and quick run, we +# shorten the number of tests. The standard +# run covers all the features we need to +# test. The quick run doesn't cover the +# attribute storage change between dense and +# compact. +# The exhaustive run tries to test a feature +# per test from scratch. +# +grp_attr_list=( + "compact" + "dense" + "compact-del" + "dense-del" + "compact-add-to-dense" + "dense-del-to-compact" + "modify" + "add-vstr" + "remove-vstr" + "modify-vstr" + ) +grp_sub_attr_list=( + "dense-del-to-compact" + "modify" + "remove-vstr" + "modify-vstr" + ) + +grp_short_sub_attr_list=( + "dense" + "modify" + "remove-vstr" + "modify-vstr" + ) + +if [[ "$HDF5TestExpress" -eq 1 ]] ; then #Setting for standard run + grp_attr_list=("${grp_sub_attr_list[@]}") +elif [[ "$HDF5TestExpress" -gt 1 ]] ; then #Setting for quick run + grp_attr_list=("${grp_short_sub_attr_list[@]}") +fi + +for options in ${grp_attr_list[*]}; do + if [ ${do_groups_attrs:-no} = no ]; then + continue + fi + echo launch vfd_swmr_group attribute: $options + catch_out_err_and_rc vfd_swmr_group_writer \ + ../vfd_swmr_group_writer -q -c 1 -n $GROUP_attr_n -a 1 -A $options & + pid_writer=$! + + catch_out_err_and_rc vfd_swmr_group_reader \ + ../vfd_swmr_group_reader -q -c 1 -n $GROUP_attr_n -a 1 -A $options & + pid_reader=$! + + # Wait for the reader to finish before signalling the + # writer to quit: the writer holds the file open so that the + # reader will find the shadow file when it opens + # the .h5 file. + wait $pid_reader + wait $pid_writer + + # Collect exit code of the reader + if [ $(cat vfd_swmr_group_reader.rc) -ne 0 ]; then + echo reader had error + nerrors=$((nerrors + 1)) + fi + + # Collect exit code of the writer + if [ $(cat vfd_swmr_group_writer.rc) -ne 0 ]; then + echo writer had error + nerrors=$((nerrors + 1)) + fi + + # Clean up output files + rm -f vfd_swmr_group_writer.{out,rc} + rm -f vfd_swmr_group_reader.*.{out,rc} +done + +# The following tests are for add/del/modify attributes for +# groups created with the old-style. +# Check https://portal.hdfgroup.org/display/HDF5/Groups for +# the detailed group implementation note. +# The 'compact' and 'compact-del' are the attribute addition +# and deletion tests. Other test names have the same meaning +# as those of the new-style group tests. +# +os_grp_attr_list=( + "compact" + "compact-del" + "modify" + "add-vstr" + "remove-vstr" + "modify-vstr" + ) +os_grp_sub_attr_list=( + "modify" + "remove-vstr" + "modify-vstr" + ) +if [[ "$HDF5TestExpress" -gt 0 ]] ; then #Setting for standard run + os_grp_attr_list=("${os_grp_sub_attr_list[@]}") +fi + +for options in ${os_grp_attr_list[*]}; do + if [ ${do_os_groups_attrs:-no} = no ]; then + continue + fi + echo launch vfd_swmr_group attribute with old-style group: $options + catch_out_err_and_rc vfd_swmr_group_writer \ + ../vfd_swmr_group_writer -q -G -c 1 -n $GROUP_attr_n -a 1 -A $options & + pid_writer=$! + + catch_out_err_and_rc vfd_swmr_group_reader \ + ../vfd_swmr_group_reader -q -G -c 1 -n $GROUP_attr_n -a 1 -A $options & + pid_reader=$! + + # Wait for the reader to finish before signalling the + # writer to quit: the writer holds the file open so that the + # reader will find the shadow file when it opens + # the .h5 file. + wait $pid_reader + wait $pid_writer + + # Collect exit code of the reader + if [ $(cat vfd_swmr_group_reader.rc) -ne 0 ]; then + echo reader had error + nerrors=$((nerrors + 1)) + fi + + # Collect exit code of the writer + if [ $(cat vfd_swmr_group_writer.rc) -ne 0 ]; then + echo writer had error + nerrors=$((nerrors + 1)) + fi + + # Clean up output files + rm -f vfd_swmr_group_writer.{out,rc} + rm -f vfd_swmr_group_reader.*.{out,rc} +done + + + + for options in "-d 1" "-d 1 -F" "-d 2" "-d 2 -F" "-d 1 -V" "-d 1 -M" "-d 1 -V -F" "-d 1 -M -F"; do if [ ${do_many_small:-no} = no ]; then continue diff --git a/test/vfd_swmr.c b/test/vfd_swmr.c index 304b12a..09dee38 100644 --- a/test/vfd_swmr.c +++ b/test/vfd_swmr.c @@ -848,15 +848,6 @@ error: return 1; } /* test_writer_create_open_flush() */ -/* Sleep for `tenths` tenths of a second */ -static void -decisleep(uint32_t tenths) -{ - uint64_t nsec = tenths * 100 * 1000 * 1000; - - H5_nanosleep(nsec); -} - /*------------------------------------------------------------------------- * Function: test_writer_md() * diff --git a/test/vfd_swmr_attrdset_writer.c b/test/vfd_swmr_attrdset_writer.c new file mode 100644 index 0000000..0871898 --- /dev/null +++ b/test/vfd_swmr_attrdset_writer.c @@ -0,0 +1,2017 @@ +/* + * Copyright by The HDF Group. + * Copyright by the Board of Trustees of the University of Illinois. + * All rights reserved. + * + * This file is part of HDF5. The full HDF5 copyright notice, including + * terms governing use, modification, and redistribution, is contained in + * the COPYING file, which can be found at the root of the source code + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. + * If you do not have access to either file, you may request a copy from + * help@hdfgroup.org. + */ + +/* + * Purpose: To test attribute handling for different dataset types. + * Dataset types: + * --dataset with compact layout + * --dataset with contiguous layout + * --dataset with chunked layout: + * 1. single indexing type + * 2. implicit indexing type + * 3. fixed array indexing type + * 4. extensible array indexing type + * 5. version btree 2 indexing type + * Attribute handling: + * -- Add attribute + * -- Delete attribute + * -- Modify attribute + * -- Add variable length attribute + * -- Delete variable length attribute + * -- Modify variable length attribute + * -- Add sufficient attributes to force creation of object header continuation block + * -- Remove sufficient attributes to allow deletion of object header continuation block + * -- Transition from compact to dense attribute storage + * -- Transition from dense to compact attribute storage + * + * Please see verify_storage_cont() on verification of + * compact<->dense storage and with/without continuation block. + * + */ +#include <err.h> +#include <libgen.h> +#include <unistd.h> /* getopt(3) */ + +#include "hdf5.h" +#include "testhdf5.h" +#include "vfd_swmr_common.h" + +#ifndef H5_HAVE_WIN32_API + +#define READER_WAIT_TICKS 4 + +/* Structure to hold info for options specified */ +typedef struct { + hid_t file; /* File ID */ + hid_t filetype; /* ID for default datatype */ + hid_t one_by_one_sid; /* ID for default dataspace */ + char filename[PATH_MAX]; /* File name */ + char progname[PATH_MAX]; /* Program name */ + unsigned int update_interval; /* For -u option */ + unsigned int asteps; /* For -a <nattrs> option */ + unsigned int csteps; /* For -c <csteps> option */ + unsigned int dattrs; /* For -d <dattrs> option */ + bool compact; /* For -p option */ + bool contig; /* For -g option */ + bool chunked; /* For -k option */ + bool vl_attr; /* For -v option */ + bool mod_attr; /* For -m option */ + bool use_np; /* For -N option */ + bool use_vfd_swmr; /* For -S option */ +} state_t; + +/* Initializations for state_t */ +#define ALL_HID_INITIALIZER (state_t) { \ + .file = H5I_INVALID_HID \ + , .one_by_one_sid = H5I_INVALID_HID \ + , .filename = "" \ + , .filetype = H5T_NATIVE_UINT32 \ + , .asteps = 0 \ + , .csteps = 1 \ + , .dattrs = 0 \ + , .use_np = true \ + , .use_vfd_swmr = true \ + , .compact = false \ + , .contig = false \ + , .chunked = false \ + , .vl_attr = false \ + , .mod_attr = false \ + , .update_interval = READER_WAIT_TICKS } + +/* Structure to hold info for different dataset types */ +typedef struct { + hid_t compact_did; /* ID for compact dataset */ + hid_t contig_did; /* ID for contiguous dataset */ + hid_t single_did; /* ID for chunked dataset: single index */ + hid_t implicit_did; /* ID for chunked dataset: implicit index */ + hid_t fa_did; /* ID for chunked dataset: fixed array index */ + hid_t ea_did; /* ID for chunked dataset: extensible array index */ + hid_t bt2_did; /* ID for chunked dataset: version 2 btree index */ + unsigned p_max_compact; /* Value of max_compact storage for -p */ + unsigned g_max_compact; /* Value of max_compact storage for -g */ + unsigned single_max_compact; /* Value of max_compact storage for -k: single index */ + unsigned implicit_max_compact; /* Value of max_compact storage for -k: implicit index */ + unsigned fa_max_compact; /* Value of max_compact storage for -k: fixed array index */ + unsigned ea_max_compact; /* Value of max_compact storage for -k: extensible array index */ + unsigned bt2_max_compact; /* Value of max_compact storage for -k: version 2 btree index */ + unsigned p_min_dense; /* Value of min_dense storage for -p */ + unsigned g_min_dense; /* Value of min_dense storage for -g */ + unsigned single_min_dense; /* Value of min_dense storage for -k: single index */ + unsigned implicit_min_dense; /* Value of min_dense storage for -k: implicit index */ + unsigned fa_min_dense; /* Value of min_dense storage for -k: fixed array index */ + unsigned ea_min_dense; /* Value of min_dense storage for -k: extensible array index */ + unsigned bt2_min_dense; /* Value of min_dense storage for -k: version 2 btree index */ +} dsets_state_t; + +/* Initializations for dsets_state_t */ +#define DSETS_INITIALIZER (dsets_state_t) { \ + .compact_did = H5I_INVALID_HID \ + , .contig_did = H5I_INVALID_HID \ + , .single_did = H5I_INVALID_HID \ + , .implicit_did = H5I_INVALID_HID \ + , .fa_did = H5I_INVALID_HID \ + , .ea_did = H5I_INVALID_HID \ + , .bt2_did = H5I_INVALID_HID \ + , .p_max_compact = 0 \ + , .g_max_compact = 0 \ + , .single_max_compact = 0 \ + , .implicit_max_compact = 0 \ + , .fa_max_compact = 0 \ + , .ea_max_compact = 0 \ + , .bt2_max_compact = 0 } + +/* Structure to hold info for named pipes */ +typedef struct { + const char *fifo_writer_to_reader; /* Name of fifo for writer to reader */ + const char *fifo_reader_to_writer; /* Name of fifo for reader to writer */ + int fd_writer_to_reader; /* File ID for fifo from writer to reader */ + int fd_reader_to_writer; /* File ID for fifo from reader to writer */ + int notify; /* Value to notify between writer and reader */ + int verify; /* Value to verify between writer and reader */ +} np_state_t; + +/* Initializations for np_state_t */ +#define NP_INITIALIZER (np_state_t) { \ + .fifo_writer_to_reader = "./fifo_attrdset_writer_to_reader" \ + , .fifo_reader_to_writer = "./fifo_attrdset_reader_to_writer" \ + , .fd_writer_to_reader = -1 \ + , .fd_reader_to_writer = -1 \ + , .notify = 0 \ + , .verify = 0 } + +static bool state_init(state_t *, int, char **); + +static bool np_init(np_state_t *np, bool writer); +static bool np_close(np_state_t *np, bool writer); +static bool np_writer(bool result, unsigned step, const state_t *s, np_state_t *np, H5F_vfd_swmr_config_t *config); +static bool np_reader(bool result, unsigned step, const state_t *s, np_state_t *np); +static bool np_confirm_verify_notify(int fd, unsigned step, const state_t *s, np_state_t *np); +static bool np_reader_no_verification(const state_t *s, np_state_t *np, H5F_vfd_swmr_config_t *config); + +static bool create_dsets(const state_t *s, dsets_state_t *ds); +static bool open_dsets(const state_t *s, dsets_state_t *ds); +static bool open_dset_real(hid_t fid, hid_t *did, const char *name, unsigned *max_compact, unsigned *min_dense); +static bool close_dsets(const dsets_state_t *ds); + +static bool attr_dsets_action(unsigned action, const state_t *s, const dsets_state_t *ds, unsigned which); +static bool attr_action(unsigned action, const state_t *s, hid_t did, unsigned which); +static bool add_attr(const state_t *s, hid_t did, unsigned int which); +static bool modify_attr(const state_t *s, hid_t did, unsigned int which); +static bool delete_attr(hid_t did, unsigned int which); + +static bool verify_attr_dsets_action(unsigned action, const state_t *s, const dsets_state_t *ds, unsigned which); +static bool verify_attr_action(unsigned action, hid_t did, unsigned which); +static bool verify_add_or_modify_attr(unsigned action, hid_t did, char *attr_name, unsigned int which); +static bool verify_delete_attr(hid_t did, char *attr_name); +static bool verify_storage_cont(unsigned action, hid_t did, unsigned int which, unsigned max_compact, unsigned min_dense, unsigned asteps); +static bool verify_storage_cont_real(hid_t did, unsigned int which, unsigned cut_point); + +static const hid_t badhid = H5I_INVALID_HID; + +/* Names for datasets */ +#define DSET_COMPACT_NAME "compact_dset" +#define DSET_CONTIG_NAME "contig_dset" +#define DSET_SINGLE_NAME "chunked_single" +#define DSET_IMPLICIT_NAME "chunked_implicit" +#define DSET_FA_NAME "chunked_fa" +#define DSET_EA_NAME "chunked_ea" +#define DSET_BT2_NAME "chunked_bt2" + +/* Action for attribute handling */ +#define ADD_ATTR 1 +#define MODIFY_ATTR 2 +#define DELETE_ATTR 3 + +/* Test program usage info */ +static void +usage(const char *progname) +{ + fprintf(stderr, "usage: %s -a nattrs [-p] [-g] [-k] [-v] [-m]\n" + " [-d dattrs] [-u nticks] [-c csteps] [-S] [-N]\n" + "\n" + "-p: create a dataset with compact layout\n" + "-g: create a dataset with contiguous layout\n" + "-k: create datasets with chunked layout for the 5 indexing types\n" + "-m: modify attributes to all datasets after addition\n" + "-v: add variable length attribute to datasets\n" + " (default is H5T_NATIVE_UINT32)\n" + "-a nattrs: add `nattrs` attributes to all datasets\n" + "-d dattrs: delete `dattrs` attributes to all datasets after addition\n" + "-u nticks: `nticks` ticks for the reader to wait before verification\n" + " (default is 4)\n" + "-c csteps: `csteps` steps communication interval between reader and writer\n" + " (default is 1)\n" + "-S: do not use VFD SWMR\n" + "-N: do not use named pipes for test synchronization\n" + "-b: write data in big-endian byte order if no -v option\n" + " (default is H5T_NATIVE_UINT32)\n\n" + "Note:\n" + "1. Require to specify at least -p, -g or -k option\n" + "2. -c <csteps> option cannot exceed -a <nattrs> option\n" + "3. -d <dattrs> option cannot exceed -a <nattrs> option\n" + "\n", + progname); + exit(EXIT_FAILURE); +} /* usage() */ + +/* + * Initialize option info in state_t + */ +static bool +state_init(state_t *s, int argc, char **argv) +{ + unsigned long tmp; + int ch; + const hsize_t dims = 1; + char tfile[PATH_MAX]; + char *end; + + *s = ALL_HID_INITIALIZER; + esnprintf(tfile, sizeof(tfile), "%s", argv[0]); + esnprintf(s->progname, sizeof(s->progname), "%s", basename(tfile)); + + while ((ch = getopt(argc, argv, "pgkvmbqSNa:d:u:c:")) != -1) { + switch (ch) { + + case 'p': + s->compact = true; + break; + + case 'g': + s->contig = true; + break; + + case 'k': + s->chunked = true; + break; + + case 'v': + s->vl_attr = true; + break; + + case 'm': + s->mod_attr = true; + break; + case 'b': + s->filetype = H5T_STD_U32BE; + break; + case 'q': + verbosity = 0; + break; + case 'S': + s->use_vfd_swmr = false; + break; + case 'N': + s->use_np = false; + break; + case 'a': + case 'd': + case 'u': + case 'c': + errno = 0; + tmp = strtoul(optarg, &end, 0); + if (end == optarg || *end != '\0') { + printf("couldn't parse `-%c` argument `%s`\n", ch, optarg); + TEST_ERROR; + } else if (errno != 0) { + printf("couldn't parse `-%c` argument `%s`\n", ch, optarg); + TEST_ERROR; + } else if (tmp > UINT_MAX) { + printf("`-%c` argument `%lu` too large\n", ch, tmp); + TEST_ERROR; + } + + if (ch == 'a') + s->asteps = (unsigned)tmp; + else if (ch == 'd') + s->dattrs = (unsigned)tmp; + else if (ch == 'u') + s->update_interval = (unsigned)tmp; + else if (ch == 'c') + s->csteps = (unsigned)tmp; + break; + + case '?': + default: + usage(s->progname); + break; + } + } + argc -= optind; + argv += optind; + + /* Require to specify at least -p, -g or -k option */ + if(!s->compact && !s->contig && !s->chunked) { + printf("Require to specify at least -p, -g or -k option\n"); + usage(s->progname); + goto error; + } + + /* -c <csteps> cannot be zero */ + if(!s->csteps) { + printf("communication interval cannot be zero\n"); + TEST_ERROR; + } + + /* -c <csteps> and -a <nattrs> options */ + if(s->asteps && s->csteps > s->asteps) { + printf("communication interval is out of bounds\n"); + TEST_ERROR; + } + + /* -d and -a */ + if(s->dattrs > s->asteps) { + printf("# of attributes to be deleted exceeds # of attributes created\n"); + TEST_ERROR; + } + + /* Dataspace for attributes added to datasets */ + /* Dataspace for compact and contiguous datasets */ + if ((s->one_by_one_sid = H5Screate_simple(1, &dims, &dims)) < 0) { + printf("H5Screate_simple failed\n"); + TEST_ERROR; + } + + /* The test file name */ + esnprintf(s->filename, sizeof(s->filename), "vfd_swmr_attrdset.h5"); + + return true; + +error: + return false; + +} /* state_init() */ + +/* + * Create the datasets as specified on the command line. + */ +static bool +create_dsets(const state_t *s, dsets_state_t *ds) +{ + hid_t dcpl = badhid; + hid_t dtid = badhid; + hid_t tmp_did = badhid; + hid_t cmpd_tid = badhid; + hid_t array_tid = badhid; + hid_t vl_tid = badhid; + hid_t sid = badhid; + + *ds = DSETS_INITIALIZER; + + /* Dataset with compact layout, compound datatype */ + if(s->compact) { + const hsize_t dims = 2; + typedef struct { + int a; + int b[2]; + } cmpd; + cmpd wdata; /* Data for compact dataset */ + + wdata.a = 1; + wdata.b[0] = 2; + wdata.b[1] = 3; + + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) { + printf("H5Pcreate failed\n"); + TEST_ERROR; + } + if(H5Pset_layout(dcpl, H5D_COMPACT) < 0) { + printf("H5Pset_layout failed\n"); + TEST_ERROR; + } + + /* Create compound datatype */ + if((cmpd_tid = H5Tcreate(H5T_COMPOUND, sizeof(cmpd))) < 0) { + printf("H5Tcreate failed\n"); + TEST_ERROR; + } + + /* Create the array for the second element in the compound type */ + if((array_tid = H5Tarray_create2(H5T_NATIVE_INT, 1, &dims)) < 0) { + printf("H5Tarray_create2 failed\n"); + TEST_ERROR; + } + + /* First element in the compound type */ + if(H5Tinsert(cmpd_tid, "a", HOFFSET(cmpd, a), H5T_NATIVE_INT) < 0) { + printf("H5Tinsert failed\n"); + TEST_ERROR; + } + /* Second element in the compound type */ + if(H5Tinsert(cmpd_tid, "b", HOFFSET(cmpd, b), array_tid) < 0) { + printf("H5Tinsert failed\n"); + TEST_ERROR; + } + + /* Create the compact dataset with compound datatype */ + if((ds->compact_did = H5Dcreate2(s->file, DSET_COMPACT_NAME, cmpd_tid, + s->one_by_one_sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) { + printf("H5Dcreate2 compact dataset failed\n"); + TEST_ERROR; + } + + /* Write data to the dataset */ + if(H5Dwrite(ds->compact_did, cmpd_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, &wdata) < 0) { + printf("H5Dwrite to compact dataset failed\n"); + TEST_ERROR; + } + + /* In order to trigger continuation block if -p is used alone by itself */ + if((tmp_did = H5Dcreate2(s->file, "JUNK_IGNORE", cmpd_tid, + s->one_by_one_sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) { + printf("H5Dcreate2 failed\n"); + TEST_ERROR; + } + if(H5Dclose(tmp_did) < 0) { + printf("H5Dclose failed\n"); + TEST_ERROR; + } + + if(H5Pclose(dcpl) < 0) { + printf("H5Pclose failed\n"); + TEST_ERROR; + } + } + + /* Dataset with contiguous layout, early allocation, non-default attr phase change, named datatype */ + if(s->contig) { + int wdata1 = 9; /* Data for contiguous dataset */ + unsigned def_max_compact = 0; + unsigned def_min_dense = 0; + + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) { + printf("H5Pcreate failed\n"); + TEST_ERROR; + } + + if(H5Pset_layout(dcpl, H5D_CONTIGUOUS) < 0) { + printf("H5Pset_layout failed\n"); + TEST_ERROR; + } + + if(H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0) { + printf("H5Pset_alloc_time failed\n"); + TEST_ERROR; + } + + if(H5Pget_attr_phase_change(dcpl, &def_max_compact, &def_min_dense) < 0) { + printf("H5Pget_attr_phase_change failed\n"); + TEST_ERROR; + } + + if(H5Pset_attr_phase_change(dcpl, def_max_compact+2, def_min_dense+2) < 0) { + printf("H5Pset_attr_phase_change failed\n"); + TEST_ERROR; + } + + /* Create the named datatype */ + if((dtid = H5Tcopy(H5T_NATIVE_INT)) < 0) { + printf("H5Tcopy failed\n"); + TEST_ERROR; + } + + if(H5Tcommit2(s->file, "named_dtype", dtid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) < 0) { + printf("H5Tcommit2 failed\n"); + TEST_ERROR; + } + + /* Create the contiguous dataset with the named datatype */ + if((ds->contig_did = H5Dcreate2(s->file, DSET_CONTIG_NAME, dtid, + s->one_by_one_sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) { + printf("H5Dcreate2 contiguous dataset failed\n"); + TEST_ERROR; + } + + /* Write to the dataset */ + if(H5Dwrite(ds->contig_did, dtid, H5S_ALL, H5S_ALL, H5P_DEFAULT, &wdata1) < 0) { + printf("H5Dwrite to contiguous dataset failed\n"); + TEST_ERROR; + } + + if(H5Pclose(dcpl) < 0) { + printf("H5Pclose failed\n"); + TEST_ERROR; + } + + if(H5Tclose(dtid) < 0) { + printf("H5Tclose failed\n"); + TEST_ERROR; + } + } + + /* Datasets with the 5 indexes: single, implicit, fa, ea, bt2 */ + /* All with variable length datatype */ + if(s->chunked) { + + /* For index: single, implicit and fa */ + hsize_t dims1[1] = {5}; + hsize_t max_dims1[1] = {100}; + hsize_t chunk_dims1[1] = {2}; + + /* The variable length data */ + const char *vdata[5] = {"one", "two", "three", "four", "five" }; + + /* For index: ea and bt2 */ + hsize_t dims2[2] = {5, 5}; + hsize_t max_dims2[2] = {100, H5S_UNLIMITED}; + hsize_t chunk_dims2[2] = {2, 2}; + const char *vdata2[5][5] = { + { "one", "two", "three", "four", "five" }, + { "two", "three", "four", "five", "six" }, + { "three", "four", "five", "six", "seven" }, + { "four", "five", "six", "seven", "eight" }, + { "five", "six", "seven", "eight", "nine" } }; + + /* Create variable length datatype */ + if((vl_tid = H5Tcopy(H5T_C_S1)) < 0) { + printf("H5Tcopy failed\n"); + TEST_ERROR; + } + + if (H5Tset_size(vl_tid, H5T_VARIABLE) < 0) { + printf("H5Tset_size failed\n"); + TEST_ERROR; + } + + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) { + printf("H5Pcreate failed\n"); + TEST_ERROR; + } + + if(H5Pset_layout(dcpl, H5D_CHUNKED) < 0) { + printf("H5Pset_layout failed\n"); + TEST_ERROR; + } + + if(H5Pset_chunk(dcpl, 1, dims1) < 0) { + printf("H5Pset_chunk failed\n"); + TEST_ERROR; + } + + /* Create 1-D chunked dataset with single index */ + /* Chunked, dims=max_dims=chunk_dims */ + + if((sid = H5Screate_simple(1, dims1, dims1)) < 0) { + printf("H5Screate_simple failed\n"); + TEST_ERROR; + } + + if((ds->single_did = H5Dcreate2(s->file, DSET_SINGLE_NAME, vl_tid, + sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) { + printf("H5Dcreate2 chunked dataset: single index failed\n"); + TEST_ERROR; + } + + if(H5Dwrite(ds->single_did, vl_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, &vdata) < 0) { + printf("H5Dwrite to chunked dataset: single index failed\n"); + TEST_ERROR; + } + + /* Create 1-D chunked dataset with implicit index */ + /* Chunked, dims=max_dims, early allocation */ + + if(H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_EARLY) < 0) { + printf("H5Pset_alloc_time\n"); + TEST_ERROR; + } + + if(H5Pset_chunk(dcpl, 1, chunk_dims1) < 0) { + printf("H5Pset_chunk failed\n"); + TEST_ERROR; + } + + if((ds->implicit_did = H5Dcreate2(s->file, DSET_IMPLICIT_NAME, vl_tid, + sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) { + printf("H5Dcreate2 chunked dataset: implicit index failed\n"); + TEST_ERROR; + } + + if(H5Dwrite(ds->implicit_did, vl_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, &vdata) < 0) { + printf("H5Dwrite to chunked dataset: implicit index failed\n"); + TEST_ERROR; + } + + if(H5Pclose(dcpl) < 0) { + printf("H5Pclose failed\n"); + TEST_ERROR; + } + + if(H5Sclose(sid) < 0) { + printf("H5Sclose failed\n"); + TEST_ERROR; + } + + /* Create 1-D chunked dataset with fixed array index */ + /* Chunked, fixed max_dims */ + + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) { + printf("H5Pcreate failed\n"); + TEST_ERROR; + } + + if(H5Pset_chunk(dcpl, 1, chunk_dims1) < 0) { + printf("H5Pset_chunk failed\n"); + TEST_ERROR; + } + + if((sid = H5Screate_simple(1, dims1, max_dims1)) < 0) { + printf("H5Screate_simple failed\n"); + TEST_ERROR; + } + + if((ds->fa_did = H5Dcreate2(s->file, DSET_FA_NAME, vl_tid, + sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) { + printf("H5Dcreaet2 chunked dataset: fa index failed\n"); + TEST_ERROR; + } + + if(H5Dwrite(ds->fa_did, vl_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, &vdata) < 0) { + printf("H5Dwrite to chunked dataset: fa index failed\n"); + TEST_ERROR; + } + + if(H5Pclose(dcpl) < 0) { + printf("H5Pclose failed\n"); + TEST_ERROR; + } + + if(H5Sclose(sid) < 0) { + printf("H5Sclose failed\n"); + TEST_ERROR; + } + + /* Create 2-D chunked dataset with extensible array index */ + /* Chunked, 1 unlimited max_dims */ + + if((sid = H5Screate_simple(2, dims2, max_dims2)) < 0) { + printf("H5Screate_simple failed\n"); + TEST_ERROR; + } + + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) { + printf("H5Pcreate failed\n"); + TEST_ERROR; + } + + if(H5Pset_chunk(dcpl, 2, chunk_dims2) < 0) { + printf("H5Pset_chunk failed\n"); + TEST_ERROR; + } + + if((ds->ea_did = H5Dcreate2(s->file, DSET_EA_NAME, vl_tid, + sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) { + printf("H5Dcreate2 chunked dataset: ea index failed\n"); + TEST_ERROR; + } + + if(H5Dwrite(ds->ea_did, vl_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, &vdata2) < 0) { + printf("H5Dwrite to chunked dataset: ea index failed\n"); + TEST_ERROR; + } + + + /* Create 2-D chunked dataset with bt2 index */ + /* Chunked, 2 unlimited max_dims */ + max_dims2[0] = H5S_UNLIMITED; + + if((sid = H5Screate_simple(2, dims2, max_dims2)) < 0) { + printf("H5Screate_simple failed\n"); + TEST_ERROR; + } + + if((ds->bt2_did = H5Dcreate2(s->file, DSET_BT2_NAME, vl_tid, + sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) { + printf("H5Dcreate2 chunked dataset: bt2 index failed\n"); + TEST_ERROR; + } + + if(H5Dwrite(ds->bt2_did, vl_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, &vdata2) < 0) { + printf("H5Dwrite to chunked dataset: bt2 index failed\n"); + TEST_ERROR; + } + + if(H5Pclose(dcpl) < 0) { + printf("H5Pclose failed\n"); + TEST_ERROR; + } + + if(H5Sclose(sid) < 0) { + printf("H5Sclose failed\n"); + TEST_ERROR; + } + + if(H5Tclose(vl_tid) < 0) { + printf("H5Tclose failed\n"); + TEST_ERROR; + } + } + + return true; + +error: + H5E_BEGIN_TRY { + H5Pclose(dcpl); + H5Tclose(cmpd_tid); + H5Tclose(array_tid); + H5Tclose(dtid); + H5Tclose(vl_tid); + H5Sclose(sid); + H5Dclose(ds->compact_did); + H5Dclose(tmp_did); + H5Dclose(ds->contig_did); + H5Dclose(ds->single_did); + H5Dclose(ds->implicit_did); + H5Dclose(ds->fa_did); + H5Dclose(ds->ea_did); + H5Dclose(ds->bt2_did); + } H5E_END_TRY; + + return false; + +} /* create_dsets() */ + + +/* + * Open the datasets as specified. + */ +static bool +open_dsets(const state_t *s, dsets_state_t *ds) +{ + *ds = DSETS_INITIALIZER; + + if(s->compact) { + if(!open_dset_real(s->file, &ds->compact_did, DSET_COMPACT_NAME, + &ds->p_max_compact, &ds->p_min_dense)) { + printf("open_dset_real() for compact dataset failed\n"); + TEST_ERROR; + } + } + + if(s->contig) { + if(!open_dset_real(s->file, &ds->contig_did, DSET_CONTIG_NAME, + &ds->g_max_compact, &ds->g_min_dense)) { + printf("open_dset_real() for contiguous dataset failed\n"); + TEST_ERROR; + } + } + + if(s->chunked) { + if(!open_dset_real(s->file, &ds->single_did, DSET_SINGLE_NAME, + &ds->single_max_compact, &ds->single_min_dense)) { + printf("open_dset_real() for chunked dataset: single failed\n"); + TEST_ERROR; + } + + if(!open_dset_real(s->file, &ds->implicit_did, DSET_IMPLICIT_NAME, + &ds->implicit_max_compact, &ds->implicit_min_dense)) { + printf("open_dset_real() for chunked dataset: implicit failed\n"); + TEST_ERROR; + } + + if(!open_dset_real(s->file, &ds->fa_did, DSET_FA_NAME, + &ds->fa_max_compact, &ds->fa_min_dense)) { + printf("open_dset_real() for chunked dataset: fa failed\n"); + TEST_ERROR; + } + if(!open_dset_real(s->file, &ds->ea_did, DSET_FA_NAME, + &ds->ea_max_compact, &ds->ea_min_dense)) { + printf("open_dset_real() for chunked dataset: ea failed\n"); + TEST_ERROR; + } + if(!open_dset_real(s->file, &ds->bt2_did, DSET_BT2_NAME, + &ds->bt2_max_compact, &ds->bt2_min_dense)) { + printf("open_dset_real() for chunked dataset: bt2 failed\n"); + TEST_ERROR; + } + } + + return true; + +error: + return false; + +} /* open_dsets() */ + +/* + * Do the real work of opening the dataset. + * Retrieve the max_compact and min_dense values for the dataset. + */ +static bool +open_dset_real(hid_t fid, hid_t *did, const char *name, unsigned *max_compact, unsigned *min_dense) +{ + hid_t dcpl = badhid; + + if((*did = H5Dopen2(fid, name, H5P_DEFAULT)) < 0) { + printf("H5Dopen dataset failed\n"); + TEST_ERROR; + } + + if((dcpl = H5Dget_create_plist(*did)) < 0) { + printf("H5Dget_create_plist failed\n"); + TEST_ERROR; + } + + if(H5Pget_attr_phase_change(dcpl, max_compact, min_dense) < 0) { + printf("H5Dget_attr_phase_change failed\n"); + TEST_ERROR; + } + + if(H5Pclose(dcpl) < 0) { + printf("H5Pclose failed\n"); + TEST_ERROR; + } + + return true; + +error: + H5E_BEGIN_TRY { + H5Dclose(*did); + H5Pclose(dcpl); + } H5E_END_TRY; + + return false; +} /* open_dset_real() */ + +/* + * Close all the datasets as specified. + */ +static bool +close_dsets(const dsets_state_t *ds) +{ + if(ds->compact_did != badhid && H5Dclose(ds->compact_did) < 0) { + printf("H5Dclose compact dataset failed\n"); + TEST_ERROR; + } + + if(ds->contig_did != badhid && H5Dclose(ds->contig_did) < 0) { + printf("H5Dclose contig dataset failed\n"); + TEST_ERROR; + } + + if(ds->single_did != badhid && H5Dclose(ds->single_did) < 0) { + printf("H5Dclose chunked dataset: single index failed\n"); + TEST_ERROR; + } + + if(ds->implicit_did != badhid && H5Dclose(ds->implicit_did) < 0) { + printf("H5Dclose chunked dataset: implicit index failed\n"); + TEST_ERROR; + } + + if(ds->fa_did >= 0 && H5Dclose(ds->fa_did) < 0) { + printf("H5Dclose chunked dataset: fa index failed\n"); + TEST_ERROR; + } + + if(ds->ea_did >= 0 && H5Dclose(ds->ea_did) < 0) { + printf("H5Dclose chunked dataset: ea index failed\n"); + TEST_ERROR; + } + + if(ds->bt2_did >= 0 && H5Dclose(ds->bt2_did) < 0) { + printf("H5Dclose chunked dataset: bt2 index failed\n"); + TEST_ERROR; + } + + return true; + +error: + H5E_BEGIN_TRY { + H5Dclose(ds->compact_did); + H5Dclose(ds->contig_did); + H5Dclose(ds->single_did); + H5Dclose(ds->implicit_did); + H5Dclose(ds->fa_did); + H5Dclose(ds->ea_did); + H5Dclose(ds->bt2_did); + } H5E_END_TRY; + + return false; +} /* close_dsets() */ + +/* + * Attribute handling by the writer + */ + +/* + * Perform the "action" for each of the datasets specified on the command line. + * ADD_ATTR : -a <nattrs> option + * MODIFY_ATTR : -m option + * DELETE_ATTR : -d <dattrs> option + */ +static bool +attr_dsets_action(unsigned action, const state_t *s, const dsets_state_t *ds, unsigned which) +{ + int nerrors = 0; + bool ret = true; + + if (s->compact) { + HDassert(ds->compact_did != badhid); + dbgf(2, "to compact dataset\n"); + if(!attr_action(action, s, ds->compact_did, which)) + ++nerrors; + } + + if (s->contig) { + HDassert(ds->contig_did != badhid); + dbgf(2, "to contiguous dataset\n"); + if(!attr_action(action, s, ds->contig_did, which)) + ++nerrors; + } + + if (s->chunked) { + HDassert(ds->single_did != badhid); + dbgf(2, "to chunked dataset: single index\n"); + if(!attr_action(action, s, ds->single_did, which)) + ++nerrors; + + HDassert(ds->implicit_did != badhid); + dbgf(2, "to chunked dataset: implicit index\n"); + if(!attr_action(action, s, ds->implicit_did, which)) + ++nerrors; + + HDassert(ds->fa_did != badhid); + dbgf(2, "to chunked dataset: fixed array index\n"); + if(!attr_action(action, s, ds->fa_did, which)) + ++nerrors; + + HDassert(ds->ea_did != badhid); + dbgf(2, "to chunked dataset: extensible array index\n"); + if(!attr_action(action, s, ds->ea_did, which)) + ++nerrors; + + HDassert(ds->bt2_did != badhid); + dbgf(2, "to chunked dataset: version 2 btree index\n"); + if(!attr_action(action, s, ds->bt2_did, which)) + ++nerrors; + } + + if(nerrors) + ret = false; + + return (ret); + +} /* attr_dsets_action() */ + +/* + * Perform the action on the specified dataset. + * ADD_ATTR : add `which` attribute + * MODIFY_ATTR : modify `which` attribute + * DELETE_ATTR : delete `which` attribute + */ +static bool +attr_action(unsigned action, const state_t *s, hid_t did, unsigned which) +{ + bool ret; + + switch(action) { + case ADD_ATTR: + ret = add_attr(s, did, which); + break; + + case MODIFY_ATTR: + ret = modify_attr(s, did, which); + break; + + case DELETE_ATTR: + ret = delete_attr(did, which); + break; + + default: + HDassert(0 && "Unknown action?!?"); + } /* end switch */ + + return ret; + +} /* attr_action() */ + +/* + * Add an attribute to the specified dataset. + * The datatype can be: + * variable length (-v) or + * H5T_NATIVE_UINT32 (-b) or + * H5T_NATIVE_UINT32 (default) + */ +static bool +add_attr(const state_t *s, hid_t did, unsigned int which) +{ + hid_t aid = badhid; + hid_t tid = badhid; + hid_t vl_tid = badhid; + char name[sizeof("attr-9999999999")]; + char *val = NULL; + + esnprintf(name, sizeof(name), "attr-%u", which); + + if(s->vl_attr) { + if((vl_tid = H5Tcopy(H5T_C_S1)) < 0) { + printf("H5Tcopy failed\n"); + TEST_ERROR; + } + + if(H5Tset_size(vl_tid, H5T_VARIABLE) < 0) { + printf("H5Tset_size failed\n"); + TEST_ERROR; + } + + if((val = HDmalloc(sizeof("9999999999"))) == NULL) { + printf("H5Dmalloc failed\n"); + TEST_ERROR; + } + + HDsprintf(val, "%u", which); + + tid = vl_tid; + } else + tid = s->filetype; + + /* Attach the attribute to the dataset */ + if ((aid = H5Acreate2(did, name, tid, s->one_by_one_sid, H5P_DEFAULT, + H5P_DEFAULT)) < 0) { + printf("H5Acreate2 failed\n"); + TEST_ERROR; + } + + /* Write to the attribure */ + if (H5Awrite(aid, tid, s->vl_attr?&val:(const void *)&which) < 0) { + printf("H5Awrite failed\n"); + TEST_ERROR; + } + + /* Close the attribute */ + if (H5Aclose(aid) < 0) { + printf("H5Aclose failed\n"); + TEST_ERROR; + } + + if(vl_tid >= 0 && H5Tclose(vl_tid) < 0) { + printf("H5Tclose failed\n"); + TEST_ERROR; + } + + if(val) HDfree(val); + + return true; + +error: + H5E_BEGIN_TRY { + H5Tclose(vl_tid); + H5Aclose(aid); + } H5E_END_TRY; + + if(val) HDfree(val); + + return false; + +} /* add_attr() */ + +/* + * Modify the attribute data. + */ +static bool +modify_attr(const state_t *s, hid_t did, unsigned int which) +{ + hid_t aid = badhid; + hid_t tid = badhid; + hid_t vl_tid = badhid; + char name[sizeof("attr-9999999999")]; + char *val = NULL; + unsigned tmp_val = 0; + + esnprintf(name, sizeof(name), "attr-%u", which); + + if(s->vl_attr) { + if((vl_tid = H5Tcopy(H5T_C_S1)) < 0) { + printf("H5Tcopy failed\n"); + TEST_ERROR; + } + + if(H5Tset_size(vl_tid, H5T_VARIABLE) < 0) { + printf("H5Tset_size failed\n"); + TEST_ERROR; + } + + if((val = HDmalloc(sizeof("9999999999"))) == NULL) { + printf("HDmalloc failed\n"); + TEST_ERROR; + } + + HDsprintf(val, "%u %u %u %u %u", which, which+1, which+2, which+3, which+4); + + tid = vl_tid; + } else { + tid = s->filetype; + tmp_val = which + 1; + } + + /* Open the attribute to the dataset */ + if ((aid = H5Aopen(did, name, H5P_DEFAULT)) < 0) { + printf("H5Aopen failed\n"); + TEST_ERROR; + } + + /* Write to the attribure */ + if (H5Awrite(aid, tid, s->vl_attr?&val:(const void *)&tmp_val) < 0) { + printf("H5Awrite failed\n"); + TEST_ERROR; + } + + /* Close the attribute */ + if (H5Aclose(aid) < 0) { + printf("H5Aclose failed\n"); + TEST_ERROR; + } + + if(vl_tid >= 0 && H5Tclose(vl_tid) < 0) { + printf("H5Tclose failed\n"); + TEST_ERROR; + } + + if(val) HDfree(val); + + return true; +error: + H5E_BEGIN_TRY { + H5Aclose(aid); + H5Tclose(vl_tid); + } H5E_END_TRY; + + if(val) HDfree(val); + + return false; +} /* modify_attr() */ + +/* + * Delete the attribute + */ +static bool +delete_attr(hid_t did, unsigned int which) +{ + char name[sizeof("attr-9999999999")]; + + esnprintf(name, sizeof(name), "attr-%u", which); + + /* Delete the attribute to the dataset */ + if (H5Adelete(did, name) < 0) { + printf("H5Adelete failed\n"); + TEST_ERROR; + } + + return true; + +error: + return false; + +} /* delete_attr() */ + +/* + * Verification by the reader + */ + +/* + * Verify the action on each of the datasets specified: + * ADD_ATTR : -a <nattrs> option + * MODIFY_ATTR : -m option + * DELETE_ATTR : -d <dattrs> option + * + * Also verify continuation block and compact<->dense storage if: + * --[-c <csteps>] is 1 + * --not -m option + */ +static bool +verify_attr_dsets_action(unsigned action, const state_t *s, const dsets_state_t *ds, unsigned which) +{ + int nerrors = 0; + bool ret = true; + + if (s->compact) { + HDassert(ds->compact_did != badhid); + dbgf(2, "Verifying attribute to compact dataset\n"); + if(!verify_attr_action(action, ds->compact_did, which)) + ++nerrors; + if(s->csteps == 1 && (action != MODIFY_ATTR)) { + if(!verify_storage_cont(action, ds->compact_did, which, ds->p_max_compact, ds->p_min_dense, s->asteps)) + ++nerrors; + } + } + + if (s->contig) { + HDassert(ds->contig_did != badhid); + dbgf(2, "Verifying attribute to contiguous dataset\n"); + if(!verify_attr_action(action, ds->contig_did, which)) + ++nerrors; + if(s->csteps == 1 && (action != MODIFY_ATTR)) { + if(!verify_storage_cont(action, ds->contig_did, which, ds->g_max_compact, ds->g_min_dense, s->asteps)) + ++nerrors; + } + } + + if (s->chunked) { + HDassert(ds->single_did != badhid); + dbgf(2, "Verifying attribute to chunked dataset: single indedx\n"); + if(!verify_attr_action(action, ds->single_did, which)) + ++nerrors; + if(s->csteps == 1 && (action != MODIFY_ATTR)) { + if(!verify_storage_cont(action, ds->single_did, which, ds->single_max_compact, ds->single_min_dense, s->asteps)) + ++nerrors; + } + + HDassert(ds->implicit_did != badhid); + dbgf(2, "Verifying attribute to chunked dataset: implicit index\n"); + if(!verify_attr_action(action, ds->implicit_did, which)) + ++nerrors; + if(s->csteps == 1 && (action != MODIFY_ATTR)) { + if(!verify_storage_cont(action, ds->implicit_did, which, ds->implicit_max_compact, ds->implicit_min_dense, s->asteps)) + ++nerrors; + } + + HDassert(ds->fa_did != badhid); + dbgf(2, "Verifying attribute to chunked dataset: fa index\n"); + if(!verify_attr_action(action, ds->fa_did, which)) + ++nerrors; + if(s->csteps == 1 && (action != MODIFY_ATTR)) { + if(!verify_storage_cont(action, ds->fa_did, which, ds->fa_max_compact, ds->fa_min_dense, s->asteps)) + ++nerrors; + } + + HDassert(ds->ea_did != badhid); + dbgf(2, "Verifying attribute to chunked dataset: ea index\n"); + if(!verify_attr_action(action, ds->ea_did, which)) + ++nerrors; + if(s->csteps == 1 && (action != MODIFY_ATTR)) { + if(!verify_storage_cont(action, ds->ea_did, which, ds->ea_max_compact, ds->ea_min_dense, s->asteps)) + ++nerrors; + } + + HDassert(ds->bt2_did != badhid); + dbgf(2, "Verifying attribute to chunked dataset: bt2 index\n"); + if(!verify_attr_action(action, ds->bt2_did, which)) + ++nerrors; + if(s->csteps == 1 && (action != MODIFY_ATTR)) { + if(!verify_storage_cont(action, ds->bt2_did, which, ds->bt2_max_compact, ds->bt2_min_dense, s->asteps)) + ++nerrors; + } + } + + if(nerrors) + ret = false; + + return (ret); + +} /* verify_attr_dsets_action() */ + +/* + * Verify the attribute action on the specified dataset. + */ +static bool +verify_attr_action(unsigned action, hid_t did, unsigned which) +{ + char name[sizeof("attr-9999999999")]; + bool ret; + + esnprintf(name, sizeof(name), "attr-%u", which); + + switch(action) { + case ADD_ATTR: + ret = verify_add_or_modify_attr(action, did, name, which); + break; + + case MODIFY_ATTR: + ret = verify_add_or_modify_attr(action, did, name, which); + break; + + case DELETE_ATTR: + ret = verify_delete_attr(did, name); + break; + + default: + HDassert(0 && "Unknown action?!?"); + } /* end switch */ + + return ret; +} /* verify_attr_action() */ + +/* + * Verify the attribute is added or modified + */ +static bool +verify_add_or_modify_attr(unsigned action, hid_t did, char *attr_name, unsigned int which) +{ + unsigned int read_which; + char vl_which[sizeof("attr-9999999999")]; + char *read_vl_which = NULL; + bool is_vl = false; + hid_t aid, atid; + bool ret; + + HDassert(did != badhid); + HDassert(action == ADD_ATTR || action == MODIFY_ATTR); + + if ((aid = H5Aopen(did, attr_name, H5P_DEFAULT)) < 0) { + printf("H5Aopen failed\n"); + TEST_ERROR; + } + + if((atid = H5Aget_type(aid)) < 0) { + printf("H5Aget_type failed\n"); + TEST_ERROR; + } + + if((is_vl = H5Tis_variable_str(atid))) { + if(action == ADD_ATTR) + HDsprintf(vl_which, "%u", which); + else + HDsprintf(vl_which, "%u %u %u %u %u", which, which+1, which+2, which+3, which+4); + + if((read_vl_which = HDmalloc(sizeof("9999999999"))) == NULL) { + printf("HDmalloc failed\n"); + TEST_ERROR; + } + } + + if (H5Aread(aid, atid, is_vl? (void *)&read_vl_which : (void *)&read_which) < 0) { + printf("H5Aread failed\n"); + TEST_ERROR; + } + + if (H5Aclose(aid) < 0) { + printf("H5Aclose failed\n"); + TEST_ERROR; + } + + if(H5Tclose(atid) < 0) { + printf("H5Tclose failed\n"); + TEST_ERROR; + } + + if(is_vl) { + if(!HDstrcmp(vl_which, read_vl_which)) + ret = true; + } else + ret = (read_which == which); + + if(read_vl_which) + HDfree(read_vl_which); + + return ret; + +error: + H5E_BEGIN_TRY { + H5Aclose(aid); + H5Tclose(atid); + } H5E_END_TRY; + + if(read_vl_which) + HDfree(read_vl_which); + + return false; + +} /* verify_add_or_modify_attr() */ + +/* + * Verify the attribute does not exist. + */ +static bool +verify_delete_attr(hid_t did, char *attr_name) +{ + int ret; + + if((ret = H5Aexists(did, attr_name)) < 0) { + printf("H5Aexists failed\n"); + TEST_ERROR; + + } else if(!ret) /* attribute does not exist */ + ret = true; + else /* attribute exist */ + ret = false; + + return ret; + +error: + return false; + +} /* verify_delete_attr() */ + +/* + * `which` is indexed by 0, 1, 2, 3... + * + * Checkpoints: + * --`which` is 0: no continuation block + * + * For addition: + * For deletion but [-a <nattrs>] is not yet to dense storage: + * --`which` is at (max_compact - 1): compact storage, continuation block exists + * --`which` is at max_compact: dense storage, no continuation block + * For deletion: + * --`which` is at min_dense: dense storage, no continuation block + * --`which` is at (min_dense - 1): compact storage, continuation block exists + */ +static bool +verify_storage_cont(unsigned action, hid_t did, unsigned int which, unsigned max_compact, unsigned min_dense, unsigned asteps) +{ + bool ret = true; + + HDassert(action == ADD_ATTR || action == DELETE_ATTR); + + /* Verify no cont */ + if (!which) + ret = verify_storage_cont_real(did, which, max_compact); + + /* For addition: */ + /* For deletion, if [-a <nattrs>] is not yet to dense storage */ + else if (action == ADD_ATTR || (action == DELETE_ATTR && asteps <= max_compact)) { + + /* Verify compact storage & cont */ + if (which == (max_compact - 1)) + ret = verify_storage_cont_real(did, which, max_compact); + + /* Verify dense storage & no cont */ + else if (which == max_compact) + ret = verify_storage_cont_real(did, which, max_compact); + + /* For deletion */ + } else if (action == DELETE_ATTR) { + + /* Verify compact storage & cont */ + if(which == (min_dense - 1)) + ret = verify_storage_cont_real(did, which, min_dense); + + /* Verify dense storage & no cont */ + else if(which == min_dense) + ret = verify_storage_cont_real(did, which, min_dense); + + } + + return ret; + +} /* verify_storage_cont() */ + +/* + * Verify the storage condition at the specific checkpoint + */ +static bool +verify_storage_cont_real(hid_t did, unsigned int which, unsigned cut_point) +{ + H5O_native_info_t ninfo; + + /* Get the object information */ + if(H5Oget_native_info(did, &ninfo, H5O_NATIVE_INFO_HDR|H5O_NATIVE_INFO_META_SIZE) < 0) { + printf("H5Oget_native_info failed\n"); + TEST_ERROR; + } + + if(!which) { + dbgf(2, "Verifying no cont\n"); + return(ninfo.hdr.nchunks == 1); + + } else if(which < cut_point) { + dbgf(2, "Verifying storage compact & cont\n"); + return(ninfo.meta_size.attr.index_size == 0 && + ninfo.meta_size.attr.heap_size == 0 && + ninfo.hdr.nchunks > 1); + } else { + dbgf(2, "Verifying storage dense & no cont\n"); + return(ninfo.meta_size.attr.index_size != 0 && + ninfo.meta_size.attr.heap_size != 0 && + ninfo.hdr.nchunks == 1); + } + +error: + return false; + +} /* verify_storage_cont_real() */ + + +/* + * Named pipes handling + */ + +/* + * Initialize the named pipes for test synchronization. + */ +static bool +np_init(np_state_t *np, bool writer) +{ + *np = NP_INITIALIZER; + + /* + * Use two named pipes(FIFO) to coordinate the writer and reader for + * two-way communication so that the two sides can move forward together. + * One is for the writer to write to the reader. + * The other one is for the reader to signal the writer. + */ + if (writer) { + /* If the named pipes are present at the start of the test, remove them */ + if (HDaccess(np->fifo_writer_to_reader, F_OK) == 0) + if(HDremove(np->fifo_writer_to_reader) != 0) { + printf("HDremove fifo_writer_to_reader failed\n"); + TEST_ERROR; + } + + if (HDaccess(np->fifo_reader_to_writer, F_OK) == 0) + if(HDremove(np->fifo_reader_to_writer) != 0) { + printf("HDremove fifo_reader_to_writer failed\n"); + TEST_ERROR; + } + + /* Writer creates two named pipes(FIFO) */ + if (HDmkfifo(np->fifo_writer_to_reader, 0600) < 0) { + printf("HDmkfifo fifo_writer_to_reader failed\n"); + TEST_ERROR; + } + + if (HDmkfifo(np->fifo_reader_to_writer, 0600) < 0) { + printf("HDmkfifo fifo_reader_to_writer failed\n"); + TEST_ERROR; + } + } + + /* Both the writer and reader open the pipes */ + if ((np->fd_writer_to_reader = HDopen(np->fifo_writer_to_reader, O_RDWR)) < 0) { + printf("HDopen fifo_writer_to_reader failed\n"); + TEST_ERROR; + } + + if ((np->fd_reader_to_writer = HDopen(np->fifo_reader_to_writer, O_RDWR)) < 0) { + printf("HDopen fifo_reader_to_writer failed\n"); + TEST_ERROR; + } + + return true; + +error: + return false; + +} /* np_init() */ + +/* + * Close the named pipes. + */ +static bool +np_close(np_state_t *np, bool writer) +{ + /* Both the writer and reader close the named pipes */ + if (HDclose(np->fd_writer_to_reader) < 0) { + printf("HDclose fd_writer_to_reader failed\n"); + TEST_ERROR; + } + + if (HDclose(np->fd_reader_to_writer) < 0) { + printf("HDclose fd_reader_to_writer failed\n"); + TEST_ERROR; + } + + /* Reader finishes last and deletes the named pipes */ + if(!writer) { + if(HDremove(np->fifo_writer_to_reader) != 0) { + printf("HDremove fifo_writer_to_reader failed\n"); + TEST_ERROR; + } + + if(HDremove(np->fifo_reader_to_writer) != 0) { + printf("HDremove fifo_reader_to_writer failed\n"); + TEST_ERROR; + } + } + return true; + +error: + return false; +} /* np_close() */ + +/* + * Writer synchronization depending on the result from the attribute action performed. + */ +static bool +np_writer(bool result, unsigned step, const state_t *s, np_state_t *np, H5F_vfd_swmr_config_t *config) +{ + unsigned int i; + + /* The action fails */ + if(!result) { + printf("attribute action failed\n"); + H5_FAILED(); AT(); + + /* At communication interval, notify the reader about the failure and quit */ + if (step % s->csteps == 0) { + np->notify = -1; + HDwrite(np->fd_writer_to_reader, &np->notify, sizeof(int)); + goto error; + } + /* The action succeeds */ + } else { + /* At communication interval, notify the reader and wait for its response */ + if (step % s->csteps == 0) { + /* Bump up the value of notify to tell the reader to start reading */ + np->notify++; + if (HDwrite(np->fd_writer_to_reader, &np->notify, sizeof(int)) < 0) { + printf("HDwrite failed\n"); + TEST_ERROR; + } + + /* During the wait, writer makes repeated HDF5 API calls + * to trigger EOT at approximately the correct time */ + for(i = 0; i < config->max_lag + 1; i++) { + decisleep(config->tick_len); + H5E_BEGIN_TRY { + H5Aexists(s->file, "nonexistent"); + } H5E_END_TRY; + } + + /* Handshake between writer and reader */ + if(!np_confirm_verify_notify(np->fd_reader_to_writer, step, s, np)) { + printf("np_confirm_verify_notify() verify/notify not in sync failed\n"); + TEST_ERROR; + } + } + } + return true; + +error: + return false; + +} /* np_writer() */ + +/* + * + * Reader synchronization depending on the result from the verification. + */ +static bool +np_reader(bool result, unsigned step, const state_t *s, np_state_t *np) +{ + /* The verification fails */ + if(!result) { + printf("verify action failed\n"); + H5_FAILED(); AT(); + + /* At communication interval, tell the writer about the failure and exit */ + if (step % s->csteps == 0) { + np->notify = -1; + HDwrite(np->fd_reader_to_writer, &np->notify, sizeof(int)); + goto error; + } + /* The verification succeeds */ + } else { + if (step % s->csteps == 0) { + /* Send back the same notify value for acknowledgement: + * --inform the writer to move to the next step */ + if (HDwrite(np->fd_reader_to_writer, &np->notify, sizeof(int)) < 0) { + printf("HDwrite failed\n"); + TEST_ERROR; + } + } + } + return true; + +error: + return false; + +} /* np_reader() */ + +/* + * Handshake between writer and reader: + * Confirm `verify` is same as `notify`. + */ +static bool +np_confirm_verify_notify(int fd, unsigned step, const state_t *s, np_state_t *np) +{ + if (step % s->csteps == 0) { + np->verify++; + if (HDread(fd, &np->notify, sizeof(int)) < 0) { + printf("HDread failed\n"); + TEST_ERROR; + } + + if (np->notify == -1) { + printf("reader/writer failed to verify\n"); + TEST_ERROR; + } + + if (np->notify != np->verify) { + printf("received message %d, expecting %d\n", np->notify, np->verify); + TEST_ERROR; + } + } + + return true; + +error: + return false; +} /* confirm_verify_notify() */ + +/* + * Synchronization done by the reader before moving onto the + * next verification phase. + */ +static bool +np_reader_no_verification(const state_t *s, np_state_t *np, H5F_vfd_swmr_config_t *config) +{ + if(s->use_np) { + if (!np_confirm_verify_notify(np->fd_writer_to_reader, 0, s, np)) { + printf("np_confirm_verify_notify() verify/notify not in sync failed\n"); + TEST_ERROR; + } + } + + /* Wait for a few ticks for the update to happen */ + decisleep(config->tick_len * s->update_interval); + + if(s->use_np) { + /* Send back the same notify value for acknowledgement: + * --inform the writer to move to the next step */ + if (HDwrite(np->fd_reader_to_writer, &np->notify, sizeof(int)) < 0) { + printf("HDwrite failed\n"); + TEST_ERROR; + } + } + + return true; + +error: + return false; + +} /* np_reader_no_verification() */ + +int +main(int argc, char **argv) +{ + hid_t fapl, fcpl; + unsigned step; + bool writer; + state_t s; + const char *personality; + H5F_vfd_swmr_config_t config; + np_state_t np; + dsets_state_t ds; + unsigned dd; + bool result; + + if(!state_init(&s, argc, argv)) { + printf("state_init() failed\n"); + TEST_ERROR; + } + + personality = strstr(s.progname, "vfd_swmr_attrdset_"); + + if (personality != NULL && + strcmp(personality, "vfd_swmr_attrdset_writer") == 0) + writer = true; + else if (personality != NULL && + strcmp(personality, "vfd_swmr_attrdset_reader") == 0) + writer = false; + else { + printf("unknown personality, expected vfd_swmr_attrdset_{reader,writer}\n"); + TEST_ERROR; + } + + /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ + init_vfd_swmr_config(&config, 4, 7, writer, FALSE, 128, "./attrdset-shadow"); + + /* use_latest_format, use_vfd_swmr, only_meta_page, config */ + if((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, &config)) < 0) { + printf("vfd_swmr_create_fapl() failed\n"); + TEST_ERROR; + } + + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) { + printf("H5Pcreate failed\n"); + TEST_ERROR; + } + + if(H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, false, 1) < 0) { + printf("H5Pset_file_space_strategy failed\n"); + TEST_ERROR; + } + + if (writer) { + if((s.file = H5Fcreate(s.filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) { + printf("H5Fcreate failed\n"); + TEST_ERROR; + } + + if(!create_dsets(&s, &ds)) { + printf("create_dsets() failed\n"); + TEST_ERROR; + } + + } else { + if ((s.file = H5Fopen(s.filename, H5F_ACC_RDONLY, fapl)) < 0) { + printf("H5Fopen failed\n"); + TEST_ERROR; + } + if(!open_dsets(&s, &ds)) { + printf("open_dsets() failed\n"); + TEST_ERROR; + } + } + + /* Initiailze named pipes */ + if(s.use_np && !np_init(&np, writer)) { + printf("np_init() failed\n"); + TEST_ERROR; + } + + if (writer) { + for (step = 0; step < s.asteps; step++) { + dbgf(2, "Adding attribute %d\n", step); + + result = attr_dsets_action(ADD_ATTR, &s, &ds, step); + + if(s.use_np && !np_writer(result, step, &s, &np, &config)) { + printf("np_writer() for addition failed\n"); + TEST_ERROR; + } + + } + + if(s.mod_attr) { + + /* Need to sync up writer/reader before moving onto the next phase */ + if(s.use_np && !np_writer(true, 0, &s, &np, &config)) { + printf("np_writer() for modification failed\n"); + TEST_ERROR; + } + + /* Start modification */ + for (step = 0; step < s.asteps; step++) { + dbgf(2, "Modifying attribute %d\n", step); + + result = attr_dsets_action(MODIFY_ATTR, &s, &ds, step); + + if(s.use_np && !np_writer(result, step, &s, &np, &config)) { + printf("np_writer() for modification failed\n"); + TEST_ERROR; + } + } + } + + if(s.dattrs) { + + /* Need to sync up writer/reader before moving onto the next phase */ + if(s.use_np && !np_writer(true, 0, &s, &np, &config)) { + printf("np_writer() for deletion failed\n"); + TEST_ERROR; + } + + /* Start deletion */ + for (dd = 0, step = s.asteps - 1; dd < s.dattrs; dd++, --step) { + dbgf(2, "Deleting attribute %d\n", step); + + result = attr_dsets_action(DELETE_ATTR, &s, &ds, step); + + if(s.use_np && !np_writer(result, step, &s, &np, &config)) { + printf("np_writer() for deletion failed\n"); + TEST_ERROR; + } + } + + } + + } else { + + /* Start verifying addition */ + for (step = 0; step < s.asteps; step++) { + dbgf(2, "Verifying...attribute %d\n", step); + + if(s.use_np && !np_confirm_verify_notify(np.fd_writer_to_reader, step, &s, &np)) { + printf("np_confirm_verify_notify() verify/notify not in sync failed\n"); + TEST_ERROR; + } + + /* Wait for a few ticks for the update to happen */ + decisleep(config.tick_len * s.update_interval); + + result = verify_attr_dsets_action(ADD_ATTR, &s, &ds, step); + + if(s.use_np && !np_reader(result, step, &s, &np)) { + printf("np_reader() for verifying addition failed\n"); + TEST_ERROR; + } + } + + if(s.mod_attr) { + /* Need to sync up writer/reader before moving onto the next phase */ + if(!np_reader_no_verification(&s, &np, &config)) { + printf("np_reader_no_verification() for verifying modification failed\n"); + TEST_ERROR; + } + + /* Start verifying modification */ + for (step = 0; step < s.asteps; step++) { + dbgf(2, "Verifying...modify attribute %d\n", step); + + if(s.use_np && !np_confirm_verify_notify(np.fd_writer_to_reader, step, &s, &np)) { + printf("np_confirm_verify_notify() verify/notify not in sync failed\n"); + TEST_ERROR; + } + + /* Wait for a few ticks for the update to happen */ + decisleep(config.tick_len * s.update_interval); + + result = verify_attr_dsets_action(MODIFY_ATTR, &s, &ds, step); + + if(s.use_np && !np_reader(result, step, &s, &np)) { + printf("np_reader() for verifying modification failed\n"); + TEST_ERROR; + } + + } + } + + if(s.dattrs) { + + /* Need to sync up writer/reader before moving onto the next phase */ + if(!np_reader_no_verification(&s, &np, &config)) { + printf("np_reader_no_verification() for verifying modification failed\n"); + TEST_ERROR; + } + + /* Start verifying deletion */ + for (dd = 0, step = s.asteps - 1; dd < s.dattrs; dd++, --step) { + dbgf(2, "Verifying...delete attribute %d\n", step); + + if(s.use_np && !np_confirm_verify_notify(np.fd_writer_to_reader, step, &s, &np)) { + printf("np_confirm_verify_notify() verify/notify not in sync failed\n"); + TEST_ERROR; + } + + /* Wait for a few ticks for the update to happen */ + decisleep(config.tick_len * s.update_interval); + + result = verify_attr_dsets_action(DELETE_ATTR, &s, &ds, step); + + if(s.use_np && !np_reader(result, step, &s, &np)) { + printf("np_reader() for verifying deletion failed\n"); + TEST_ERROR; + } + + } + + } + } + + if(!close_dsets(&ds)) { + printf("close_dsets() failed\n"); + TEST_ERROR; + } + + if (H5Pclose(fapl) < 0) { + printf("H5Pclose failed\n"); + TEST_ERROR; + } + + if (H5Pclose(fcpl) < 0) { + printf("H5Pclose failed\n"); + TEST_ERROR; + } + + if (H5Fclose(s.file) < 0) { + printf("H5Fclose failed\n"); + TEST_ERROR; + } + + if (H5Sclose(s.one_by_one_sid) < 0) { + printf("H5Sclose failed\n"); + TEST_ERROR; + } + + if(s.use_np && !np_close(&np, writer)) { + printf("np_close() failed\n"); + TEST_ERROR; + } + + return EXIT_SUCCESS; + +error: + H5E_BEGIN_TRY { + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(s.file); + H5Sclose(s.one_by_one_sid); + } H5E_END_TRY; + + if (s.use_np && np.fd_writer_to_reader >= 0) + HDclose(np.fd_writer_to_reader); + + if (s.use_np && np.fd_reader_to_writer >= 0) + HDclose(np.fd_reader_to_writer); + + if(s.use_np && !writer) { + HDremove(np.fifo_writer_to_reader); + HDremove(np.fifo_reader_to_writer); + } + + return EXIT_FAILURE; +} /* main */ + +#endif /* H5_HAVE_WIN32_API */ diff --git a/test/vfd_swmr_common.c b/test/vfd_swmr_common.c index dad9e3d..aed9cbe 100644 --- a/test/vfd_swmr_common.c +++ b/test/vfd_swmr_common.c @@ -72,6 +72,15 @@ below_speed_limit(struct timespec *last, const struct timespec *ival) return result; } +/* Sleep for `tenths` tenths of a second. */ +void +decisleep(uint32_t tenths) +{ + uint64_t nsec = tenths * 100 * 1000 * 1000; + + H5_nanosleep(nsec); +} + /* Like vsnprintf(3), but abort the program with an error message on * `stderr` if the buffer is too small or some other error occurs. */ @@ -382,6 +391,10 @@ vfd_swmr_create_fapl(bool use_latest_format, bool use_vfd_swmr, bool only_meta_p if (H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) return H5I_INVALID_HID; } + else {/* Currently this is used only for old-styled group implementation tests.*/ + if (H5Pset_libver_bounds(fapl, H5F_LIBVER_EARLIEST, H5F_LIBVER_LATEST) < 0) + return H5I_INVALID_HID; + } /* Enable page buffering */ if (H5Pset_page_buffer_size(fapl, 4096, only_meta_pages ? 100 : 0, 0) < 0) diff --git a/test/vfd_swmr_common.h b/test/vfd_swmr_common.h index e70c316..fee0725 100644 --- a/test/vfd_swmr_common.h +++ b/test/vfd_swmr_common.h @@ -57,7 +57,8 @@ H5TEST_DLLVAR int verbosity; extern "C" { #endif -H5TEST_DLL hbool_t below_speed_limit(struct timespec *, const struct timespec *); +H5TEST_DLL bool below_speed_limit(struct timespec *, const struct timespec *); +H5TEST_DLL void decisleep(uint32_t tenths); H5TEST_DLL estack_state_t estack_get_state(void); H5TEST_DLL estack_state_t disable_estack(void); diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index 33f0f7f..7f083c8 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -24,17 +24,27 @@ #ifndef H5_HAVE_WIN32_API #define READER_WAIT_TICKS 3 +#define VS_ATTR_NAME_LEN 21 typedef struct { - hid_t file, filetype, one_by_one_sid; - char filename[PATH_MAX]; - char progname[PATH_MAX]; - unsigned int asteps; - unsigned int csteps; - unsigned int nsteps; - unsigned int update_interval; - bool use_vfd_swmr; - bool use_named_pipes; + hid_t file, filetype, one_by_one_sid; + char filename[PATH_MAX]; + char progname[PATH_MAX]; + unsigned int asteps; + unsigned int csteps; + unsigned int nsteps; + unsigned int update_interval; + bool use_vfd_swmr; + bool old_style_grp; + bool use_named_pipes; + char at_pattern; + bool attr_test; + uint32_t max_lag; + uint32_t tick_len; + int np_fd_w_to_r; + int np_fd_r_to_w; + int np_notify; + int np_verify; } state_t; #define ALL_HID_INITIALIZER \ @@ -43,26 +53,58 @@ typedef struct { .file = H5I_INVALID_HID, .one_by_one_sid = H5I_INVALID_HID, .filename = "", \ .filetype = H5T_NATIVE_UINT32, .asteps = 10, .csteps = 10, .nsteps = 100, .update_interval = READER_WAIT_TICKS, \ .use_vfd_swmr = true, \ + .old_style_grp = false, \ .use_named_pipes = true \ - } + , .at_pattern = ' ' \ + , .attr_test = false \ + , .tick_len = 4 \ + , .max_lag = 7 \ + , .np_fd_w_to_r = -1 \ + , .np_fd_r_to_w = -1 \ + , .np_notify = 0 \ + , .np_verify = 0 } + static void usage(const char *progname) { - fprintf(stderr, "usage: %s [-S] [-a steps] [-b] [-c]\n" - " [-n iterations] [-N] [-q] [-u numb_ticks]\n" - "\n" - "-S: do not use VFD SWMR\n" - "-a steps: `steps` between adding attributes\n" - "-b: write data in big-endian byte order\n" - "-c steps: `steps` between communication between the writer and reader\n" - "-n ngroups: the number of groups\n" - "-N: do not use named pipes, mainly for running the writer and reader seperately\n" - "-u numb_tcks: `numb_ticks` for the reader to wait before verification\n" - "-q: silence printouts, few messages\n" - "\n", - progname); - exit(EXIT_FAILURE); + fprintf(stderr, "usage: %s [-S] [-G] [-a steps] [-b] [-c]\n" + " [-n iterations] [-N] [-q] [-u numb_ticks] [-A at_pattern]\n" + "\n" + "-S: do not use VFD SWMR\n" + "-G: old-style type of group\n" + "-a steps: `steps` between adding attributes\n" + "-b: write data in big-endian byte order\n" + "-c steps: `steps` between communication between the writer and reader\n" + "-n ngroups: the number of groups\n" + "-N: do not use named pipes, \n" + " mainly for running the writer and reader seperately\n" + "-u numb_ticks: `numb_ticks` for the reader to wait before verification\n" + "-A at_pattern: `at_pattern' for different attribute tests\n" + " The value of `at_pattern` is one of the following:\n" + " `compact` - Attributes added in compact storage\n" + " `dense` - An attribute added in dense storage\n" + " `compact-del` - Attributes added and then one\n" + " attribute deleted, in compact \n" + " `dense-del` - Attributes added until the storage\n" + " is dense then an attribute deleted\n" + " the storge still in dense\n" + " `compact-add-to-dense` - Attributes added first in compact\n" + " then in dense storage\n" + " `dense-del-to-compact` - Attributes added until the storage\n" + " is dense, then several attributes \n" + " deleted, the storage changed to\n" + " compact\n" + " `modify` - An attribute added then modified\n" + " `add-vstr` - A VL string attribute added\n" + " `remove-vstr` - A VL string attribute added then\n" + " deleted\n" + " `modify-vstr` - A VL string attribute added then \n" + " modified \n" + "-q: silence printouts, few messages\n" + "\n", + progname); + exit(EXIT_FAILURE); } static bool @@ -87,17 +129,20 @@ state_init(state_t *s, int argc, char **argv) if (tfile) HDfree(tfile); - while ((ch = getopt(argc, argv, "Sa:bc:n:Nqu:")) != -1) { + while ((ch = getopt(argc, argv, "SGa:bc:n:Nqu:A:")) != -1) { switch (ch) { case 'S': s->use_vfd_swmr = false; break; + case 'G': + s->old_style_grp = true; + break; case 'a': case 'c': case 'n': case 'u': errno = 0; - tmp = strtoul(optarg, &end, 0); + tmp = HDstrtoul(optarg, &end, 0); if (end == optarg || *end != '\0') { H5_FAILED(); AT(); printf("couldn't parse `-%c` argument `%s`\n", ch, optarg); @@ -127,6 +172,33 @@ state_init(state_t *s, int argc, char **argv) case 'N': s->use_named_pipes = false; break; + case 'A': + if (HDstrcmp(optarg, "compact") == 0) + s->at_pattern = 'c'; + else if (HDstrcmp(optarg, "dense") == 0) + s->at_pattern = 'd'; + else if (HDstrcmp(optarg, "compact-add-to-dense") == 0) + s->at_pattern = 't'; + else if (HDstrcmp(optarg, "compact-del") == 0) + s->at_pattern = 'C'; + else if (HDstrcmp(optarg, "dense-del") == 0) + s->at_pattern = 'D'; + else if (HDstrcmp(optarg, "dense-del-to-compact") == 0) + s->at_pattern = 'T'; + else if (HDstrcmp(optarg, "modify") == 0) + s->at_pattern = 'M'; + else if (HDstrcmp(optarg,"add-vstr") ==0) + s->at_pattern = 'v'; + else if (HDstrcmp(optarg, "remove-vstr") == 0) + s->at_pattern = 'r'; + else if (HDstrcmp(optarg, "modify-vstr") == 0) + s->at_pattern = 'm'; + else { + H5_FAILED(); AT(); + printf("Invalid -A argument \"%s\"", optarg); + goto error; + } + break; case 'q': verbosity = 0; break; @@ -171,72 +243,1428 @@ state_init(state_t *s, int argc, char **argv) error: if (tfile) HDfree(tfile); + return false; +} + +/* Named Pipe Subroutine: np_wr_send_receive + * Description: + * The writer sends a message to the reader, + * then waits for max_lag ticks, + * then checks the returned message from the reader. + * Return: + * True if succeed + * False if an error occurs in any step above. + * An error is mostly caused by an unexpected + * notification number from the message sent + * by the reader. + */ +static bool np_wr_send_receive(state_t *s) { + + unsigned int i; + /* Bump up the value of notify to notice the reader to start to read */ + s->np_notify++; + if (HDwrite(s->np_fd_w_to_r, &(s->np_notify), sizeof(int)) < 0) { + H5_FAILED(); AT(); + printf("HDwrite failed\n"); + goto error; + } + + /* During the wait, writer makes repeated HDF5 API calls + * to trigger EOT at approximately the correct time */ + for(i = 0; i < s->max_lag + 1; i++) { + decisleep(s->tick_len); + H5E_BEGIN_TRY { + H5Aexists(s->file, "nonexistent"); + } H5E_END_TRY; + } + /* Receive the same value from the reader and verify it before + * going to the next step */ + (s->np_verify)++; + if (HDread(s->np_fd_r_to_w, &(s->np_notify), sizeof(int)) < 0){ + H5_FAILED(); AT(); + printf("HDread failed\n"); + goto error; + } + + if (s->np_notify == -1) { + H5_FAILED(); AT(); + printf("reader failed to verify group or attribute operation.\n"); + goto error; + } + + if (s->np_notify != s->np_verify) { + H5_FAILED(); AT(); + printf("received message %d, expecting %d\n", s->np_notify, s->np_verify); + goto error; + } + + return true; + +error: return false; + } -static bool -add_group_attribute(const state_t *s, hid_t g, hid_t sid, unsigned int which) -{ - hid_t aid; - char name[sizeof("attr-9999999999")]; +/* Named Pipe Subroutine: np_rd_receive + * Description: + * The reader receives a message from the writer, + * then checks if the notification number from + * the writer is expected. + * Return: + * True if succeed + * False if an error occurs in any step above. + * An error is mostly caused by an unexpected + * notification number from the message sent + * by the writer. + */ +static bool np_rd_receive(state_t *s) { + + /* The writer should have bumped up the value of notify. + * Do the same with verify and confirm it */ + s->np_verify++; + + /* Receive the notify that the writer bumped up the value */ + if (HDread(s->np_fd_w_to_r, &(s->np_notify), sizeof(int)) < 0) { + H5_FAILED(); AT(); + printf("HDread failed\n"); + goto error; + } + + if (s->np_notify == -1) { + H5_FAILED(); AT(); + printf("writer failed to create group or carry out an attribute operation.\n"); + goto error; + } + + if (s->np_notify != s->np_verify) { + H5_FAILED(); AT(); + printf("received message %d, expecting %d\n", s->np_notify, s->np_verify); + goto error; + } + + return true; + +error: + return false; +} + +/* Named Pipe Subroutine: np_rd_send + * Description: + * The reader sends an acknowledgement to the writer + * Return: + * True if succeed + * False if an error occurs in sending the message. + */ +static bool np_rd_send(state_t *s) { + + if (HDwrite(s->np_fd_r_to_w, &(s->np_notify), sizeof(int)) < 0) { + H5_FAILED(); AT(); + printf("HDwrite failed\n"); + return false; + } + else + return true; +} + +/* Named Pipe Subroutine: np_send_error + * Description: + * An error (notification number is 1) message is sent + * either from the reader or the writer. + * A boolean input parameter is used to choose + * either reader or writer. + * Return: + * None + */ +static void np_send_error(state_t *s,bool writer) { + s->np_notify = -1; + if(writer) + HDwrite(s->np_fd_w_to_r, &(s->np_notify), sizeof(int)); + else + HDwrite(s->np_fd_r_to_w, &(s->np_notify), sizeof(int)); +} + +/*------------------------------------------------------------------------- + * Function: add_attr + * + * Purpose: Add attributes to a group. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t oid + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group name. The group name is "group-which". + * + * unsigned num_attrs + * The number of attributes to be created + * + * const char*aname_fmt + * The attribute name template used to create unique attribute names. + * + * unsigned int g_which + * This parameter is used to generate correct group name in a key + * debugging message. + * + * Return: Success: true + * Failure: false + * + *------------------------------------------------------------------------- +*/ + +static bool +add_attr(state_t *s, + hid_t oid, + unsigned int which, + unsigned num_attrs, + const char*aname_fmt, + unsigned int g_which) { + + char attrname[VS_ATTR_NAME_LEN]; + unsigned u; + unsigned attr_value; + hid_t aid = H5I_INVALID_HID; + hid_t amtype = H5I_INVALID_HID; + hid_t atype = s->filetype; + hid_t sid = s->one_by_one_sid; + + /* Need to obtain native datatype for H5Aread */ + if((amtype = H5Tget_native_type(atype,H5T_DIR_ASCEND)) <0) { + H5_FAILED(); AT(); + printf("H5Tget_native_type failed\n"); + goto error; + } + + for (u = 0; u < num_attrs; u++) { + + /* Create attribute */ + /* Construct attribute name like attr-0-0 */ + HDsprintf(attrname, aname_fmt, which,u); + if((aid = H5Acreate2(oid, attrname, atype, sid, H5P_DEFAULT, + H5P_DEFAULT)) < 0) { + H5_FAILED(); AT(); + printf("H5Acreate2 failed\n"); + goto error; + } + + attr_value = u+which; +#if 0 + // Just for debugging to check if error handling works. + attr_value = u+which+1; +#endif + + dbgf(1, "setting attribute %s on group %u to %u\n", attrname, g_which, u+which); + + /* Write data into the attribute */ + if (H5Awrite(aid, amtype, &attr_value) < 0) { + H5_FAILED(); AT(); + printf("H5Awrite failed\n"); + goto error; + } + + /* Close attribute */ + if(H5Aclose(aid) < 0) { + H5_FAILED(); AT(); + printf("H5Aclose failed\n"); + goto error; + } + + /* Writer sends a message to reader: an attribute is successfully generated. + then wait for the reader to verify and send an acknowledgement message back.*/ + if (s->use_named_pipes && s->attr_test == true) { + dbgf(2, "writer: write attr - ready to send/receive message: %d\n", s->np_notify+1); + if(np_wr_send_receive(s) == false) { + H5_FAILED(); AT(); + dbgf(2, "writer: write attr - verification failed.\n"); + /* Note: This is (mostly) because the verification failure message + * from the reader. So don't send the error message back to + * the reader. Just stop the test. */ + goto error2; + } + } + + } /* end for */ + + if(H5Tclose(amtype) < 0) { + H5_FAILED(); AT(); + goto error; + } + return true; + +error: + /* Writer needs to send an error message to the reader to stop the test*/ + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); + +error2: + H5E_BEGIN_TRY { + H5Aclose(aid); + H5Tclose(amtype); + } H5E_END_TRY; + + return false; + +} + +/*------------------------------------------------------------------------- + * Function: add_default_group_attr + * + * Purpose: Add an attribute to a group. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * The group name is "group-which" and the attribute name + * is "attr-which". + * + * + * Return: Success: true + * Failure: false + * + * Note: This function is used for the "dense" storage test. + * It is also used by the group-only test. + *------------------------------------------------------------------------- +*/ + +static bool +add_default_group_attr(state_t *s, hid_t g, unsigned int which) { + + const char* aname_format ="attr-%u"; + + /* Note: Since we only add one attribute, the parameter for + * the number of attributes is 1. */ + return add_attr(s,g,which,1,aname_format,which); + +} + +/*------------------------------------------------------------------------- + * Function: add_vlstr_attr + * + * Purpose: Add a variable length string attribute to a group. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * The group name is "group-which" and the attribute name + * is "attr-which". + * + * + * Return: Success: true + * Failure: false + * + * Note: This is for the "vstr" test. + *------------------------------------------------------------------------- +*/ + + +static bool +add_vlstr_attr(state_t*s, hid_t g, unsigned int which) { + + hid_t aid = H5I_INVALID_HID; + hid_t atype = H5I_INVALID_HID; + char name[VS_ATTR_NAME_LEN]; + char *astr_val = NULL; + hid_t sid = s->one_by_one_sid; + + /* Allocate buffer for the VL string value */ + astr_val = HDmalloc(VS_ATTR_NAME_LEN); + if (astr_val == NULL) { + H5_FAILED(); AT(); + printf("Allocate memory for VL string failed.\n"); + goto error; + } + + /* Assign the VL string value and the attribute name.. */ + HDsprintf(astr_val,"%u",which); esnprintf(name, sizeof(name), "attr-%u", which); dbgf(1, "setting attribute %s on group %u to %u\n", name, which, which); - if ((aid = H5Acreate2(g, name, s->filetype, sid, H5P_DEFAULT, - H5P_DEFAULT)) < 0) { + /* Create a datatype to refer to. */ + if ((atype = H5Tcopy(H5T_C_S1)) < 0) { + H5_FAILED(); AT(); + printf("Cannot create variable length datatype.\n"); + goto error; + } + + if (H5Tset_size(atype, H5T_VARIABLE) < 0) { + H5_FAILED(); AT(); + printf("Cannot set variable length datatype.\n"); + goto error; + } + + /* Generate the VL string attribute.*/ + if ((aid = H5Acreate2(g, name, atype, sid, H5P_DEFAULT, + H5P_DEFAULT)) < 0) { + H5_FAILED(); AT(); + printf("H5Acreate2 failed.\n"); + goto error; + } + + if (H5Awrite(aid, atype, &astr_val) < 0) { + H5_FAILED(); AT(); + printf("H5Awrite failed.\n"); + goto error; + } + + if (H5Tclose(atype) < 0) { + H5_FAILED(); AT(); + printf("H5Tclose() failed\n"); + goto error; + } + if (H5Aclose(aid) < 0) { + H5_FAILED(); AT(); + printf("H5Aclose() failed\n"); + goto error; + } + + HDfree(astr_val); + + /* Writer sends a message to reader: a VL string attribute is successfully generated. + then wait for the reader to verify and send an acknowledgement message back. */ + if (s->use_named_pipes && s->attr_test == true) { + dbgf(2, "writer: write attr - ready to send the message: %d\n", s->np_notify+1); + if(np_wr_send_receive(s) == false) { + H5_FAILED(); AT(); + dbgf(2, "writer: write attr - verification failed.\n"); + goto error2; + } + } + + return true; + +error: + /* Writer needs to send an error message to the reader to stop the test*/ + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); + H5E_BEGIN_TRY { + H5Aclose(aid); + H5Tclose(atype); + } H5E_END_TRY; + + if(astr_val) + HDfree(astr_val); + +error2: + return false; +} + +/*------------------------------------------------------------------------- + * Function: del_one_attr + * + * Purpose: delete one attribute in a group. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t obj_id + * HDF5 object ID (in this file: means group ID) + * + * bool is_dense + * if the deleted attribute is for checking the dense storage + * + * bool is_vl + * if the deleted attribute is a VL string + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * The group name is "group-which" and the attribute names + * according to if this attribute is a VL string or for checking + * the dense storage or the storage transition from dense to + * compact. + * + * + * Return: Success: true + * Failure: false + * + *------------------------------------------------------------------------- +*/ + +static bool +del_one_attr(state_t *s, hid_t obj_id,bool is_dense,bool is_vl,unsigned int which) { + + char attrname[VS_ATTR_NAME_LEN]; + + /*attribute name template for the dense storage related deletion operation */ + const char* aname_format_d = "attr-d-%u-%u"; + + /*attribute name template used for general attribute deletion operation */ + const char* aname_format = "attr-%u-%u"; + + /*attribute name template used for VL string attribute deletion operation */ + const char* aname_format_vl="attr-%u"; + + dbgf(2, "writer: coming to delete the attribute.\n"); + + /* Construct the attribute name */ + if(is_dense == true) + HDsprintf(attrname, aname_format_d, which,0); + else if(is_vl == true) + HDsprintf(attrname, aname_format_vl, which,0); + else + HDsprintf(attrname, aname_format, which,0); + + /* Delete the attribute */ + if(H5Adelete(obj_id, attrname) <0) { + H5_FAILED(); AT(); + printf("H5Adelete() failed\n"); + goto error; + } + + /* Writer sends a message to reader: an attribute is successfully generated. + then wait for the reader to verify and send an acknowledgement message back. */ + if(s->use_named_pipes && s->attr_test == true) { + dbgf(2, "writer: delete attr - ready to send the message: %d\n", s->np_notify+1); + if(np_wr_send_receive(s) == false) { + H5_FAILED(); AT(); + dbgf(2, "writer: delete attr - verification failed.\n"); + goto error2; + } + } + + return true; + +error: + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); + +error2: + return false; +} + +/*------------------------------------------------------------------------- + * Function: add_del_vlstr_attr + * + * Purpose: Add a variable length string attribute + * then delete this attribute in this a group. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * The group name is "group-which" and the attribute name + * is "attr-which". + * + * + * Return: Success: true + * Failure: false + * + * Note: This is for the "remove-vstr" test. + *------------------------------------------------------------------------- +*/ + + +static bool +add_del_vlstr_attr(state_t *s, hid_t g, unsigned int which) { + + bool ret_value = false; + + /* Add a VL string attribute then delete it. */ + ret_value = add_vlstr_attr(s,g,which); + if(ret_value == true) + ret_value = del_one_attr(s,g,false,true,which); + + return ret_value; + +} + +/*------------------------------------------------------------------------- + * Function: modify_attr + * + * Purpose: Modify the value of an attribute in a group. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * const char*aname_fmt + * The attribute name template used to create unique attribute names. + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group name. The group name is "group-which". + * + * + * Return: Success: true + * Failure: false + * + *------------------------------------------------------------------------- +*/ + + +static bool +modify_attr(state_t *s, hid_t g, const char* aname_fmt,unsigned int which) { + + char attrname[VS_ATTR_NAME_LEN]; + hid_t aid = H5I_INVALID_HID; + hid_t amtype = H5I_INVALID_HID; + unsigned int modify_value; + + HDsprintf(attrname,aname_fmt,which,0); + if((aid = H5Aopen(g,attrname,H5P_DEFAULT))<0) { + H5_FAILED(); AT(); + printf("H5Aopen failed\n"); + goto error; + } + + if((amtype = H5Tget_native_type(s->filetype,H5T_DIR_ASCEND))<0) { H5_FAILED(); AT(); - printf("H5Acreate2 failed\n"); + printf("H5Tget_native_type failed\n"); goto error; } + + /* Make a large number to verify the change easily */ + modify_value = which+10000; - if (H5Awrite(aid, H5T_NATIVE_UINT, &which) < 0) { + if (H5Awrite(aid,amtype,&modify_value) <0) { H5_FAILED(); AT(); printf("H5Awrite failed\n"); goto error; } + if (H5Tclose(amtype) < 0) { + H5_FAILED(); AT(); + goto error; + } + if (H5Aclose(aid) < 0) { + H5_FAILED(); AT(); + goto error; + } + + /* Writer sends a message to reader: an attribute is successfully modified. + then wait for the reader to verify and send an acknowledgement message back.*/ + if (s->use_named_pipes && s->attr_test == true) { + dbgf(2, "writer: modify attr - ready to send the message: %d\n", s->np_notify+1); + if(np_wr_send_receive(s) == false) { + H5_FAILED(); AT(); + dbgf(2, "writer: write attr - verification failed.\n"); + /* Note: This is (mostly) because the verification failure message + * from the reader. So don't send the error message back to + * the reader. Just stop the test. */ + goto error2; + } + } + + return true; +error: + /* Writer needs to send an error message to the reader to stop the test*/ + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); + H5E_BEGIN_TRY { + H5Aclose(aid); + H5Tclose(aid); + } H5E_END_TRY; + +error2: + return false; + +} + +/*------------------------------------------------------------------------- + * Function: modify_vlstr_attr + * + * Purpose: Modify the value of an VL string attribute in a group. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group name. The group name is "group-which". + * + * + * Return: Success: true + * Failure: false + * + *------------------------------------------------------------------------- +*/ + + + +static bool +modify_vlstr_attr(state_t*s,hid_t g, unsigned int which) { + + hid_t aid = H5I_INVALID_HID; + hid_t atype = H5I_INVALID_HID; + char name[VS_ATTR_NAME_LEN]; + char *astr_val = NULL; + + astr_val = HDmalloc(VS_ATTR_NAME_LEN); + if (astr_val == NULL) { + H5_FAILED(); AT(); + printf("Allocate memory for VL string failed.\n"); + goto error; + } + + /* Change the VL string value and create the attribute name. */ + HDsprintf(astr_val,"%u%c",which,'A'); + esnprintf(name, sizeof(name), "attr-%u", which); + + dbgf(1, "setting attribute %s on group %u to %u\n", name, which, which); + + /* Create a datatype to refer to. */ + if ((atype = H5Tcopy(H5T_C_S1)) < 0) { + H5_FAILED(); AT(); + printf("Cannot create variable length datatype.\n"); + goto error; + } + + if (H5Tset_size(atype, H5T_VARIABLE) < 0) { + H5_FAILED(); AT(); + printf("Cannot set variable length datatype.\n"); + goto error; + } + + /* Open this attribute. */ + if ((aid = H5Aopen(g, name, H5P_DEFAULT))<0) { + H5_FAILED(); AT(); + printf("H5Aopen failed.\n"); + goto error; + } + + dbgf(1, "The modified VL string value is %s \n", astr_val); + + if (H5Awrite(aid, atype, &astr_val) < 0) { + H5_FAILED(); AT(); + printf("H5Awrite failed.\n"); + goto error; + } + + if (H5Tclose(atype) < 0) { + H5_FAILED(); AT(); + printf("H5Tclose() failed\n"); + goto error; + } if (H5Aclose(aid) < 0) { H5_FAILED(); AT(); - printf("H5Aclose failed\n"); + printf("H5Aclose() failed\n"); goto error; } + HDfree(astr_val); + + /* Writer sends a message to reader: a VL string attribute is successfully generated. + then wait for the reader to verify and send an acknowledgement message back. */ + if (s->use_named_pipes && s->attr_test == true) { + dbgf(2, "writer: modify vl attr - ready to send the message: %d\n", s->np_notify+1); + if(np_wr_send_receive(s) == false) { + H5_FAILED(); AT(); + dbgf(2, "writer: write attr - verification failed.\n"); + goto error2; + } + } + return true; error: H5E_BEGIN_TRY { H5Aclose(aid); + H5Tclose(atype); } H5E_END_TRY; + if(astr_val) + HDfree(astr_val); + + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); + +error2: return false; + } +/*------------------------------------------------------------------------- + * Function: add_modify_vlstr_attr + * + * Purpose: Add a variable length string attribute + * then modify this attribute in this a group. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * The group name is "group-which" and the attribute name + * is "attr-which". + * + * + * Return: Success: true + * Failure: false + * + * Note: This is for the "modify-vstr" test. + *------------------------------------------------------------------------- +*/ + +static bool +add_modify_vlstr_attr(state_t *s, hid_t g, unsigned int which) { + + bool ret_value = false; + ret_value = add_vlstr_attr(s,g,which); + if (true == ret_value) + ret_value = modify_vlstr_attr(s,g,which); + + return ret_value; +} + +/*------------------------------------------------------------------------- + * Function: add_attrs_compact + * + * Purpose: Add some attributes to the group. + * the number of attributes should be the maximal number of + * attributes that the compact storage can hold + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * hid_t gcpl + * Object creation property list ID + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * The group name is "group-which" and the attribute name + * is "attr-which". + * + * + * Return: Success: true + * Failure: false + * + * Note: This is for the "modify-vstr" test. + * For attribute compact/dense storage, check the reference + * manual of H5Pget_attr_phase_change. + *------------------------------------------------------------------------- +*/ + +static bool +add_attrs_compact(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { + + unsigned max_compact = 0; + unsigned min_dense = 0; + const char* aname_format="attr-%u-%u"; + + if(s->old_style_grp) + max_compact = 2; + else { + /* Obtain the maximal number of attributes to be stored in compact + * storage and the minimal number of attributes to be stored in + * dense storage. */ + if(H5Pget_attr_phase_change(gcpl, &max_compact, &min_dense)<0) { + H5_FAILED(); AT(); + printf("H5Pget_attr_phase_change() failed\n"); + goto error; + } + } + + /* Add max_compact attributes, these attributes are stored in + * compact storage. */ + return add_attr(s,g,which,max_compact,aname_format,which); + +error: + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); + return false; +} + +/*------------------------------------------------------------------------- + * Function: add_attrs_compact_dense + * + * Purpose: Add some attributes to the group. + * First, the number of attributes should be the maximal number + * of attributes that the compact storage can hold. + * Then, add another atribute, the storage becomes dense. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * hid_t gcpl + * Object creation property list ID + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * + * Return: Success: true + * Failure: false + * + * Note: This is for the "compact-to-dense" test. + * For attribute compact/dense storage, check the reference + * manual of H5Pget_attr_phase_change. + *------------------------------------------------------------------------- +*/ + +static bool +add_attrs_compact_dense(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { + + unsigned max_compact = 0; + unsigned min_dense = 0; + const char* aname_format="attr-d-%u-%u"; + bool ret_value = false; + + if(H5Pget_attr_phase_change(gcpl, &max_compact, &min_dense) < 0) { + H5_FAILED(); AT(); + printf("H5Pget_attr_phase_change failed\n"); + goto error; + } + + /* Add attributes, until just before converting to dense storage */ + ret_value = add_attrs_compact(s, g, gcpl, which); + + /* Add another attribute, the storage becomes dense. */ + if(ret_value == true) + ret_value = add_attr(s,g,which+max_compact,1,aname_format,which); + + return ret_value; + +error: + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); + return false; +} + +/*------------------------------------------------------------------------- + * Function: del_attrs_compact_dense_compact + * + * Purpose: delete some attributes in the group. + * The number of attributes are deleted in such a way + * that the attribute storage changes from compact to + * dense then to compact again. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * hid_t gcpl + * Object creation property list ID + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * + * Return: Success: true + * Failure: false + * + * Note: This is an internal function used by the + * "dense-del-to-compact" test. + * For attribute compact/dense storage, check the reference + * manual of H5Pget_attr_phase_change. + *------------------------------------------------------------------------- +*/ + +static bool +del_attrs_compact_dense_compact(state_t *s, + hid_t obj_id, + hid_t gcpl, + unsigned int which) { + + unsigned max_compact = 0; + unsigned min_dense = 0; + unsigned u = 0; + + char attrname[VS_ATTR_NAME_LEN]; + const char* aname_format="attr-%u-%u"; + const char* adname_format="attr-d-%u-%u"; + + /* Obtain the maximal number of attributes to be stored in compact + * storage and the minimal number of attributes to be stored in + * dense storage. */ + if (H5Pget_attr_phase_change(gcpl, &max_compact, &min_dense) < 0) { + H5_FAILED(); AT(); + printf("H5Pget_attr_phase_change failed\n"); + goto error; + } + u= max_compact +1; + + + /* delete a number of attributes so that the attribute storage just becomes dense.*/ + for(u--;u>=(min_dense-1);u--) { + HDsprintf(attrname, aname_format, which,max_compact-u); + if (H5Adelete(obj_id,attrname) < 0) { + H5_FAILED(); AT(); + printf("H5Adelete failed\n"); + goto error; + } + + /* For each attribute deletion, we want to ensure the verification + * from the reader. + * So writer sends a message to reader: an attribute is successfully deleted. + then wait for reader to verify and send an acknowledgement message back. */ + if(s->use_named_pipes && s->attr_test == true) { + dbgf(2, "writer: delete attr - ready to send the message: %d\n", s->np_notify+1); + if(np_wr_send_receive(s) == false) { + H5_FAILED(); AT(); + dbgf(2, "writer: delete attr - verification failed.\n"); + goto error2; + } + } + } + + /* The writer deletes another attribute, the storage is + * still dense. However, the attribute to be deleted + * doesn't follow the previous for loop. It may be + * in different location in the object header. Just add + * a litter variation to check if this operation is successful. + * The attribute name to be deleted is attr-max_compact+which-0 + */ + + HDsprintf(attrname,adname_format,max_compact+which,0); + if (H5Adelete(obj_id,attrname) < 0) { + H5_FAILED(); AT(); + printf("H5Adelete failed\n"); + goto error; + } + /* Again we need to notify the reader via Named pipe. */ + if(s->use_named_pipes && s->attr_test == true) { + dbgf(2, "writer: delete attr - ready to send the message: %d\n", s->np_notify+1); + if(np_wr_send_receive(s) == false) { + H5_FAILED(); AT(); + dbgf(2, "writer: delete attr - verification failed.\n"); + goto error2; + } + } + + /* The following comments are left here in case in the future we want to + * use HDF5 function to verify the attribute storage */ +#if 0 + // May H5Oget_info3 -- obtain the number of attributes. + //Check the number of attributes >=min_dense. + //We may use the internal function + //is_dense = H5O__is_attr_dense_test(dataset) to check if it is dense in the future. + // +#endif + + return true; + +error: + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); + +error2: + return false; +} + +/*------------------------------------------------------------------------- + * Function: add_del_attrs_compact + * + * Purpose: Add some attributes to the group and then delete one attribute. + * First, the number of attributes to be added should be the + * maximal number of attributes that the compact storage can hold. + * Then, delete one atribute, the storage is still compact. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * hid_t gcpl + * Object creation property list ID + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * + * Return: Success: true + * Failure: false + * + * Note: This is for the "compact-del" test. + * For attribute compact/dense storage, check the reference + * manual of H5Pget_attr_phase_change. + *------------------------------------------------------------------------- +*/ + +static bool +add_del_attrs_compact(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { + + bool ret_value = false; + ret_value = add_attrs_compact(s, g, gcpl, which); + if(ret_value == true) { + dbgf(2, "writer: before deleting the attribute.\n"); + ret_value = del_one_attr(s,g,false,false,which); + } + + return ret_value; + +} + +/*------------------------------------------------------------------------- + * Function: add_del_attrs_compact_dense + * + * Purpose: Add some attributes to the group and then delete one attribute. + * First, the number of attributes to be added exceeds + * the maximal number of attributes that the compact storage can hold. + * The storage changes from compact to dense. + * Then, delete one atribute, the storage is still dense. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * hid_t gcpl + * Object creation property list ID + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * + * Return: Success: true + * Failure: false + * + * Note: This is for the "dense-del" test. + * For attribute compact/dense storage, check the reference + * manual of H5Pget_attr_phase_change. + *------------------------------------------------------------------------- +*/ + + +static bool +add_del_attrs_compact_dense(state_t *s, hid_t g, hid_t gcpl, unsigned int which) { + + bool ret_value = false; + unsigned max_compact = 0; + unsigned min_dense = 0; + + if( H5Pget_attr_phase_change(gcpl, &max_compact, &min_dense) < 0) { + H5_FAILED(); AT(); + printf("H5Pget_attr_phase_change failed\n"); + goto error; + } + + ret_value = add_attrs_compact_dense(s,g,gcpl,which); + if(ret_value == true) + ret_value = del_one_attr(s,g,true,false,which+max_compact); + + return ret_value; + +error: + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); + return false; + +} + + +/*------------------------------------------------------------------------- + * Function: add_del_attrs_compact_dense_compact + * + * Purpose: Add attributes to the group and then delete some of them. + * First, the number of attributes to be added exceeds + * the maximal number of attributes that the compact storage can hold. + * The storage changes from compact to dense. + * Then, delete some attributes, the storage changes from + * dense to compact again. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * hid_t gcpl + * Object creation property list ID + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * + * Return: Success: true + * Failure: false + * + * Note: This is for the "dense-del-to-compact" test. + * For attribute compact/dense storage, check the reference + * manual of H5Pget_attr_phase_change. + *------------------------------------------------------------------------- +*/ + + +static bool +add_del_attrs_compact_dense_compact(state_t *s, + hid_t g, + hid_t gcpl, + unsigned int which) { + + bool ret_value = false; + ret_value = add_attrs_compact_dense(s,g,gcpl,which); + if(ret_value == true) + ret_value = del_attrs_compact_dense_compact(s,g,gcpl,which); + + return ret_value; +} + +/*------------------------------------------------------------------------- + * Function: add_modify_default_group_attr + * + * Purpose: Add an attribute then modify the value to a group. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * The group name is "group-which" and the attribute name + * is "attr-which". + * + * + * Return: Success: true + * Failure: false + * + * Note: This function is used for the "modify" storage test. + *------------------------------------------------------------------------- +*/ + + +static bool +add_modify_default_group_attr(state_t *s, hid_t g, unsigned int which) { + + bool ret_value = false; + const char* aname_format ="attr-%u"; + ret_value = add_default_group_attr(s,g,which); + if(ret_value == true) + ret_value = modify_attr(s,g,aname_format,which); + return ret_value; + +} + +/*------------------------------------------------------------------------- + * Function: add_group_attribute + * + * Purpose: Check the attribute test pattern and then call the + * correponding test function.. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * hid_t gcpl + * Object creation property list ID + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group and attribute names. + * + * + * Return: Success: true + * Failure: false + * + * Note: This is called by the write_group() function. + *------------------------------------------------------------------------- +*/ + +static bool +add_group_attribute(state_t *s, hid_t g, hid_t gcpl, unsigned int which) +{ + + bool ret_value = false; + char test_pattern = s->at_pattern; + + switch (test_pattern) { + case 'c': + ret_value = add_attrs_compact(s, g, gcpl, which); + break; + case 't': + ret_value = add_attrs_compact_dense(s, g, gcpl, which); + break; + case 'C': + ret_value = add_del_attrs_compact(s, g, gcpl, which); + break; + case 'D': + ret_value = add_del_attrs_compact_dense(s, g, gcpl, which); + break; + case 'T': + ret_value = add_del_attrs_compact_dense_compact(s, g, gcpl, which); + break; + case 'M': + ret_value = add_modify_default_group_attr(s, g, which); + break; + case 'v': + ret_value = add_vlstr_attr(s,g, which); + break; + case 'r': + ret_value = add_del_vlstr_attr(s, g, which); + break; + case 'm': + ret_value = add_modify_vlstr_attr(s,g, which); + break; + case 'd': + case ' ': + default: + ret_value = add_default_group_attr(s, g, which); + break; + } + return ret_value; + +} + +/*------------------------------------------------------------------------- + * Function: write_group + * + * Purpose: Create a group and carry out attribute operations(add,delete etc.) + * according to the attribute test pattern. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * unsigned int which + * The number of iterations for group creation + * + * + * Return: Success: true + * Failure: false + * + * Note: This is called by the main() function. + *------------------------------------------------------------------------- +*/ + static bool write_group(state_t *s, unsigned int which) { char name[sizeof("/group-9999999999")]; hid_t g = H5I_INVALID_HID; + hid_t gcpl = H5I_INVALID_HID; bool result = true; + H5G_info_t group_info; if (which >= s->nsteps) { H5_FAILED(); AT(); - printf("group order is out of bounds\n"); + printf("Number of created groups is out of bounds\n"); goto error; } esnprintf(name, sizeof(name), "/group-%d", which); - if ((g = H5Gcreate2(s->file, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) { + if(s->old_style_grp) + gcpl = H5P_DEFAULT; + else { + gcpl = H5Pcreate(H5P_GROUP_CREATE); + if(gcpl <0) { + H5_FAILED(); AT(); + printf("H5Pcreate failed\n"); + goto error; + } + + /* If we test the dense storage, change the attribute phase. */ + if(s->at_pattern =='d') { + if(H5Pset_attr_phase_change(gcpl, 0, 0) <0) { + H5_FAILED(); AT(); + printf("H5Pset_attr_phase_change failed for the dense storage.\n"); + goto error; + } + } + } + + if ((g = H5Gcreate2(s->file, name, H5P_DEFAULT, gcpl, + H5P_DEFAULT)) < 0) { H5_FAILED(); AT(); printf("H5Gcreate2 failed\n"); goto error; } - if (s->asteps != 0 && which % s->asteps == 0) - result = add_group_attribute(s, g, s->one_by_one_sid, which); + if(H5Gget_info(g,&group_info) <0) { + H5_FAILED(); AT(); + printf("H5Gget_info failed\n"); + goto error; + } + + if(s->old_style_grp) { + if(group_info.storage_type != H5G_STORAGE_TYPE_SYMBOL_TABLE) { + H5_FAILED(); AT(); + printf("Old-styled group test: but the group is not in old-style. \n"); + goto error; + } + dbgf(2,"Writer: group is created with the old-style.\n"); + } + else { + if(group_info.storage_type == H5G_STORAGE_TYPE_SYMBOL_TABLE) { + H5_FAILED(); AT(); + printf("The created group should NOT be in old-style . \n"); + goto error; + } + dbgf(2,"Writer: group is created with the new-style.\n"); + + } + /* If an attribute test is turned on and named pipes are used, + * the writer should send and receive messages after the group creation. + * This will distinguish an attribute operation error from an + * group creation error. + * Writer sends a message to reader: an attribute is successfully generated. + * then wait for the reader to verify and send an acknowledgement message back.*/ + if (s->use_named_pipes && s->attr_test == true) { + dbgf(2, "writer: ready to send the message: %d\n", s->np_notify+1); + if(np_wr_send_receive(s) == false) { + H5_FAILED(); AT(); + /* Note: This is (mostly) because the verification failure message + * from the reader. So don't send the error message back to + * the reader. Just stop the test. */ + goto error2; + } + } + + /* Then carry out the attribute operation. */ + if (s->asteps != 0 && which % s->asteps == 0) + result = add_group_attribute(s, g, gcpl,which); if (H5Gclose(g) < 0) { H5_FAILED(); AT(); @@ -244,26 +1672,454 @@ write_group(state_t *s, unsigned int which) goto error; } + if(!s->old_style_grp && H5Pclose(gcpl) <0) { + H5_FAILED(); AT(); + printf("H5Pclose failed\n"); + goto error; + } + return result; error: + /* Writer needs to send an error message to the reader to stop the test*/ + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,true); + +error2: + H5E_BEGIN_TRY { H5Gclose(g); + if(!s->old_style_grp) + H5Pclose(gcpl); } H5E_END_TRY; return false; + } +/*------------------------------------------------------------------------- + * Function: check_attr_storage_type + * + * Purpose: Check if the attribute storage type is correct + * + * Parameters: hid_t oid + * HDF5 object ID (in this file: means group ID) + * + * bool is_compact + * true if the attribute is stored in compact storage + * false if the attribute is stored in dense storage + * + * + * Return: Success: true + * Failure: false + * + *------------------------------------------------------------------------- +*/ static bool -verify_group_attribute(hid_t g, unsigned int which) +check_attr_storage_type(hid_t g, + bool is_compact) { + + H5O_native_info_t ninfo; + + /* Get the object information */ + if(H5Oget_native_info(g, &ninfo, H5O_NATIVE_INFO_HDR|H5O_NATIVE_INFO_META_SIZE) < 0) { + H5_FAILED(); AT(); + printf("H5Oget_native_info failed\n"); + goto error; + } + + if(is_compact) { + if(ninfo.meta_size.attr.index_size != 0 || + ninfo.meta_size.attr.heap_size != 0) { + H5_FAILED(); AT(); + printf("Should be in compact storage,but it is not.\n"); + goto error; + } + } + else { + if(ninfo.meta_size.attr.index_size == 0 || + ninfo.meta_size.attr.heap_size == 0) { + H5_FAILED(); AT(); + printf("Should be in dense storage,but it is not.\n"); + goto error; + } + } + + return true; + +error: + return false; + +} + + +/*------------------------------------------------------------------------- + * Function: vrfy_attr + * + * Purpose: Verify is a group attribute value is as expected. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t oid + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The number of iterations for group creation, use to generate + * newly created group name. The group name is "group-which". + * + * const char*aname + * The attribute name + * + * unsigned int g_which + * This parameter is used to generate correct group name in a key + * debugging message. + * + * bool check_storage + * a flag to indicate if the storage check is on + * + * bool is_compact + * true if the attribute is stored in compact storage + * false if the attribute is stored in dense storage + * Note: this parameter is not used if the check_storage + * is set to false. + * + * + * Return: Success: true + * Failure: false + * + *------------------------------------------------------------------------- +*/ + +static bool +vrfy_attr(state_t *s, + hid_t g, + unsigned int which, + const char* aname, + unsigned int g_which, + bool check_storage, + bool is_compact) { + + unsigned int read_which; + hid_t aid = H5I_INVALID_HID; + hid_t amtype = H5I_INVALID_HID; + + /* The reader receives a message from the writer.Then sleep + * for a few ticks or stop the test if receiving an error + * message. + */ + if(s->use_named_pipes && true == s->attr_test) { + if(false == np_rd_receive(s)) { + H5_FAILED(); AT(); + /* Since receiving the error message from the writer, + * just stop the test. */ + goto error2; + } + decisleep(s->tick_len * s->update_interval); + dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); + } + + /* Go ahead to read the attribute. */ + dbgf(1, "verifying attribute %s on group %u equals %u\n", aname, g_which, + which); + + if ((amtype = H5Tget_native_type(s->filetype,H5T_DIR_ASCEND)) <0) { + H5_FAILED(); AT(); + printf("H5Tget_native_type failed\n"); + goto error; + } + + if ((aid = H5Aopen(g, aname, H5P_DEFAULT)) < 0) { + H5_FAILED(); AT(); + printf("H5Aopen failed\n"); + goto error; + } + + if (H5Aread(aid, amtype, &read_which) < 0) { + H5_FAILED(); AT(); + printf("H5Aread failed\n"); + goto error; + } + + if (H5Aclose(aid) < 0) { + H5_FAILED(); AT(); + printf("H5Aclose failed\n"); + goto error; + } + + if(read_which != which) { + H5_FAILED(); AT(); + dbgf(2, "reader: the add_attribute verfication failed,expected value is %d\n", which); + dbgf(2, "reader: the add_attribute verfication failed, the value is %d\n", read_which); + printf("The add_attribute verification failed\n"); + goto error; + } + + + if(!s->old_style_grp && check_storage == true) { + if(false == check_attr_storage_type(g,is_compact)) { + H5_FAILED(); AT(); + printf("The attribute storage type is wrong. \n"); + goto error; + } + dbgf(2, "reader: finish checking the storage type: %d\n", s->np_notify); + + } + + /* If the read value is expected, send back an OK message to the writer. */ + if(s->use_named_pipes && s->attr_test == true) { + if(np_rd_send(s)==false) + goto error; + dbgf(2, "reader: finish sending back the message: %d\n", s->np_notify); + } + return true; + +error: + H5E_BEGIN_TRY { + H5Tclose(amtype); + H5Aclose(aid); + } H5E_END_TRY; + + /* Send back an error message to the writer so that the writer can stop. */ + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,false); +error2: + return false; + +} + +/*------------------------------------------------------------------------- + * Function: verify_default_group_attr + * + * Purpose: Check if the reader can retrieve the correct value of a + * group attribute corrected by the writer. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The expected attribute value. It is also used to construct the + * group name. + * + * Return: Success: true + * Failure: false + * + * Note: This function is used for the "dense" storage test. + * It is also used by the group-only test. + *------------------------------------------------------------------------- +*/ + +static bool +verify_default_group_attr(state_t*s,hid_t g, unsigned int which) { + char attrname[VS_ATTR_NAME_LEN]; + const char* aname_format = "attr-%u"; + HDsprintf(attrname, aname_format, which); + return vrfy_attr(s,g,which,attrname,which,false,true); + +} + +/*------------------------------------------------------------------------- + * Function: verify_modify_attr + * + * Purpose: Check if the reader can retrieve the correct value of + * an attribute in a group, first the original value then + * the modified value. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The expected attribute value. It is also used to construct the + * group name. The modified attribute value can be derived from + * the expected attribute value. + * + * Return: Success: true + * Failure: false + * + * Note: This function is used for the "modified" test. + *------------------------------------------------------------------------- +*/ + +static bool +verify_modify_attr(state_t *s, hid_t g, unsigned int which) { + + bool ret = false; + const char* aname_fmt ="attr-%u"; unsigned int read_which; - hid_t aid; - char name[sizeof("attr-9999999999")]; + hid_t aid = H5I_INVALID_HID; + hid_t amtype = H5I_INVALID_HID; + char attrname[VS_ATTR_NAME_LEN]; + + /* First verify the original attribute value */ + ret = verify_default_group_attr(s,g,which); + + /* Then the modified value */ + if(ret == true) { + + /* The reader receives a message from the writer.Then sleep + * for a few ticks or stop the test if receiving an error + * message. + */ + if(s->use_named_pipes && true == s->attr_test) { + if(false == np_rd_receive(s)) { + H5_FAILED(); AT(); + goto error2; + } + decisleep(s->tick_len * s->update_interval); + dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); + } + + /* Go ahead to read the attribute. */ + esnprintf(attrname, sizeof(attrname), aname_fmt, which); + if ((amtype = H5Tget_native_type(s->filetype,H5T_DIR_ASCEND)) < 0) { + H5_FAILED(); AT(); + printf("H5Tget_native_type failed\n"); + goto error; + } + + if ((aid = H5Aopen(g, attrname, H5P_DEFAULT)) < 0) { + H5_FAILED(); AT(); + printf("H5Aopen failed\n"); + goto error; + } + + if (H5Aread(aid, amtype, &read_which) < 0) { + H5_FAILED(); AT(); + printf("H5Aread failed\n"); + goto error; + } + + if(H5Tclose(amtype) <0) { + H5_FAILED(); AT(); + printf("H5Tclose failed.\n"); + goto error; + } + + if (H5Aclose(aid) < 0) { + H5_FAILED(); AT(); + printf("H5Aclose failed\n"); + goto error; + } + + /* verify the modified value */ + if(read_which != (which+10000)) { + H5_FAILED(); AT(); + dbgf(2, "reader: the modified_attr() expected value is %d\n", (-1)*(int)which); + dbgf(2, "reader: the modified_attr() actual value is %d\n", read_which); + printf("The modify_attribute verification failed.\n"); + goto error; + } + + /* The reader sends an OK message back to the writer. */ + if(s->use_named_pipes && s->attr_test == true) { + if(np_rd_send(s)==false) + goto error2; + dbgf(2, "reader: modify_attr finish sending back the message: %d\n", s->np_notify); + } + return true; + + } + return false; + +error: + H5E_BEGIN_TRY { + H5Aclose(aid); + H5Tclose(amtype); + } H5E_END_TRY; + + /* The reader needs to send an error message back to the writer to stop the test.*/ + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,false); + +error2: + + return false; +} + +/*------------------------------------------------------------------------- + * Function: verify_group_vlstr_attr + * + * Purpose: Check if the reader can retrieve the correct value of + * a variable length string attribute created by the writer. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * Use to derieve the expected attribute value. It is also used + * to construct the group name. + * + * bool vrfy_mod + * true if this function is used for the modified VL string test. + * false if this function is just used for the VL string test. + * + * Return: Success: true + * Failure: false + * + * Note: This function is an internal function used by + * both the "vlstr" and the "modify-vstr" tests. + *------------------------------------------------------------------------- +*/ + + +static bool +verify_group_vlstr_attr(state_t*s, hid_t g, unsigned int which, bool vrfy_mod) +{ + hid_t aid = H5I_INVALID_HID; + hid_t atype = H5I_INVALID_HID; + char name[VS_ATTR_NAME_LEN]; + + char *astr_val_exp; + char * astr_val; + + /* The reader receives a message from the writer.Then sleep + * for a few ticks or stop the test if the received message + * is an error message. + */ + if(s->use_named_pipes && true == s->attr_test) { + if(false == np_rd_receive(s)) { + H5_FAILED(); AT(); + goto error2; + } + decisleep(s->tick_len * s->update_interval); + dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); + } + + /* Go ahead to read the VL string attribute. */ + astr_val_exp = HDmalloc(VS_ATTR_NAME_LEN); + if (astr_val_exp == NULL) { + H5_FAILED(); AT(); + printf("Allocate memory for expected buffer failed.\n"); + goto error; + } esnprintf(name, sizeof(name), "attr-%u", which); - dbgf(1, "verifying attribute %s on group %u equals %u\n", name, which, which); + /* Construct the expected VL string value,depending if + * it is the modified value or the original value. */ + if(vrfy_mod == true) + HDsprintf(astr_val_exp,"%u%c",which,'A'); + else + HDsprintf(astr_val_exp,"%u",which); + + dbgf(1, "verifying attribute %s on group %u equals %u\n", name, which, + which); + + dbgf(1,"expected vl attr is= %s\n",astr_val_exp); if ((aid = H5Aopen(g, name, H5P_DEFAULT)) < 0) { H5_FAILED(); AT(); @@ -271,57 +2127,766 @@ verify_group_attribute(hid_t g, unsigned int which) goto error; } - if (H5Aread(aid, H5T_NATIVE_UINT, &read_which) < 0) { + /* Create a VL string datatype */ + if ((atype = H5Tcopy(H5T_C_S1)) < 0) { H5_FAILED(); AT(); - printf("H5Aread failed\n"); + printf("Cannot create variable length datatype.\n"); goto error; } - if (read_which != which) { + if (H5Tset_size(atype, H5T_VARIABLE) < 0) { H5_FAILED(); AT(); - printf("H5Aread wrong value\n"); + printf("Cannot set variable length datatype.\n"); goto error; } + if (H5Aread(aid, atype, &astr_val) < 0) { + H5_FAILED(); AT(); + printf("Cannot read the attribute.\n"); + goto error; + } + + dbgf(1,"read attr is= %s\n",astr_val); + if (HDstrcmp(astr_val, astr_val_exp) != 0) { + H5_FAILED(); AT(); + dbgf(2, "reader: the vl add_attribute verfication failed,expected value is %s\n", astr_val_exp); + dbgf(2, "reader: the vl add_attribute verfication failed, the value is %s\n", astr_val); + printf("The vl add_attribute verification failed\n"); + goto error; + } + + if(H5Tclose(atype) <0) { + H5_FAILED(); AT(); + printf("H5Tclose failed.\n"); + goto error; + } + if (H5Aclose(aid) < 0) { H5_FAILED(); AT(); - printf("H5Aread failed\n"); + printf("H5Aclose failed.\n"); goto error; } + H5free_memory(astr_val); + HDfree(astr_val_exp); + + /* Reader sends an OK message back to the reader */ + if(s->use_named_pipes && s->attr_test == true) { + if(np_rd_send(s)==false) + goto error2; + dbgf(2, "reader: finish sending back the message: %d\n", s->np_notify); + } + return true; error: H5E_BEGIN_TRY { H5Aclose(aid); + H5Tclose(atype); } H5E_END_TRY; + if(astr_val) + H5free_memory(astr_val); + if(astr_val_exp) + HDfree(astr_val_exp); + + /* The reader sends an error message to the writer to stop the test.*/ + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,false); + +error2: return false; + +} + +/*------------------------------------------------------------------------- + * Function: verify_del_one_attr + * + * Purpose: verify if an attribute is successfully deleted. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * const char* aname + * The name of the attribute to be deleted. + * + * Return: Success: true + * Failure: false + * + * Note: This is an internal function used by "remove-vlstr", + * "compact-del","dense-del",dense-del-to-compact" tests. + *------------------------------------------------------------------------- +*/ + + +static bool +verify_del_one_attr(state_t *s, + hid_t g, + const char *aname, + bool check_storage, + bool is_compact) { + + + htri_t attr_exists = FALSE; + + /* The reader receives a message from the writer.Then sleep + * for a few ticks or stop the test if the received message + * is an error message. + */ + if(s->use_named_pipes && true == s->attr_test) { + if(false == np_rd_receive(s)) { + H5_FAILED(); AT(); + goto error2; + } + decisleep(s->tick_len * s->update_interval); + dbgf(1, "Reader: finish reading the message: %d\n",s->np_notify); + } + + /* Check if the deleted attribute still exists. */ + attr_exists = H5Aexists_by_name(g,".",aname,H5P_DEFAULT); + if(attr_exists == FALSE) { + dbgf(1,"verify_del_attrs_compact() test: \n"); + dbgf(1," attribute %s is successfully deleted. \n",aname); + } + else if(attr_exists == TRUE) { + dbgf(1,"verify_del_attrs_compact() test failed \n"); + goto error; + } + else{ + dbgf(1,"H5Aexists_by_name failed \n"); + goto error; + } + + if(!s->old_style_grp && check_storage == true) { + if(false == check_attr_storage_type(g,is_compact)) { + H5_FAILED(); AT(); + printf("The attribute storage type is wrong. \n"); + goto error; + } + dbgf(2, "reader: finish checking the storage type: %d\n", s->np_notify); + + } + + /* Reader sends an OK message back to the reader */ + if(s->use_named_pipes && s->attr_test == true) { + if(np_rd_send(s)==false) + goto error; + dbgf(2, "reader: finish sending back the message: %d\n", s->np_notify); + } + + return true; +error: + /* The reader sends an error message to the writer to stop the test.*/ + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,false); + +error2: + return false; +} + +/*------------------------------------------------------------------------- + * Function: verify_remove_vlstr_attr + * + * Purpose: Verify if an variable length string attribute is + * successfully deleted by the writer. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * Use to derieve the expected attribute value added + * by the writer. It is also used to construct + * the attribute name. + * + * + * Return: Success: true + * Failure: false + * + * Note: This function is for the "remove-vstr" test. + * Also note this function first verifies if + * a variable length attribute is added then + * it verifies if it is deleted successfully. + *------------------------------------------------------------------------- +*/ + +static bool +verify_remove_vlstr_attr(state_t* s,hid_t g, unsigned int which) +{ + bool ret = false; + char attrname[VS_ATTR_NAME_LEN]; + const char* aname_format = "attr-%u"; + + ret = verify_group_vlstr_attr(s,g,which,false); + if(ret == true) { + HDsprintf(attrname,aname_format,which); + ret = verify_del_one_attr(s,g,attrname,false,false); + } + return ret; +} + +/*------------------------------------------------------------------------- + * Function: verify_modify_vlstr_attr + * + * Purpose: Verify if an variable length string attribute is + * successfully modified by the writer. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * Use to derieve the expected attribute value added + * by the writer. It is also used to construct + * the attribute name. + * + * + * Return: Success: true + * Failure: false + * + * Note: This function is for the "modify-vstr" test. + * Also note this function first verifies if + * a variable length attribute is added then + * it verifies if it is modified successfully. + *------------------------------------------------------------------------- +*/ + +static bool +verify_modify_vlstr_attr(state_t *s, hid_t g, unsigned int which){ + + bool ret = false; + + ret = verify_group_vlstr_attr(s,g,which,false); + if(ret == true) + ret = verify_group_vlstr_attr(s,g,which,true); + return ret; + +} + +/*------------------------------------------------------------------------- + * Function: verify_attrs_compact + * + * Purpose: verify if attributes are successfully added for the compact + * storage. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigend max_c + * The maximal number of attributes the compact storage + * can hold + * + * unsigned int which + * Use to derieve the expected attribute value added + * by the writer. It is also used to construct the + * attribute names. + * + * + * Return: Success: true + * Failure: false + * + * Note: This function is used by the "compact" test. + *------------------------------------------------------------------------- +*/ + +static bool +verify_attrs_compact(state_t *s, hid_t g, unsigned max_c, unsigned int which) { + + unsigned u; + bool ret = true; + const char* aname_format = "attr-%u-%u"; + char attrname[VS_ATTR_NAME_LEN]; + + /* Need to verify the added attribute one by one. */ + for (u = 0; u < max_c; u++) { + + HDsprintf(attrname, aname_format, which,u); + if(false == vrfy_attr(s,g,u+which,attrname,which,true,true)) { + ret = false; + break; + } + + } + return ret; + } +/*------------------------------------------------------------------------- + * Function: verify_attrs_compact_dense + * + * Purpose: verify if attributes are successfully added first in the + * compact storage then in the dense storage. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigend max_c + * The maximal number of attributes the compact storage + * can hold + * + * unsigned int which + * Use to derieve the expected attribute value added + * by the writer. It is also used to construct + * attribute names. + * + * + * Return: Success: true + * Failure: false + * + * Note: This function is used by the "compact-dense" test. + *------------------------------------------------------------------------- +*/ + + +static bool +verify_attrs_compact_dense(state_t *s, hid_t g, unsigned max_c, unsigned int which) { + + const char* aname_format = "attr-d-%u-%u"; + char attrname[VS_ATTR_NAME_LEN]; + + bool ret = verify_attrs_compact(s,g,max_c,which); + + if(ret == true) { + + /* Now the storage is in dense. Verify if the + * retrieved value is correct. */ + HDsprintf(attrname, aname_format, max_c+which,0); + ret = vrfy_attr(s,g,which+max_c,attrname,which,true,false); + if(ret == false) + dbgf(1,"verify_attrs_compact_dense failed \n"); + + } + return ret; +} + +/*------------------------------------------------------------------------- + * Function: verify_del_attrs_compact + * + * Purpose: verify if an attribute in compact storage is successfully + * deleted. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigend max_c + * The maximal number of attributes the compact storage + * can hold + * + * unsigned int which + * Use to derieve the expected attribute value added + * by the writer. It is also used to construct + * attribute names. + * + * + * Return: Success: true + * Failure: false + * + * Note: This function is used by the "compact-del" test. + * Also note this function first verifies if + * attributes are successfully added in compact storage then + * it verifies if one added attribute is deleted successfully. + *------------------------------------------------------------------------- +*/ + +static bool +verify_del_attrs_compact(state_t *s, hid_t g, unsigned max_c, unsigned int which) { + + const char* aname_format = "attr-%u-%u"; + char attrname[VS_ATTR_NAME_LEN]; + + bool ret = verify_attrs_compact(s,g,max_c,which); + + if(ret == true) { + /* The writer only deletes the attribute attr-which-0 */ + HDsprintf(attrname,aname_format,which,0); + ret = verify_del_one_attr(s,g,attrname,true,true); + } + + return ret; +} + +/*------------------------------------------------------------------------- + * Function: verify_del_attrs_compact_dense + * + * Purpose: verify if an attribute in dense storage is successfully + * deleted. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigend max_c + * The maximal number of attributes the compact storage + * can hold + * + * unsigned int which + * Use to derieve the expected attribute value added + * by the writer. It is also used to construct + * attribute names. + * + * + * Return: Success: true + * Failure: false + * + * Note: This function is used by the "dense-del" test. + * Also note this function first verifies if + * attributes are successfully added in compact storage then + * in dense storage. Afterwards, + * it verifies if one added attribute is deleted successfully. + *------------------------------------------------------------------------- +*/ + + +static bool +verify_del_attrs_compact_dense(state_t *s, + hid_t g, + unsigned max_c, + unsigned int which) { + + const char* aname_format = "attr-d-%u-%u"; + char attrname[VS_ATTR_NAME_LEN]; + + bool ret = verify_attrs_compact_dense(s,g,max_c,which); + + if(ret == true) { + /* The writer only deletes the attribute attr-d-which-0 */ + HDsprintf(attrname,aname_format,max_c+which,0); + ret = verify_del_one_attr(s,g,attrname,true,false); + } + + return ret; + +} + +/*------------------------------------------------------------------------- + * Function: verify_del_attrs_compact_dense_compact + * + * Purpose: verify that the attributes are deleted successfully + * even the attribute storage changes from dense to + * compact. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigend max_c + * The maximal number of attributes the compact storage + * can hold + * + * unsigend min_d + * The minimal number of attributes to be stored in + * dense storage + * + * unsigned int which + * Use to derieve the expected attribute value added + * by the writer. It is also used to construct + * attribute names. + * + * + * Return: Success: true + * Failure: false + * + * Note: This function is used by the "dense-del-to-compact" test. + * Also note this function first verifies if + * attributes are successfully added in compact storage then + * in dense storage. Afterwards, + * it verifies if some added attributes are deleted successfully + * until the storage changes from dense to compact. + *------------------------------------------------------------------------- +*/ + + +static bool +verify_del_attrs_compact_dense_compact(state_t *s, + hid_t g, + unsigned max_c, + unsigned min_d, + unsigned int which) { + + unsigned u; + const char* aname_format = "attr-%u-%u"; + char attrname[VS_ATTR_NAME_LEN]; + + /* Verify the attributes are added correctly from + * compact to dense storage*/ + bool ret = verify_attrs_compact_dense(s,g,max_c,which); + + if(ret == true) { + + /* Then verify the deletion of attributes + * from dense to compact. + */ + u = max_c + 1; + for(u--;u>=(min_d-1);u--) { + HDsprintf(attrname, aname_format, which,max_c-u); + if(u==(min_d-1)) + ret = verify_del_one_attr(s,g,attrname,true,true); + else + ret = verify_del_one_attr(s,g,attrname,true,false); + } + + /* Just verify one more deleted attribute by the writer. + The storage is still compact. */ + HDsprintf(attrname,aname_format,max_c+which,0); + ret = verify_del_one_attr(s,g,attrname,true,true); + } + + return ret; + +} + +/*------------------------------------------------------------------------- + * Function: verify_group_attribute + * + * Purpose: Check the attribute test pattern and then call the + * correponding verification function. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * hid_t g + * HDF5 object ID (in this file: means group ID) + * + * unsigned int which + * The number of iterations for group creation, use to generate + * group and attribute names. + * + * + * Return: Success: true + * Failure: false + * + * Note: This is called by the verify_group() function. + *------------------------------------------------------------------------- +*/ + + +static bool +verify_group_attribute(state_t *s, hid_t g, unsigned int which) +{ + char test_pattern = s->at_pattern; + bool ret = false; + unsigned max_compact = 0; + unsigned min_dense = 0; + hid_t gcpl = H5I_INVALID_HID; + + /* For tests "compact","compact-to-dense","compact-del", + * "dense-del", "dense-del-to-compact", + * the maximal number of attributes for the compact storage + * and the minimal number of attributes for the dense storage + * are needed. So obtain them here + * When testing the old-style group creation case, only max_compact + * matters. To reduce the testing time, we set max_compact to 2.*/ + switch (test_pattern) { + case 'c': + case 't': + case 'C': + case 'D': + case 'T': + if(s->old_style_grp) + max_compact = 2; + else { + if((gcpl = H5Gget_create_plist(g)) < 0) { + H5_FAILED(); AT(); + printf("H5Gget_create_plist failed\n"); + goto error; + } + if (H5Pget_attr_phase_change(gcpl,&max_compact,&min_dense) < 0) { + H5_FAILED(); AT(); + printf("H5Pget_attr_phase_change failed\n"); + goto error; + } + if(H5Pclose(gcpl) < 0) { + H5_FAILED(); AT(); + printf("H5Pclose failed\n"); + goto error; + } + } + break; + case 'v': + case 'd': + case 'M': + case 'm': + case 'r': + case ' ': + default: + break; + } + + /* Distribute the verification test. */ + switch (test_pattern) { + case 'c': + ret = verify_attrs_compact(s, g, max_compact, which); + break; + case 't': + ret = verify_attrs_compact_dense(s, g, max_compact, which); + break; + case 'C': + ret = verify_del_attrs_compact(s, g, max_compact, which); + break; + case 'D': + ret = verify_del_attrs_compact_dense(s, g, max_compact, which); + break; + case 'T': + ret = verify_del_attrs_compact_dense_compact(s, g, max_compact, min_dense, which); + break; + case 'M': + ret = verify_modify_attr(s, g, which); + break; + case 'v': + ret = verify_group_vlstr_attr(s,g, which,false); + break; + case 'r': + ret = verify_remove_vlstr_attr(s,g, which); + break; + case 'm': + ret = verify_modify_vlstr_attr(s,g, which); + break; + case 'd': + case ' ': + default: + ret = verify_default_group_attr(s, g, which); + break; + } + + return ret; + +error: + /* Still to finish the handshaking */ + if(s->use_named_pipes && s->attr_test == true) { + np_rd_receive(s); + np_send_error(s,false); + } + return false; +} + +/*------------------------------------------------------------------------- + * Function: verify_group + * + * Purpose: verify the success of group creation and + * carry out the test for attribute operations(add,delete etc.) + * according to the attribute test pattern. + * + * Parameters: state_t *s + * The struct that stores information of HDF5 file, named pipe + * and some VFD SWMR configuration parameters + * + * unsigned int which + * The number of iterations for group creation + * + * + * Return: Success: true + * Failure: false + * + * Note: This is called by the main() function. + *------------------------------------------------------------------------- +*/ + + + static bool verify_group(state_t *s, unsigned int which) { char name[sizeof("/group-9999999999")]; hid_t g = H5I_INVALID_HID; bool result = true; + H5G_info_t group_info; + + /* The reader receives a message from the writer.Then sleep + * for a few ticks or stop the test if the received message + * is an error message. + */ + if(s->use_named_pipes && true == s->attr_test) { + + if(false == np_rd_receive(s)) { + H5_FAILED(); AT(); + goto error2; + } + decisleep(s->tick_len * s->update_interval); + dbgf(1, "reader: finish reading the message: %d\n",s->np_notify); + + } if (which >= s->nsteps) { H5_FAILED(); AT(); - printf("Group order is out of bounds\n"); + printf("Number of created groups is out of bounds\n"); goto error; } esnprintf(name, sizeof(name), "/group-%d", which); - if ((g = H5Gopen(s->file, name, H5P_DEFAULT)) < 0) { + + if((g = H5Gopen(s->file, name, H5P_DEFAULT)) <0) { H5_FAILED(); AT(); printf("H5Gopen failed\n"); goto error; } + if(H5Gget_info(g,&group_info) <0) { + H5_FAILED(); AT(); + printf("H5Gget_info failed\n"); + goto error; + } + + dbgf(2,"Storage info is %d\n",group_info.storage_type); + if(s->old_style_grp) { + if(group_info.storage_type != H5G_STORAGE_TYPE_SYMBOL_TABLE) { + H5_FAILED(); AT(); + printf("Reader - Old-styled group: but the group is not in old-style. \n"); + goto error; + } + dbgf(2,"Reader: verify that the group is created with the old-style.\n"); + } + else { + if(group_info.storage_type == H5G_STORAGE_TYPE_SYMBOL_TABLE) { + H5_FAILED(); AT(); + printf("Reader - The created group should NOT be in old-style . \n"); + goto error; + } + dbgf(2,"Reader: verify that the group is created with the new-style.\n"); + + } + + /* Reader sends an OK message back to the reader */ + if(s->use_named_pipes && s->attr_test == true) { + + if(np_rd_send(s)==false) + goto error; + dbgf(1, "Reader: finish sending back the message: %d\n",s->np_notify); + + } + + /* Check if we need to skip the attribute test for this group. */ if (s->asteps != 0 && which % s->asteps == 0) - result = verify_group_attribute(g, which); + result = verify_group_attribute(s, g, which); else result = true; @@ -334,20 +2899,19 @@ verify_group(state_t *s, unsigned int which) return result; error: + H5E_BEGIN_TRY { H5Gclose(g); } H5E_END_TRY; - return false; -} + /* The reader sends an error message to the writer to stop the test.*/ + if(s->use_named_pipes && s->attr_test == true) + np_send_error(s,false); -/* Sleep for `tenths` tenths of a second */ -static void -decisleep(uint32_t tenths) -{ - uint64_t nsec = tenths * 100 * 1000 * 1000; +error2: + + return false; - H5_nanosleep(nsec); } int @@ -363,7 +2927,8 @@ main(int argc, char **argv) const char *fifo_reader_to_writer = "./fifo_group_reader_to_writer"; int fd_writer_to_reader = -1, fd_reader_to_writer = -1; int notify = 0, verify = 0; - unsigned int i; + bool wg_ret = false; + bool vg_ret = false; if (!state_init(&s, argc, argv)) { H5_FAILED(); AT(); @@ -371,11 +2936,13 @@ main(int argc, char **argv) goto error; } - personality = strstr(s.progname, "vfd_swmr_group_"); + personality = HDstrstr(s.progname, "vfd_swmr_group_"); - if (personality != NULL && strcmp(personality, "vfd_swmr_group_writer") == 0) + if (personality != NULL && + HDstrcmp(personality, "vfd_swmr_group_writer") == 0) writer = true; - else if (personality != NULL && strcmp(personality, "vfd_swmr_group_reader") == 0) + else if (personality != NULL && + HDstrcmp(personality, "vfd_swmr_group_reader") == 0) writer = false; else { H5_FAILED(); AT(); @@ -386,8 +2953,12 @@ main(int argc, char **argv) /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ init_vfd_swmr_config(&config, 4, 7, writer, FALSE, 128, "./group-shadow"); - /* use_latest_format, use_vfd_swmr, only_meta_page, config */ - if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, &config)) < 0) { + /* If old-style option is chosen, use the earliest file format(H5F_LIBVER_EARLIEST) + * as the second parameter of H5Pset_libver_bound() that is called by + * vfd_swmr_create_fapl. Otherwise, the latest file format(H5F_LIBVER_LATEST) + * should be used as the second parameter of H5Pset_libver_bound(). + * Also pass the use_vfd_swmr, only_meta_page, config to vfd_swmr_create_fapl().*/ + if ((fapl = vfd_swmr_create_fapl(!s.old_style_grp, s.use_vfd_swmr, true, &config)) < 0) { H5_FAILED(); AT(); printf("vfd_swmr_create_fapl failed\n"); goto error; @@ -449,123 +3020,94 @@ main(int argc, char **argv) goto error; } + /* Pass the named pipe information to the struct of state_t s, for attribute tests.*/ + if(s.use_named_pipes) { + s.np_fd_w_to_r = fd_writer_to_reader; + s.np_fd_r_to_w = fd_reader_to_writer; + s.np_notify = notify; + s.np_verify = verify; + s.tick_len = config.tick_len; + s.max_lag = config.max_lag; + } + + /* For attribute test, force the named pipe to communicate in every step. + * This will avoid the fake verification error from the reader when using the named pipe. + * If the named pipe is not forced to communicate in every step, the reader may go ahead + * to verify the group and the attribute operations before the writer has a chance to + * carry out the corresponding operations. */ + if (s.at_pattern != ' ') { + s.attr_test = true; + if(s.use_named_pipes) + s.csteps = 1; + } + if (writer) { for (step = 0; step < s.nsteps; step++) { dbgf(2, "writer: step %d\n", step); - if (!write_group(&s, step)) { + wg_ret = write_group(&s, step); + + if(wg_ret == false) { H5_FAILED(); AT(); - printf("write_group failed\n"); + printf("write_group failed at step %d\n",step); /* At communication interval, notifies the reader about the failture and quit */ - if (s.use_named_pipes && (step % s.csteps == 0)) { - notify = -1; - HDwrite(fd_writer_to_reader, ¬ify, sizeof(int)); - } - + if (s.use_named_pipes && s.attr_test !=true && step % s.csteps == 0) + np_send_error(&s,true); goto error; - } else { - /* At communication interval, notifies the reader and waits for its response */ - if (s.use_named_pipes && (step % s.csteps == 0)) { - /* Bump up the value of notify to notice the reader to start to read */ - notify++; - if (HDwrite(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) { - H5_FAILED(); AT(); - printf("HDwrite failed\n"); - goto error; - } - - /* During the wait, writer makes repeated HDF5 API calls - * to trigger EOT at approximately the correct time */ - for(i = 0; i < config.max_lag + 1; i++) { - decisleep(config.tick_len); - H5E_BEGIN_TRY { - H5Aexists(s.file, "nonexistent"); - } H5E_END_TRY; - } - - /* Receive the same value from the reader and verify it before - * going to the next step */ - verify++; - if (HDread(fd_reader_to_writer, ¬ify, sizeof(int)) < 0) { - H5_FAILED(); AT(); - printf("HDread failed\n"); - goto error; - } + } + else { - if (notify == -1) { - H5_FAILED(); AT(); - printf("reader failed to verify group\n"); - goto error; - } + /* At communication interval, notifies the reader and waits for its response */ + if (s.use_named_pipes && s.attr_test != true && step % s.csteps == 0) { - if (notify != verify) { + if(np_wr_send_receive(&s) == false) { H5_FAILED(); AT(); - printf("received message %d, expecting %d\n", notify, verify); + dbgf(2, "writer: write group - verification failed.\n"); goto error; } } } } - } - else { - for (step = 0; step < s.nsteps; step++) { - dbgf(2, "reader: step %d\n", step); - - /* At communication interval, waits for the writer to finish creation before starting verification - */ - if (s.use_named_pipes && (step % s.csteps == 0)) { - /* The writer should have bumped up the value of notify. - * Do the same with verify and confirm it */ - verify++; - - /* Receive the notify that the writer bumped up the value */ - if (HDread(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) { - H5_FAILED(); AT(); - printf("HDread failed\n"); - goto error; - } - - if (notify == -1) { - H5_FAILED(); AT(); - printf("writer failed to create group\n"); - goto error; - } + } else { + for (step = 0; step < s.nsteps;step++) { + dbgf(1, "reader: step %d\n", step); - if (notify != verify) { + /* At communication interval, waits for the writer to finish creation before starting verification */ + if (s.use_named_pipes && s.attr_test != true && step % s.csteps == 0) { + if(false == np_rd_receive(&s)) { H5_FAILED(); AT(); - printf("received message %d, expecting %d\n", notify, verify); goto error; } } - /* Wait for a few ticks for the update to happen */ - if (s.use_named_pipes) + /* For the default test, wait for a few ticks for the update to happen */ + if(s.use_named_pipes && s.attr_test== false) decisleep(config.tick_len * s.update_interval); - /* Start to verify group */ - if (!verify_group(&s, step)) { - H5_FAILED(); AT(); + vg_ret = verify_group(&s, step); + + if (vg_ret == false) { + printf("verify_group failed\n"); + H5_FAILED(); AT(); /* At communication interval, tell the writer about the failure and exit */ - if (s.use_named_pipes && (step % s.csteps == 0)) { - notify = -1; - HDwrite(fd_reader_to_writer, ¬ify, sizeof(int)); - } - + if (s.use_named_pipes && s.attr_test != true && step % s.csteps == 0) + np_send_error(&s,false); goto error; - } else { - if (s.use_named_pipes && (step % s.csteps == 0)) { - /* Send back the same nofity value for acknowledgement to tell the writer - * move to the next step */ - if (HDwrite(fd_reader_to_writer, ¬ify, sizeof(int)) < 0) { - H5_FAILED(); AT(); - printf("HDwrite failed\n"); + + } + else { + + /* Send back the same nofity value for acknowledgement to tell the writer + * move to the next step. */ + if (s.use_named_pipes && s.attr_test!=true && step % s.csteps == 0) { + if(np_rd_send(&s)==false) goto error; - } } } + } } @@ -581,6 +3123,12 @@ main(int argc, char **argv) goto error; } + if (H5Sclose(s.one_by_one_sid) < 0) { + H5_FAILED(); AT(); + printf("H5Sclose failed\n"); + goto error; + } + if (H5Fclose(s.file) < 0) { H5_FAILED(); AT(); printf("H5Fclose failed\n"); @@ -621,6 +3169,7 @@ error: H5E_BEGIN_TRY { H5Pclose(fapl); H5Pclose(fcpl); + H5Sclose(s.one_by_one_sid); H5Fclose(s.file); } H5E_END_TRY; @@ -636,6 +3185,7 @@ error: } return EXIT_FAILURE; + } #else /* H5_HAVE_WIN32_API */ diff --git a/test/vfd_swmr_zoo_writer.c b/test/vfd_swmr_zoo_writer.c index 75f3338..00dcd9e 100644 --- a/test/vfd_swmr_zoo_writer.c +++ b/test/vfd_swmr_zoo_writer.c @@ -11,7 +11,6 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include <err.h> -#include <libgen.h> #define H5C_FRIEND /* suppress error about including H5Cpkg */ #define H5F_FRIEND /* suppress error about including H5Fpkg */ @@ -29,31 +28,30 @@ #include "genall5.h" #include "vfd_swmr_common.h" -#ifndef _arraycount -#define _arraycount(_a) (sizeof(_a) / sizeof(_a[0])) -#endif +#define MAX_READ_LEN_IN_SECONDS 2 +#define TICK_LEN 4 typedef struct _shared_ticks { uint64_t reader_tick; } shared_ticks_t; -typedef struct _tick_stats { - uint64_t writer_tried_increase; - uint64_t writer_aborted_increase; - uint64_t writer_read_shared_file; - uint64_t reader_tick_was_zero; // writer read reader tick equal to 0 - uint64_t reader_tick_lead_writer; // writer read reader tick greater than - // proposed writer tick - uint64_t writer_lead_reader_by[1]; // proposed writer tick lead reader - // tick by `lead` ticks - // `writer_lead_reader_by[lead]` - // times, for `0 <= lead <= max_lag - 1` -} tick_stats_t; - +int fd_writer_to_reader = -1, fd_reader_to_writer = -1; +const char *fifo_writer_to_reader = "./fifo_writer_to_reader"; +const char *fifo_reader_to_writer = "./fifo_reader_to_writer"; +bool use_vfd_swmr = true; +bool use_named_pipe = true; +bool print_estack = false; static H5F_vfd_swmr_config_t swmr_config; -static tick_stats_t * tick_stats = NULL; -static const hid_t badhid = H5I_INVALID_HID; static bool writer; +struct timespec ival = {MAX_READ_LEN_IN_SECONDS, 0}; /* Expected maximal time for reader's validation */ + +zoo_config_t config = { + .proc_num = 0 + , .skip_compact = false + , .skip_varlen = true + , .max_pause_msecs = 0 + , .msgival = {.tv_sec = 0, .tv_nsec = 0} +}; static void #ifndef H5C_COLLECT_CACHE_STATS @@ -74,335 +72,578 @@ print_cache_hits(H5C_t H5_ATTR_UNUSED *cache) #endif void -zoo_create_hook(hid_t fid) +zoo_create_hook(hid_t H5_ATTR_UNUSED fid) { dbgf(3, "%s: enter\n", __func__); if (writer) - H5Fvfd_swmr_end_tick(fid); + decisleep(1); } +/* Print out the menu for the command-line options */ static void usage(const char *progname) { - fprintf(stderr, "usage: %s [-C] [-S] [-W] [-a] [-e] [-m] [-q] [-v]\n", progname); + fprintf(stderr, "usage: %s [-C] [-S] [-a] [-e] [-p] [-q] [-v]\n", progname); fprintf(stderr, "\n -C: skip compact dataset tests\n"); - fprintf(stderr, " -S: do not use VFD SWMR\n"); - fprintf(stderr, " -W: do not wait for SIGINT or SIGUSR1\n"); - fprintf(stderr, " -a: run all tests, including variable-length data\n"); - fprintf(stderr, " -e: print error stacks\n"); - fprintf(stderr, " -m ms: maximum `ms` milliseconds pause between\n"); - fprintf(stderr, " each create/delete step\n"); - fprintf(stderr, " -q: be quiet: few/no progress messages\n"); - fprintf(stderr, " -v: be verbose: most progress messages\n"); + fprintf(stderr, " -S: do not use VFD SWMR\n"); + fprintf(stderr, " -a: run all tests, including variable-length data\n"); + fprintf(stderr, " -e: print error stacks\n"); + fprintf(stderr, " -l tick_num: expected maximal number of ticks from \n"); + fprintf(stderr, " the writer's finishing zoo creation or deletion to the reader's finishing validation\n"); + fprintf(stderr, " -N: do not use named pipes\n"); + fprintf(stderr, " -q: be quiet: few/no progress messages\n"); + fprintf(stderr, " -v: be verbose: most progress messages\n"); exit(EXIT_FAILURE); } -bool -vfd_swmr_writer_may_increase_tick_to(uint64_t new_tick, bool wait_for_reader) +/* Private function to help parsing command-line options */ +static int +parse_command_line_options(int argc, char **argv) { - static int fd = -1; - shared_ticks_t shared; - ssize_t nread; - h5_retry_t retry; - bool do_try; + int ch; + unsigned long tmpl; + char *end; + + while ((ch = getopt(argc, argv, "CSael:Nqv")) != -1) { + switch(ch) { + case 'C': + config.skip_compact = true; + break; + case 'S': + use_vfd_swmr = false; + break; + case 'a': + config.skip_varlen = false; + break; + case 'e': + print_estack = true; + break; + case 'l': + /* Expected maximal number of ticks from the writer's finishing zoo creation or deletion + * to the reader's finishing validation of zoo creation or deletion */ + errno = 0; + tmpl = HDstrtoul(optarg, &end, 0); + + if (end == optarg || *end != '\0') { + printf("couldn't parse `-l` argument `%s`", optarg); + goto error; + } else if (errno != 0) { + printf("couldn't parse `-l` argument `%s`", optarg); + goto error; + } else if (tmpl > UINT_MAX) { + printf("`-l` argument `%lu` too large", tmpl); + goto error; + } + + { + /* Translate the tick number to time represented by the timespec struct */ + float time = (float)(((unsigned)tmpl * TICK_LEN) / 10.0); + unsigned sec = (unsigned)time; + unsigned nsec = (unsigned)((time - sec) * 10 * 1000 * 1000); + + ival.tv_sec = sec; + ival.tv_nsec = nsec; + } + break; + case 'N': + /* Disable named pipes, mainly for running the writer and reader seperately */ + use_named_pipe = false; + break; + case 'q': + verbosity = 1; + break; + case 'v': + verbosity = 3; + break; + default: + usage(argv[0]); + break; + } + } + argv += optind; + argc -= optind; - dbgf(3, "%s: enter\n", __func__); + if (argc > 0) { + H5_FAILED(); AT(); + printf("unexpected command-line arguments"); + goto error; + } + + return 0; + +error: + return -1; +} - if (fd == -1) { - fd = open("./shared_tick_num", O_RDONLY); - if (fd == -1) { - warn("%s: open", __func__); // TBD ratelimit/silence this warning - return true; +/* Writer creates two named pipes(FIFO) to coordinate two-way communication + * between the writer and the reader. Both the writer and reader open the named pipes */ +static int +create_open_named_pipes(void) +{ + /* Writer creates two named pipes(FIFO) to coordinate two-way communication */ + if (writer) { + if (HDmkfifo(fifo_writer_to_reader, 0600) < 0) { + H5_FAILED(); AT(); + printf("HDmkfifo failed"); + goto error; } - assert(tick_stats == NULL); - tick_stats = calloc(1, sizeof(*tick_stats) + - (swmr_config.max_lag - 1) * sizeof(tick_stats->writer_lead_reader_by[0])); - if (tick_stats == NULL) - err(EXIT_FAILURE, "%s: calloc", __func__); + + if (HDmkfifo(fifo_reader_to_writer, 0600) < 0) { + H5_FAILED(); AT(); + printf("HDmkfifo failed"); + goto error; + } + } + + /* Both the writer and reader open the pipes */ + if ((fd_writer_to_reader = HDopen(fifo_writer_to_reader, O_RDWR)) < 0) { + H5_FAILED(); AT(); + printf("fifo_writer_to_reader open failed"); + goto error; } - tick_stats->writer_tried_increase++; + if ((fd_reader_to_writer = HDopen(fifo_reader_to_writer, O_RDWR)) < 0) { + H5_FAILED(); AT(); + printf("fifo_reader_to_writer open failed"); + goto error; + } - for (do_try = h5_retry_init(&retry, 14, 10 * 1000 * 1000, 100 * 1000 * 1000); do_try; - do_try = wait_for_reader && h5_retry_next(&retry)) { + return 0; - tick_stats->writer_read_shared_file++; +error: + return -1; +} + +/* Notify the reader of finishing zoo creation by sending the timestamp + * and wait for the reader to finish validation before proceeding */ +static int +notify_and_wait_for_reader(hid_t fid, int verify) +{ + int notify; + unsigned int i; + struct timespec last = {0, 0}; + + /* Get the time when finishing zoo creation */ + if (HDclock_gettime(CLOCK_MONOTONIC, &last) < 0) { + H5_FAILED(); AT(); + printf("HDclock_gettime failed"); + goto error; + } - if ((nread = pread(fd, &shared, sizeof(shared), 0)) == -1) - err(EXIT_FAILURE, "%s: pread", __func__); + /* Notify the reader of finishing zoo creation by sending the timestamp */ + if (HDwrite(fd_writer_to_reader, &last, sizeof(last)) < 0) { + H5_FAILED(); AT(); + printf("HDwrite failed"); + goto error; + } - if (nread != sizeof(shared)) - errx(EXIT_FAILURE, "%s: pread", __func__); + /* During the wait, writer makes repeated HDF5 API calls so as to trigger + * EOT at approximately the correct time */ + for(i = 0; i < swmr_config.max_lag + 1; i++) { + decisleep(swmr_config.tick_len); - // TBD convert endianness + H5E_BEGIN_TRY { + H5Aexists(fid, "nonexistent"); + } H5E_END_TRY; + } - if (shared.reader_tick == 0) { - tick_stats->reader_tick_was_zero++; - return true; - } + /* Wait until the reader finishes validating zoo creation */ + if (HDread(fd_reader_to_writer, ¬ify, sizeof(int)) < 0) { + H5_FAILED(); AT(); + printf("HDread failed"); + goto error; + } - if (new_tick < shared.reader_tick) { - tick_stats->reader_tick_lead_writer++; - return true; - } - if (new_tick <= shared.reader_tick + swmr_config.max_lag - 1) { - uint64_t lead = new_tick - shared.reader_tick; - assert(lead <= swmr_config.max_lag - 1); - tick_stats->writer_lead_reader_by[lead]++; - return true; - } + if (notify != verify) { + H5_FAILED(); AT(); + printf("expected %d but read %d", verify, notify); + goto error; } - if (wait_for_reader && !do_try) - errx(EXIT_FAILURE, "%s: timed out waiting for reader", __func__); - tick_stats->writer_aborted_increase++; + return 0; - return false; +error: + return -1; } -void -vfd_swmr_reader_did_increase_tick_to(uint64_t new_tick) +/* Notify the reader of finishing zoo deletion by sending the timestamp */ +static int +notify_reader(void) { - static int fd = -1; - shared_ticks_t shared; - ssize_t nwritten; + struct timespec last = {0, 0}; - dbgf(3, "%s: enter\n", __func__); + /* Get the time when finishing zoo deletion */ + if (HDclock_gettime(CLOCK_MONOTONIC, &last) < 0) { + H5_FAILED(); AT(); + printf("HDclock_gettime failed"); + goto error; + } + + /* Notify the reader about finishing zoo deletion by sending the timestamp */ + if (HDwrite(fd_writer_to_reader, &last, sizeof(last)) < 0) { + H5_FAILED(); AT(); + printf("HDwrite failed"); + goto error; + } + + return 0; + +error: + return -1; +} + +/* Wait for the writer's notice before starting to zoo validation */ +static int +reader_verify(int verify) +{ + int notify; + + if (HDread(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) { + H5_FAILED(); AT(); + printf("HDread failed"); + goto error; + } + + if (notify != verify) { + H5_FAILED(); AT(); + printf("expected %d but read %d", verify, notify); + goto error; + } - if (fd == -1) { - // TBD create a temporary file, here, and move it to its final path - // after writing it. - fd = open("./shared_tick_num", O_RDWR | O_CREAT, 0600); - if (fd == -1) - err(EXIT_FAILURE, "%s: open", __func__); + return 0; + +error: + return -1; +} + +/* Receive the notice of the writer finishing zoo creation (timestamp) + * Make sure the zoo validation doesn't take longer than the expected time. + * This time period is from the writer finishing zoo creation to the reader finishing + * the validation of zoo creation */ +static int +reader_check_time_and_notify_writer(int notify) +{ + struct timespec last = {0, 0}; + + /* Receive the notice of the writer finishing zoo creation (timestamp) */ + if (HDread(fd_writer_to_reader, &last, sizeof(last)) < 0) { + H5_FAILED(); AT(); + printf("HDread failed"); + goto error; + } + + /* Make sure the zoo validation doesn't take longer than the expected time. + * This time period is from the writer finishing zoo creation to the reader finishing + * the validation of zoo creation */ + if (below_speed_limit(&last, &ival)) { + AT(); + warnx("validate_zoo took too long to finish"); } - shared.reader_tick = new_tick; + /* Notify the writer that zoo validation is finished */ + if (HDwrite(fd_reader_to_writer, ¬ify, sizeof(int)) < 0) { + H5_FAILED(); AT(); + printf("HDwrite failed"); + goto error; + } - // TBD convert endianness + return 0; - if ((nwritten = pwrite(fd, &shared, sizeof(shared), 0)) == -1) - errx(EXIT_FAILURE, "%s: pwrite", __func__); +error: + return -1; +} - if (nwritten != sizeof(shared)) - errx(EXIT_FAILURE, "%s: pwrite", __func__); +/* Receive the finish notice (timestamp) from the writer. + * Make sure validation of zoo deletion doesn't take longer than the expected time. + * This time period is from the writer finishing zoo deletion to the reader finishing + * the validation of zoo deletion */ +static int +reader_check_time_after_verify_deletion(void) +{ + struct timespec last = {0, 0}; - if (new_tick == 0) { - if (unlink("./shared_tick_num") == -1) - warn("%s: unlink", __func__); - if (close(fd) == -1) - err(EXIT_FAILURE, "%s: close", __func__); - fd = -1; + if (HDread(fd_writer_to_reader, &last, sizeof(last)) < 0) { + H5_FAILED(); AT(); + printf("HDread failed"); + goto error; } + + if (below_speed_limit(&last, &ival)) { + AT(); + warnx("validate_deleted_zoo took too long to finish"); + } + + return 0; + +error: + return -1; +} + +/* Close and remove the named pipes */ +static int +close_named_pipes(void) +{ + /* Close the named pipes */ + if (HDclose(fd_writer_to_reader) < 0) { + H5_FAILED(); AT(); + printf("HDclose failed\n"); + goto error; + } + + if (HDclose(fd_reader_to_writer) < 0) { + H5_FAILED(); AT(); + printf("HDclose failed\n"); + goto error; + } + + /* Reader finishes last and deletes the named pipes */ + if(!writer) { + if(HDremove(fifo_writer_to_reader) != 0) { + H5_FAILED(); AT(); + printf("HDremove failed\n"); + goto error; + } + + if(HDremove(fifo_reader_to_writer) != 0) { + H5_FAILED(); AT(); + printf("HDremove failed\n"); + goto error; + } + } + + return 0; + +error: + return -1; } int main(int argc, char **argv) { - hid_t fapl, fcpl, fid; - H5F_t * f; - H5C_t * cache; - sigset_t oldsigs; - herr_t ret; - zoo_config_t config = {.proc_num = 0, - .skip_compact = false, - .skip_varlen = true, - .max_pause_msecs = 0, - .msgival = {.tv_sec = 5, .tv_nsec = 0}}; - struct timespec lastmsgtime = {.tv_sec = 0, .tv_nsec = 0}; - bool wait_for_signal; - int ch; - unsigned seed; - unsigned long tmpl; - char * end; - const char * seedvar = "H5_ZOO_STEP_SEED"; - bool use_vfd_swmr = true; - bool print_estack = false; - const char * progname = basename(argv[0]); - const char * personality = strstr(progname, "vfd_swmr_zoo_"); - estack_state_t es; - char step = 'b'; + hid_t fapl = H5I_INVALID_HID, fcpl = H5I_INVALID_HID, fid = H5I_INVALID_HID; + H5F_t *f; + H5C_t *cache; + struct timespec lastmsgtime = {.tv_sec = 0, .tv_nsec = 0}; + char *progname = NULL; + char *personality; + estack_state_t es; H5F_vfd_swmr_config_t vfd_swmr_config; + int notify = 0, verify = 0; + + if (H5_basename(argv[0], &progname) < 0) { + H5_FAILED(); AT(); + printf("H5_basename failed\n"); + goto error; + } + + personality = HDstrstr(progname, "vfd_swmr_zoo_"); if (personality != NULL && strcmp(personality, "vfd_swmr_zoo_writer") == 0) - writer = wait_for_signal = true; + writer = true; else if (personality != NULL && strcmp(personality, "vfd_swmr_zoo_reader") == 0) writer = false; else { - errx(EXIT_FAILURE, "unknown personality, expected vfd_swmr_zoo_{reader,writer}"); + H5_FAILED(); AT(); + printf("unknown personality, expected vfd_swmr_zoo_{reader,writer}"); + goto error; } - if (writer) - config.max_pause_msecs = 50; - - while ((ch = getopt(argc, argv, "CSWaem:qv")) != -1) { - switch (ch) { - case 'C': - config.skip_compact = true; - break; - case 'S': - use_vfd_swmr = false; - break; - case 'W': - wait_for_signal = false; - break; - case 'a': - config.skip_varlen = false; - break; - case 'e': - print_estack = true; - break; - case 'm': - errno = 0; - tmpl = strtoul(optarg, &end, 0); - if (end == optarg || *end != '\0') - errx(EXIT_FAILURE, "couldn't parse `-m` argument `%s`", optarg); - else if (errno != 0) - err(EXIT_FAILURE, "couldn't parse `-m` argument `%s`", optarg); - else if (tmpl > UINT_MAX) - errx(EXIT_FAILURE, "`-m` argument `%lu` too large", tmpl); - config.max_pause_msecs = (unsigned)tmpl; - break; - case 'q': - verbosity = 1; - break; - case 'v': - verbosity = 3; - break; - default: - usage(argv[0]); - break; - } - } - argv += optind; - argc -= optind; - - if (argc > 0) - errx(EXIT_FAILURE, "unexpected command-line arguments"); + parse_command_line_options(argc, argv); /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(&vfd_swmr_config, 4, 7, writer, FALSE, 128, "./zoo-shadow"); + init_vfd_swmr_config(&vfd_swmr_config, TICK_LEN, 7, writer, FALSE, 128, "./zoo-shadow"); /* ? turn off use latest format argument via 1st argument? since later on it reset to early format */ /* use_latest_format, use_vfd_swmr, only_meta_page, config */ - fapl = vfd_swmr_create_fapl(true, use_vfd_swmr, true, &vfd_swmr_config); - - if (use_vfd_swmr && H5Pget_vfd_swmr_config(fapl, &swmr_config) < 0) - errx(EXIT_FAILURE, "H5Pget_vfd_swmr_config"); + if ((fapl = vfd_swmr_create_fapl(true, use_vfd_swmr, true, &vfd_swmr_config)) < 0) { + H5_FAILED(); AT(); + printf("vfd_swmr_create_fapl"); + goto error; + } - if (fapl < 0) - errx(EXIT_FAILURE, "vfd_swmr_create_fapl"); + if (use_vfd_swmr && H5Pget_vfd_swmr_config(fapl, &swmr_config) < 0) { + H5_FAILED(); AT(); + printf("H5Pget_vfd_swmr_config failed"); + goto error; + } if (H5Pset_libver_bounds(fapl, H5F_LIBVER_EARLIEST, H5F_LIBVER_LATEST) < 0) { - errx(EXIT_FAILURE, "%s.%d: H5Pset_libver_bounds", __func__, __LINE__); + H5_FAILED(); AT(); + printf("H5Pset_libver_bounds failed"); + goto error; } - if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - errx(EXIT_FAILURE, "H5Pcreate"); + if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) { + H5_FAILED(); AT(); + printf("H5Pcreate failed"); + goto error; + } - ret = H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, false, 1); - if (ret < 0) - errx(EXIT_FAILURE, "H5Pset_file_space_strategy"); + if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, false, 1) < 0) { + H5_FAILED(); AT(); + printf("H5Pset_file_space_strategy failed"); + goto error; + } if (writer) fid = H5Fcreate("vfd_swmr_zoo.h5", H5F_ACC_TRUNC, fcpl, fapl); else fid = H5Fopen("vfd_swmr_zoo.h5", H5F_ACC_RDONLY, fapl); - if (fid == badhid) - errx(EXIT_FAILURE, writer ? "H5Fcreate" : "H5Fopen"); + if (fid < 0) { + H5_FAILED(); AT(); + printf(writer ? "H5Fcreate failed" : "H5Fopen failed"); + goto error; + } - if ((f = H5VL_object_verify(fid, H5I_FILE)) == NULL) - errx(EXIT_FAILURE, "H5VL_object_verify"); + if ((f = H5VL_object_verify(fid, H5I_FILE)) == NULL) { + H5_FAILED(); AT(); + printf("H5VL_object_verify failed"); + goto error; + } cache = f->shared->cache; - if (wait_for_signal) - block_signals(&oldsigs); + /* Writer creates two named pipes(FIFO) to coordinate two-way communication + * between the writer and the reader. Both the writer and reader open the named pipes */ + if (use_named_pipe && create_open_named_pipes() < 0) { + H5_FAILED(); AT(); + printf("create_open_named_pipes failed"); + goto error; + } print_cache_hits(cache); es = print_estack ? estack_get_state() : disable_estack(); if (writer) { - dbgf(2, "Writing zoo...\n"); - /* get seed from environment or else from time(3) */ - switch (fetch_env_ulong(seedvar, UINT_MAX, &tmpl)) { - case -1: - errx(EXIT_FAILURE, "%s: fetch_env_ulong", __func__); - case 0: - seed = (unsigned int)time(NULL); - break; - default: - seed = (unsigned int)tmpl; - break; + /* Writer tells reader to start */ + notify = 1; + if (use_named_pipe && HDwrite(fd_writer_to_reader, ¬ify, sizeof(int)) < 0) { + H5_FAILED(); AT(); + printf("HDwrite failed"); + goto error; } - dbgf(1, "To reproduce, set seed (%s) to %u.\n", seedvar, seed); - - HDsrandom(seed); - - if (!create_zoo(fid, ".", &lastmsgtime, config)) - errx(EXIT_FAILURE, "create_zoo didn't pass self-check"); - - /* Avoid deadlock: flush the file before waiting for the reader's - * message. - */ - if (H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) - errx(EXIT_FAILURE, "%s: H5Fflush failed", __func__); + /* Start to create the zoo */ + if (!create_zoo(fid, ".", &lastmsgtime, config)) { + H5_FAILED(); AT(); + printf("create_zoo failed"); + goto error; + } - if (read(STDIN_FILENO, &step, sizeof(step)) == -1) - err(EXIT_FAILURE, "read"); + /* Notify the reader of finishing zoo creation by sending the timestamp + * and wait for the reader to finish validation before proceeding */ + verify = 2; + if (use_named_pipe && notify_and_wait_for_reader(fid, verify) < 0) { + H5_FAILED(); AT(); + printf("notify_and_wait_for_reader failed"); + goto error; + } - if (step != 'b') - errx(EXIT_FAILURE, "expected 'b' read '%c'", step); + /* Start to delete the zoo */ + if (!delete_zoo(fid, ".", &lastmsgtime, config)) { + H5_FAILED(); AT(); + printf("delete_zoo failed"); + goto error; + } - if (!delete_zoo(fid, ".", &lastmsgtime, config)) - errx(EXIT_FAILURE, "delete_zoo failed"); - } - else { + /* Notify the reader of finishing zoo deletion by sending the timestamp */ + if (use_named_pipe && notify_reader() < 0) { + H5_FAILED(); AT(); + printf("notify_reader failed"); + goto error; + } + } else { dbgf(2, "Reading zoo...\n"); + /* Wait for the writer's notice before starting to zoo validation */ + verify = 1; + if (use_named_pipe && reader_verify(verify) < 0) { + H5_FAILED(); AT(); + printf("reader_verify failed"); + goto error; + } + + /* Validate the zoo creation */ while (!validate_zoo(fid, ".", &lastmsgtime, config)) ; - if (write(STDOUT_FILENO, &step, sizeof(step)) == -1) - err(EXIT_FAILURE, "write"); + + /* Receive the notice of the writer finishing zoo creation (timestamp) + * Make sure the zoo validation doesn't take longer than the expected time. + * This time period is from the writer finishing zoo creation to the reader finishing + * the validation of zoo creation */ + notify = 2; + if (use_named_pipe && reader_check_time_and_notify_writer(notify) < 0) { + H5_FAILED(); AT(); + printf("reader_check_time_and_notify_writer failed"); + goto error; + } + + /* Start to validate the zoo deletion */ while (!validate_deleted_zoo(fid, ".", &lastmsgtime, config)) ; + + /* Receive the finish notice (timestamp) from the writer. + * Make sure validation of zoo deletion doesn't take longer than the expected time. + * This time period is from the writer finishing zoo deletion to the reader finishing + * the validation of zoo deletion */ + if (use_named_pipe && reader_check_time_after_verify_deletion() < 0) { + H5_FAILED(); AT(); + printf("reader_check_time_and_notify_writer failed"); + goto error; + } } restore_estack(es); - if (use_vfd_swmr && wait_for_signal) - await_signal(fid); + if (H5Pclose(fapl) < 0) { + H5_FAILED(); AT(); + printf("H5Pclose failed"); + goto error; + } + + if (H5Pclose(fcpl) < 0) { + H5_FAILED(); AT(); + printf("H5Pclose failed"); + goto error; + } - if (writer && tick_stats != NULL) { - uint64_t lead; + if (H5Fclose(fid) < 0) { + H5_FAILED(); AT(); + printf("H5Fclose failed"); + goto error; + } - dbgf(2, "writer tried tick increase %" PRIu64 "\n", tick_stats->writer_tried_increase); - dbgf(2, "writer aborted tick increase %" PRIu64 "\n", tick_stats->writer_aborted_increase); - dbgf(2, "writer read shared file %" PRIu64 "\n", tick_stats->writer_read_shared_file); - dbgf(2, "writer read reader tick equal to 0 %" PRIu64 "\n", tick_stats->reader_tick_was_zero); - dbgf(2, "writer read reader tick leading writer %" PRIu64 "\n", tick_stats->reader_tick_lead_writer); + if (progname) + HDfree(progname); - for (lead = 0; lead < swmr_config.max_lag; lead++) { - dbgf(2, "writer tick lead writer by %" PRIu64 " %" PRIu64 "\n", lead, - tick_stats->writer_lead_reader_by[lead]); - } + if (use_named_pipe && close_named_pipes() < 0) { + H5_FAILED(); AT(); + printf("close_named_pipes failed"); + goto error; } - if (H5Pclose(fapl) < 0) - errx(EXIT_FAILURE, "H5Pclose(fapl)"); + return EXIT_SUCCESS; - if (H5Pclose(fcpl) < 0) - errx(EXIT_FAILURE, "H5Pclose(fcpl)"); +error: + H5E_BEGIN_TRY { + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(fid); + } H5E_END_TRY; - if (H5Fclose(fid) < 0) - errx(EXIT_FAILURE, "H5Fclose"); + if (use_named_pipe && fd_writer_to_reader >= 0) + HDclose(fd_writer_to_reader); - if (wait_for_signal) - restore_signals(&oldsigs); + if (use_named_pipe && fd_reader_to_writer >= 0) + HDclose(fd_reader_to_writer); - return EXIT_SUCCESS; + if(use_named_pipe && !writer) { + HDremove(fifo_writer_to_reader); + HDremove(fifo_reader_to_writer); + } + + return EXIT_FAILURE; } |