diff options
51 files changed, 942 insertions, 190 deletions
diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index 33608b5..62fd0d8 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -174,6 +174,8 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - name: Get Sources uses: actions/checkout@v2 + with: + submodules: 'true' - name: Autotools Configure if: matrix.generator == 'autogen' @@ -46,6 +46,14 @@ works thereof, in binary and source code form. ----------------------------------------------------------------------------- ----------------------------------------------------------------------------- +Limited portions of HDF5 were developed by Akadio, Inc. Akadio's Copyright +Notice and Licensing Terms can be found here: COPYING_AKADIO file in this +directory or at +http://support.hdfgroup.org/ftp/HDF5/releases/COPYING_AKADIO. + +----------------------------------------------------------------------------- +----------------------------------------------------------------------------- + Limited portions of HDF5 were developed by Lawrence Berkeley National Laboratory (LBNL). LBNL's Copyright Notice and Licensing Terms can be found here: COPYING_LBNL_HDF5 file in this directory or at diff --git a/COPYING_AKADIO b/COPYING_AKADIO new file mode 100644 index 0000000..e0d58e3 --- /dev/null +++ b/COPYING_AKADIO @@ -0,0 +1,45 @@ +Copyright Notice and License Terms for +VFD SWMR (Virtual File Driver Single Writer/Multiple Reader) +----------------------------------------------------------------------------- +VFD SWMR (Virtual File Driver Single Writer/Multiple Reader) +Copyright © 2019, Akadio, Inc. + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted for any purpose (including commercial purposes) +provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions, and the following disclaimer in the + documentation and/or materials provided with the distribution. + +3. Neither the name of Akadio nor the name of any Contributor may be used + to endorse or promote products derived from this software without + specific prior written permission from Akadio, Inc. + +DISCLAIMER: +THIS SOFTWARE IS PROVIDED BY AKADIO, INC. AND THE CONTRIBUTORS "AS IS" WITH +NO WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED. IN NO EVENT SHALL +AKADIO, INC. OR THE CONTRIBUTORS BE LIABLE FOR ANY DAMAGES SUFFERED BY THE +USERS ARISING OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +You are under no obligation whatsoever to provide any bug fixes, patches, or +upgrades to the features, functionality or performance of the source code +("Enhancements") to anyone; however, if you choose to make your Enhancements +available either publicly, or directly to Akadio, Inc., without imposing a +separate written license agreement for such Enhancements, then you hereby +grant the following license: a non-exclusive, royalty-free perpetual license +to install, use, modify, prepare derivative works, incorporate into other +computer software, distribute, and sublicense such enhancements or derivative +works thereof, in binary and source code form. + +----------------------------------------------------------------------------- +This work was sponsored by the U.S. Department of Energy under Award Number +DE-SC0018504. +----------------------------------------------------------------------------- + @@ -25,6 +25,7 @@ ./.h5chkright.ini _DO_NOT_DISTRIBUTE_ ./ACKNOWLEDGMENTS ./COPYING +./COPYING_AKADIO ./COPYING_LBNL_HDF5 ./MANIFEST ./Makefile.dist @@ -3048,6 +3049,11 @@ ./utils/mirror_vfd/mirror_server_stop.c ./utils/mirror_vfd/mirror_writer.c +# VFD SWMR utilities +./utils/vfd_swmr/README.md +./utils/vfd_swmr/Makefile.am +./utils/vfd_swmr/aux_process.c + # high level libraries ./hl/Makefile.am ./hl/examples/Makefile.am diff --git a/config/cmake/libhdf5.settings.cmake.in b/config/cmake/libhdf5.settings.cmake.in index d80b0f8..a7c5f37 100644 --- a/config/cmake/libhdf5.settings.cmake.in +++ b/config/cmake/libhdf5.settings.cmake.in @@ -81,6 +81,7 @@ Dimension scales w/ new references: @DIMENSION_SCALES_WITH_NEW_REF@ Mirror VFD: @H5_HAVE_MIRROR_VFD@ (Read-Only) S3 VFD: @H5_HAVE_ROS3_VFD@ (Read-Only) HDFS VFD: @H5_HAVE_LIBHDFS@ + Auxiliary Process: @H5_HAVE_AUX_PROCESS@ dmalloc: @H5_HAVE_LIBDMALLOC@ Packages w/ extra debug output: @INTERNAL_DEBUG_OUTPUT@ API Tracing: @HDF5_ENABLE_TRACE@ diff --git a/configure.ac b/configure.ac index ab55f3e..f8a1e4d 100644 --- a/configure.ac +++ b/configure.ac @@ -3171,6 +3171,30 @@ fi AM_CONDITIONAL([MIRROR_VFD_CONDITIONAL], [test "X$MIRROR_VFD" = "Xyes"]) ## ---------------------------------------------------------------------- +## Check whether the auxiliary process for the VFD SWMR should be built. +## +AC_SUBST([AUX_PROCESS]) + +## Default is no auxiliary process +AUX_PROCESS=no + +AC_MSG_CHECKING([if the auxiliary process is enabled]) + +AC_ARG_ENABLE([aux-process], + [AS_HELP_STRING([--enable-aux-process], + [Build the auxiliary process for the VFD SWMR. + [default=no]])], + [AUX_PROCESS=$enableval], [AUX_PROCESS=no]) + +if test "X$AUX_PROCESS" = "Xyes"; then + AC_MSG_RESULT([yes]) + AC_DEFINE([HAVE_AUX_PROCESS], [1], + [Define whether the auxiliary process is compiled]) +else + AC_MSG_RESULT([no]) +fi + +## ---------------------------------------------------------------------- ## Check if Read-Only S3 virtual file driver is enabled by --enable-ros3-vfd ## AC_SUBST([ROS3_VFD]) @@ -4063,6 +4087,7 @@ AC_CONFIG_FILES([src/libhdf5.settings testpar/testpflush.sh utils/Makefile utils/mirror_vfd/Makefile + utils/vfd_swmr/Makefile tools/Makefile tools/lib/Makefile tools/libtest/Makefile diff --git a/examples/credel.c b/examples/credel.c index 4591d94..b90e70c 100644 --- a/examples/credel.c +++ b/examples/credel.c @@ -1,3 +1,16 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + #include <assert.h> #include <err.h> #include <errno.h> diff --git a/examples/gaussians.c b/examples/gaussians.c index 413bc03..515fac3 100644 --- a/examples/gaussians.c +++ b/examples/gaussians.c @@ -1,3 +1,16 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + #include <assert.h> #include <curses.h> #include <err.h> diff --git a/examples/h5_dtransform.c b/examples/h5_dtransform.c index a364ec1..acabd1e 100644 --- a/examples/h5_dtransform.c +++ b/examples/h5_dtransform.c @@ -1,6 +1,5 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * - * Copyright by the Board of Trustees of the University of Illinois. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/examples/nbcompat.c b/examples/nbcompat.c index aa67623..cfa0dcd 100644 --- a/examples/nbcompat.c +++ b/examples/nbcompat.c @@ -1,3 +1,16 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + #include "nbcompat.h" size_t diff --git a/examples/nbcompat.h b/examples/nbcompat.h index c528ca4..ff4f677 100644 --- a/examples/nbcompat.h +++ b/examples/nbcompat.h @@ -1,3 +1,16 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + #ifndef NB_COMPAT_H #define NB_COMPAT_H diff --git a/src/H5FDvfd_swmr.c b/src/H5FDvfd_swmr.c index 73d8df4..3517517 100644 --- a/src/H5FDvfd_swmr.c +++ b/src/H5FDvfd_swmr.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/src/H5FDvfd_swmr.h b/src/H5FDvfd_swmr.h index 096d5fb..dd1b452 100644 --- a/src/H5FDvfd_swmr.h +++ b/src/H5FDvfd_swmr.h @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/src/H5FDvfd_swmr_instr.c b/src/H5FDvfd_swmr_instr.c index 6f309d2..53d6161 100644 --- a/src/H5FDvfd_swmr_instr.c +++ b/src/H5FDvfd_swmr_instr.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/src/H5FDvfd_swmr_private.h b/src/H5FDvfd_swmr_private.h index 12fd2e2..5b6d300 100644 --- a/src/H5FDvfd_swmr_private.h +++ b/src/H5FDvfd_swmr_private.h @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/src/H5Fvfd_swmr.c b/src/H5Fvfd_swmr.c index 372ae19..8e29f06 100644 --- a/src/H5Fvfd_swmr.c +++ b/src/H5Fvfd_swmr.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * @@ -1,6 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * - * Copyright by the Board of Trustees of the University of Illinois. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/src/H5MVmodule.h b/src/H5MVmodule.h index fdfe036..d8862ee 100644 --- a/src/H5MVmodule.h +++ b/src/H5MVmodule.h @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * @@ -10,14 +11,17 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* - * Programmer: Quincey Koziol <koziol@hdfgroup.org> - * Saturday, September 12, 2015 +/*------------------------------------------------------------------------- * - * Purpose: This file contains declarations which define macros for the - * H5MV package. Including this header means that the source file - * is part of the H5MV package. + * Created: H5MVmodule.h + * + * Purpose: This file contains declarations which define macros for + * the H5MV package. Including this header means that the + * source file is part of the H5MV package. + * + *------------------------------------------------------------------------- */ + #ifndef _H5MVmodule_H #define _H5MVmodule_H diff --git a/src/H5MVpkg.h b/src/H5MVpkg.h index 2d564e6..bd75595 100644 --- a/src/H5MVpkg.h +++ b/src/H5MVpkg.h @@ -1,6 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * - * Copyright by the Board of Trustees of the University of Illinois. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * @@ -11,14 +11,17 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* - * Programmer: Quincey Koziol <koziol@hdfgroup.org> - * Tuesday, January 8, 2008 +/*------------------------------------------------------------------------- * - * Purpose: This file contains declarations which are visible only within - * the H5MV package. Source files outside the H5MV package should - * include H5MVprivate.h instead. + * Created: H5MVpkg.h + * + * Purpose: This file contains declarations which are visible only + * within the H5MV package. Source files outside the + * H5MV package should include H5MVprivate.h instead. + * + *------------------------------------------------------------------------- */ + #if !(defined H5MV_FRIEND || defined H5MV_MODULE) #error "Do not include this file outside the H5MV package!" #endif diff --git a/src/H5MVprivate.h b/src/H5MVprivate.h index edd4765..2332ad3 100644 --- a/src/H5MVprivate.h +++ b/src/H5MVprivate.h @@ -1,6 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * - * Copyright by the Board of Trustees of the University of Illinois. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * @@ -15,10 +15,11 @@ * * Created: H5MVprivate.h * - * Purpose: Private header file for file memory management. + * Purpose: Private header file for VFD SWMR free space mangement * *------------------------------------------------------------------------- */ + #ifndef _H5MVprivate_H #define _H5MVprivate_H diff --git a/src/H5MVsection.c b/src/H5MVsection.c index 8f80619..dc57e2f 100644 --- a/src/H5MVsection.c +++ b/src/H5MVsection.c @@ -1,6 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * - * Copyright by the Board of Trustees of the University of Illinois. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * @@ -11,12 +11,14 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* - * Programmer: Quincey Koziol <koziol@hdfgroup.org> - * Tuesday, January 8, 2008 +/*------------------------------------------------------------------------- * - * Purpose: Free space section callbacks for file. + * Created: H5MVsection.c * + * Purpose: Free-space section callbacks for VFD SWMR's metadata + * file + * + *------------------------------------------------------------------------- */ /****************/ diff --git a/src/H5retry_private.h b/src/H5retry_private.h index 4b26009..d4ef99f 100644 --- a/src/H5retry_private.h +++ b/src/H5retry_private.h @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/stubs.c b/test/stubs.c index 33ac027..e508b14 100644 --- a/test/stubs.c +++ b/test/stubs.c @@ -1,6 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * - * Copyright by the Board of Trustees of the University of Illinois. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/supervise.subr b/test/supervise.subr index d2b0395..40472d3 100644 --- a/test/supervise.subr +++ b/test/supervise.subr @@ -1,6 +1,17 @@ -#!/bin/sh - +#!/us/bin/env bash +# +# Copyright by The HDF Group. +# Copyright by Akadio, Inc. # +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Tests for the vfd swmr feature. + # catch_out_err_and_rc outbase command [arguments] # # Run `command` with any `arguments` provided. Redirect `command`'s diff --git a/test/testvfdswmr.sh.in b/test/testvfdswmr.sh.in index 176e1e5..b47516f 100644 --- a/test/testvfdswmr.sh.in +++ b/test/testvfdswmr.sh.in @@ -1,18 +1,17 @@ #!/us/bin/env bash # # Copyright by The HDF Group. -# Copyright by the Board of Trustees of the University of Illinois. -# All rights reserved. +# Copyright by Akadio, Inc. # # This file is part of HDF5. The full HDF5 copyright notice, including # terms governing use, modification, and redistribution, is contained in # the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/releases. +# distribution tree, or in https://www.hdfgroup.org/licenses. # If you do not have access to either file, you may request a copy from # help@hdfgroup.org. # # Tests for the vfd swmr feature. -# + ############################################################################### # VFD SWMR concurrent tests which are modified from existing swmr concurrent tests. # This is copied and modified from testswmr.sh.in @@ -152,11 +151,11 @@ all_tests="generator expand shrink expand_shrink sparse vlstr_null vlstr_oob zoo all_tests="${all_tests} groups groups_attrs groups_ops few_big many_small" # For exhaustive run, add: -# attrdset, dsetops, dsetchks, +# attrdset, dsetops, dsetops_ref, dsetchks # os_groups_attrs, os_groups_ops, os_groups_seg, independ_wr, # gfail_entry_length, gfail_checksum, gfail_page_size, gfail_index_space if [[ "$HDF5TestExpress" -eq 0 ]] ; then # exhaustive run - all_tests="${all_tests} attrdset dsetops dsetchks" + all_tests="${all_tests} attrdset dsetops dsetops_ref dsetchks" all_tests="${all_tests} os_groups_attrs os_groups_ops os_groups_seg independ_wr" all_tests="${all_tests} gfail_entry_length gfail_checksum gfail_page_size gfail_index_space" fi @@ -1095,6 +1094,16 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 16" "-d 2 -F -l 16" "-d 1 -t" "-d 1 -t # first dimension (up to 25 1x16x16) # + # Launch the auxiliary process for testing NFS if the --enable-aux-process option is enabled for configuration. + # If it isn't enabled, the auxiliary process simply exit without executing any code. For the testing case of + # VDS across multiple files (-M option), the program also skips for future support. + if [[ $options == *"-M"* ]]; then + catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process -a mdfile bigset_updater & + else + catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process mdfile bigset_updater & + fi + pid_aux_proc=$! + echo launch vfd_swmr_bigset_writer many small, options $options catch_out_err_and_rc vfd_swmr_bigset_writer \ ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_many_s -e 1 -r 16 -c 16 -q & @@ -1107,10 +1116,17 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 16" "-d 2 -F -l 16" "-d 1 -t" "-d 1 -t # Wait for the reader to finish before signalling the # writer to quit: the writer holds the file open so that the # reader will find the shadow file when it opens - # the .h5 file. + # the .h5 file. Also wait for the auxiliary process to finish. + wait $pid_aux_proc wait $pid_reader wait $pid_writer + # Collect exit code of the auxiliary process + if [ $(cat aux_process.rc) -ne 0 ]; then + echo the auxiliary process had error + nerrors=$((nerrors + 1)) + fi + # Collect exit code of the reader if [ $(cat vfd_swmr_bigset_reader.rc) -ne 0 ]; then echo reader had error @@ -1126,6 +1142,8 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 16" "-d 2 -F -l 16" "-d 1 -t" "-d 1 -t # Clean up output files rm -f vfd_swmr_bigset_writer.{out,rc} rm -f vfd_swmr_bigset_reader.*.{out,rc} + rm -f aux_process.{out,rc} + rm -f mdfile bigset_updater.* bigset-shadow-* done # bigset test for bigger chunks @@ -1149,6 +1167,17 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 10" "-d 2 -F -l 10" "-d 1 -t -l 10" "-d continue fi echo launch vfd_swmr_bigset_writer few big, options $options ......may take some time...... + + # Launch the auxiliary process for testing NFS if the --enable-aux-process option is enabled for configuration. + # If it isn't enabled, the auxiliary process simply exit without executing any code. For the testing case of + # VDS across multiple files (-M option), the program also skips for future support. + if [[ $options == *"-M"* ]]; then + catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process -a mdfile bigset_updater & + else + catch_out_err_and_rc aux_process ../../utils/vfd_swmr/aux_process mdfile bigset_updater & + fi + pid_aux_proc=$! + catch_out_err_and_rc vfd_swmr_bigset_writer \ ../vfd_swmr_bigset_writer -n $BIGSET_n $options -s $BIGSET_few_s -e 8 -r 256 -c 256 -q & pid_writer=$! @@ -1160,10 +1189,17 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 10" "-d 2 -F -l 10" "-d 1 -t -l 10" "-d # Wait for the reader to finish before signalling the # writer to quit: the writer holds the file open so that the # reader will find the shadow file when it opens - # the .h5 file. + # the .h5 file. Also wait for the auxiliary process to finish. + wait $pid_aux_proc wait $pid_reader wait $pid_writer + # Collect exit code of the auxiliary process + if [ $(cat aux_process.rc) -ne 0 ]; then + echo the auxiliary process had error + nerrors=$((nerrors + 1)) + fi + # Collect exit code of the reader if [ $(cat vfd_swmr_bigset_reader.rc) -ne 0 ]; then echo reader had error @@ -1179,6 +1215,8 @@ for options in "-d 1" "-d 1 -F" "-d 2 -l 10" "-d 2 -F -l 10" "-d 1 -t -l 10" "-d # Clean up output files rm -f vfd_swmr_bigset_writer.{out,rc} rm -f vfd_swmr_bigset_reader.*.{out,rc} + rm -f aux_process.{out,rc} + rm -f mdfile bigset_updater.* bigset-shadow-* done ############################################################################### @@ -1313,6 +1351,70 @@ done ############################################################################### # +# "dsetops_ref" test (this is the dsetops test with options added for testing references) +# --test object (-O) and region (-R) references +# --test with raw data flush only +# --test without flush will result in error because the references are not there +# to continue further testing +# +# Only for exhaustive run +# +############################################################################### +# +# +# Loop with +for ref in "-O" "-R" "-O -R"; do + # Loop with different operations + for options in "-p -e 20 -t -g -q" "-g -m 5 -n 2 -s 10 -w 7 -q" "-k -m 10 -n 5 -r 5 -l 10 -q"; do + # + # + if [ ${do_dsetops_ref:-no} = no ]; then + continue + fi + # Clean up any existing fifo files from previous runs + if [ -e ./$DSETOPS_FIFO_WRITER_TO_READER ]; then # If writer fifo file is found + rm -f ./$DSETOPS_FIFO_WRITER_TO_READER + fi + if [ -e ./$DSETOPS_FIFO_READER_TO_WRITER ]; then # If reader fifo file is found + rm -f ./$DSETOPS_FIFO_READER_TO_WRITER + fi + # + echo launch vfd_swmr_dsetops_writer dsetops, options $options $ref......may take some time...... + catch_out_err_and_rc vfd_swmr_dsetops_writer \ + ../vfd_swmr_dsetops_writer $options $ref & + pid_writer=$! + + catch_out_err_and_rc vfd_swmr_dsetops_reader \ + ../vfd_swmr_dsetops_reader $options $ref & + pid_reader=$! + + # Wait for the reader to finish before signaling the + # writer to quit: the writer holds the file open so that the + # reader will find the shadow file when it opens + # the .h5 file. + wait $pid_reader + wait $pid_writer + + # Collect exit code of the reader + if [ $(cat vfd_swmr_dsetops_reader.rc) -ne 0 ]; then + echo reader had error + nerrors=$((nerrors + 1)) + fi + + # Collect exit code of the writer + if [ $(cat vfd_swmr_dsetops_writer.rc) -ne 0 ]; then + echo writer had error + nerrors=$((nerrors + 1)) + fi + + # Clean up output files + rm -f vfd_swmr_dsetops_writer.{out,rc} + rm -f vfd_swmr_dsetops_reader.*.{out,rc} + done +done + +############################################################################### +# # "dsetchks" test # # Only for exhaustive run diff --git a/test/vfd_swmr.c b/test/vfd_swmr.c index 630b25d..902ce7f 100644 --- a/test/vfd_swmr.c +++ b/test/vfd_swmr.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/vfd_swmr_addrem_writer.c b/test/vfd_swmr_addrem_writer.c index ddf3f9c..4cfdeb2 100644 --- a/test/vfd_swmr_addrem_writer.c +++ b/test/vfd_swmr_addrem_writer.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/vfd_swmr_attrdset_writer.c b/test/vfd_swmr_attrdset_writer.c index a722f44..1037108 100644 --- a/test/vfd_swmr_attrdset_writer.c +++ b/test/vfd_swmr_attrdset_writer.c @@ -1,15 +1,15 @@ -/* - * Copyright by The HDF Group. - * Copyright by the Board of Trustees of the University of Illinois. - * All rights reserved. - * - * This file is part of HDF5. The full HDF5 copyright notice, including - * terms governing use, modification, and redistribution, is contained in - * the COPYING file, which can be found at the root of the source code - * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. - * If you do not have access to either file, you may request a copy from - * help@hdfgroup.org. - */ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* * Purpose: To test attribute handling for different dataset types. diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c index 14cbedf..79a571e 100644 --- a/test/vfd_swmr_bigset_writer.c +++ b/test/vfd_swmr_bigset_writer.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * @@ -151,6 +152,7 @@ typedef struct { bool use_vfd_swmr; bool use_legacy_swmr; bool use_named_pipe; + bool use_aux_proc; bool do_perf; bool cross_chunk_read; bool writer; @@ -217,6 +219,7 @@ state_initializer(void) .use_vfd_swmr = true, .use_legacy_swmr = false, .use_named_pipe = true, + .use_aux_proc = false, .do_perf = false, .cross_chunk_read = false, .writer = true, @@ -245,11 +248,12 @@ usage(const char *progname) { HDfprintf( stderr, - "usage: %s [-C] [-F] [-M] [-P] [-R] [-S] [-V] [-W] [-a steps] [-b] [-c cols]\n" + "usage: %s [-A] [-C] [-F] [-M] [-P] [-R] [-S] [-V] [-W] [-a steps] [-b] [-c cols]\n" " [-d dims] [-e depth] [-f tick_len] [-g max_lag] [-j skip_chunk] [-k part_chunk]\n" " [-l tick_num] [-n iterations] [-o page_buf_size] [-p fsp_size] [-r rows]\n" " [-s datasets] [-t] [-u over_extend] [-v chunk_cache_size] [-w deflate_level]\n" "\n" + "-A: use the auxiliary process to update the metadata file\n" "-C: cross-over chunk read during chunk verification\n" "-F: fixed maximal dimension for the chunked datasets\n" "-M: use virtual datasets and many source\n" @@ -342,8 +346,11 @@ state_init(state_t *s, int argc, char **argv) if (tfile) HDfree(tfile); - while ((ch = getopt(argc, argv, "CFMNPRSTVa:bc:d:e:f:g:j:k:l:m:n:o:p:qr:s:tu:v:w:")) != -1) { + while ((ch = getopt(argc, argv, "ACFMNPRSTVa:bc:d:e:f:g:j:k:l:m:n:o:p:qr:s:tu:v:w:")) != -1) { switch (ch) { + case 'A': + s->use_aux_proc = true; + break; case 'C': /* This flag indicates cross-over chunk read during data validation */ s->cross_chunk_read = true; @@ -488,6 +495,11 @@ state_init(state_t *s, int argc, char **argv) TEST_ERROR; } +#ifdef H5_HAVE_AUX_PROCESS + if (s->vds == vds_multi) + exit(EXIT_SUCCESS); +#endif + if (s->vds != vds_off && s->expand_2d) { HDfprintf(stderr, "virtual datasets and 2D datasets are mutually exclusive\n"); TEST_ERROR; @@ -1148,6 +1160,101 @@ error: return -1; } +/*------------------------------------------------------------------------- + * Function: md_ck_cb() + * + * Purpose: This is the callback function for debugging only. It's used + * when the H5F_ACS_GENERATE_MD_CK_CB_NAME property is set in fapl. + * --Opens and read the metadata file into a buffer. + * --Generate checksum for the metadata file + * --Write the tick number and the checksum to the checksum file + * + * Return: 0 if test is sucessful + * 1 if test fails + * + *------------------------------------------------------------------------- + */ +static herr_t +md_ck_cb(char *md_file_path, uint64_t updater_seq_num) +{ + FILE * md_fp = NULL; /* Metadata file pointer */ + FILE * chk_fp = NULL; /* Checksum file pointer */ + long size = 0; /* File size returned from HDftell() */ + void * buf = NULL; /* Buffer for holding the metadata file content */ + uint32_t chksum = 0; /* The checksum generated for the metadata file */ + char chk_name[1024 + 4]; /* Buffer for the checksum file name */ + size_t ret; /* Return value */ + + /* Open the metadata file */ + if ((md_fp = HDfopen(md_file_path, "r")) == NULL) + FAIL_STACK_ERROR; + + /* Set file pointer at end of file.*/ + if (HDfseek(md_fp, 0, SEEK_END) < 0) + FAIL_STACK_ERROR; + + /* Get the current position of the file pointer.*/ + if ((size = HDftell(md_fp)) < 0) + FAIL_STACK_ERROR; + + if (size != 0) { + + HDrewind(md_fp); + + if ((buf = HDmalloc((size_t)size)) == NULL) + FAIL_STACK_ERROR; + + /* Read the metadata file to buf */ + if ((ret = HDfread(buf, 1, (size_t)size, md_fp)) != (size_t)size) + FAIL_STACK_ERROR; + + /* Calculate checksum of the metadata file */ + chksum = H5_checksum_metadata(buf, (size_t)size, 0); + } + + /* Close the metadata file */ + if (md_fp && HDfclose(md_fp) < 0) + FAIL_STACK_ERROR; + + /* + * Checksum file + */ + + /* Generate checksum file name: <md_file_path>.chk */ + HDsprintf(chk_name, "%s.chk", md_file_path); + + /* Open checksum file for append */ + if ((chk_fp = HDfopen(chk_name, "a")) == NULL) + FAIL_STACK_ERROR; + + /* Write the updater sequence number to the checksum file */ + if ((ret = HDfwrite(&updater_seq_num, sizeof(uint64_t), 1, chk_fp)) != 1) + FAIL_STACK_ERROR; + + /* Write the checksum to the checksum file */ + if ((ret = HDfwrite(&chksum, sizeof(uint32_t), 1, chk_fp)) != 1) + FAIL_STACK_ERROR; + + /* Close the checksum file */ + if (chk_fp && HDfclose(chk_fp) != 0) + FAIL_STACK_ERROR; + + if (buf) + HDfree(buf); + + return 0; + +error: + if (buf) + HDfree(buf); + if (md_fp) + HDfclose(md_fp); + if (chk_fp) + HDfclose(chk_fp); + + return -1; +} /* md_ck_cb() */ + static bool create_extensible_dset(state_t *s, unsigned int which) { @@ -2437,12 +2544,13 @@ error: int main(int argc, char **argv) { - mat_t * mat; - hid_t fcpl = H5I_INVALID_HID; - unsigned which; - state_t s; - np_state_t np; - size_t i; + mat_t * mat; + hid_t fcpl = H5I_INVALID_HID; + unsigned which; + state_t s; + np_state_t np; + size_t i; + H5F_generate_md_ck_cb_t cb_info; /* Callback */ if (!state_init(&s, argc, argv)) { HDfprintf(stderr, "state_init failed\n"); @@ -2472,8 +2580,21 @@ main(int argc, char **argv) /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ +#ifdef H5_HAVE_AUX_PROCESS + /* If using the auxiliary process, the writer creates the updater files. + * The reader uses the metadata file generated by the auxiliary process. */ + if (s.writer) { + init_vfd_swmr_config(&config, s.tick_len, s.max_lag, s.writer, FALSE, TRUE, s.flush_raw_data, 128, + "./bigset-shadow-%zu", "bigset_updater", i); + } + else { + init_vfd_swmr_config(&config, s.tick_len, s.max_lag, s.writer, TRUE, FALSE, s.flush_raw_data, 128, + "./mdfile", NULL); + } +#else init_vfd_swmr_config(&config, s.tick_len, s.max_lag, s.writer, TRUE, FALSE, s.flush_raw_data, 128, "./bigset-shadow-%zu", NULL, i); +#endif /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, s.page_buf_size, &config)) < 0) { @@ -2502,6 +2623,17 @@ main(int argc, char **argv) } } + /* This part is for debugging only */ +#ifdef TMP + { + /* Set up callback to generate checksums for updater's metadata files */ + cb_info.func = md_ck_cb; + + /* Activate private property to generate checksums for updater's metadata file */ + H5Pset(fapl, H5F_ACS_GENERATE_MD_CK_CB_NAME, &cb_info); + } +#endif + s.file[i] = s.writer ? H5Fcreate(s.filename[i], H5F_ACC_TRUNC, fcpl, fapl) : H5Fopen(s.filename[i], H5F_ACC_RDONLY, fapl); diff --git a/test/vfd_swmr_check_compat.c b/test/vfd_swmr_check_compat.c index dec88c5..edf37a2 100644 --- a/test/vfd_swmr_check_compat.c +++ b/test/vfd_swmr_check_compat.c @@ -1,6 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * - * Copyright by the Board of Trustees of the University of Illinois. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/vfd_swmr_common.c b/test/vfd_swmr_common.c index 9b711dd..99db307 100644 --- a/test/vfd_swmr_common.c +++ b/test/vfd_swmr_common.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/vfd_swmr_common.h b/test/vfd_swmr_common.h index b842fe7..9f99c6e 100644 --- a/test/vfd_swmr_common.h +++ b/test/vfd_swmr_common.h @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/vfd_swmr_dsetchks_writer.c b/test/vfd_swmr_dsetchks_writer.c index c09c825..f2bd29b 100644 --- a/test/vfd_swmr_dsetchks_writer.c +++ b/test/vfd_swmr_dsetchks_writer.c @@ -1,15 +1,15 @@ -/* - * Copyright by The HDF Group. - * Copyright by the Board of Trustees of the University of Illinois. - * All rights reserved. - * - * This file is part of HDF5. The full HDF5 copyright notice, including - * terms governing use, modification, and redistribution, is contained in - * the COPYING file, which can be found at the root of the source code - * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. - * If you do not have access to either file, you may request a copy from - * help@hdfgroup.org. - */ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* * Purpose: To test chunk operations for chunked dataset specifically: diff --git a/test/vfd_swmr_dsetops_writer.c b/test/vfd_swmr_dsetops_writer.c index d999d64..d7ef255 100644 --- a/test/vfd_swmr_dsetops_writer.c +++ b/test/vfd_swmr_dsetops_writer.c @@ -1,15 +1,15 @@ -/* - * Copyright by The HDF Group. - * Copyright by the Board of Trustees of the University of Illinois. - * All rights reserved. - * - * This file is part of HDF5. The full HDF5 copyright notice, including - * terms governing use, modification, and redistribution, is contained in - * the COPYING file, which can be found at the root of the source code - * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. - * If you do not have access to either file, you may request a copy from - * help@hdfgroup.org. - */ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* * Purpose: To test writing operations for different dataset types. @@ -24,6 +24,35 @@ * -- Regular hyperslab writes * -- Raw data modifications */ +/* + * Modifications for testing references + * + * -O option for object reference + * --Writer: + * When creating the datasets via create_dsets(): + * --Create an object reference dataset + * --Store the dataset references via H5Rcreate_object() and save + * them in the object reference dataset + * --Reader: + * When opening the datasets via open_dsets(): + * --Open the object reference dataset + * --Retrieve the object references from the dataset + * --Open the dataset object via H5Ropen_object() + * + * -R option for region reference + * Writer: + * --When creating the datasets via create_dsets(): + * --Create a region reference dataset + * --When writing to the datasets via write_dset(): + * --Store the dataset selections via H5Rcreate_region() and save + * them in the region reference dataset + * Reader: + * --When opening the datasets via open_dsets(): + * --Open the region reference dataset + * --When verifying data written to a dataset via verify_dset(): + * --Retrieve the dataset selections from the region reference dataset + * --Get the selection via H5Ropen_region() + */ #include "hdf5.h" #include "testhdf5.h" @@ -47,6 +76,8 @@ typedef struct { bool use_np; /* For -N option */ bool use_vfd_swmr; /* For -S option */ bool flush_raw_data; /* For -U option */ + bool obj_ref; /* For -O option */ + bool reg_ref; /* For -R option */ bool compact; /* -p option: create compact dataset */ bool compact_write; /* -t option: write to the whole compact dataset */ unsigned int compact_elmts; /* -e <elmts> option: # of elments for the compact dataset */ @@ -69,25 +100,28 @@ typedef struct { .update_interval = READER_WAIT_TICKS, .csteps = 1, .use_np = true, .use_vfd_swmr = true, \ .flush_raw_data = true, .compact = false, .compact_write = false, .compact_elmts = MAX_COMPACT_ELMS, \ .contig = false, .rows = 10, .cols = 5, .swrites = 0, .rwrites = 0, .lwrites = 0, .wwrites = 0, \ - .lastwrite = 0 \ + .lastwrite = 0, .obj_ref = false, .reg_ref = false \ } /* Structure to hold info for different dataset types */ typedef struct { - hid_t compact_did; /* ID for compact dataset */ - hid_t contig_did; /* ID for contiguous dataset */ - hid_t single_did; /* ID for chunked dataset: single index */ - hid_t implicit_did; /* ID for chunked dataset: implicit index */ - hid_t fa_did; /* ID for chunked dataset: fixed array index */ - hid_t ea_did; /* ID for chunked dataset: extensible array index */ - hid_t bt2_did; /* ID for chunked dataset: version 2 btree index */ - hid_t compact_sid; /* Dataspace ID for compact dataset */ - hid_t contig_sid; /* Dataspace ID for contiguous dataset */ - hid_t single_sid; /* Dataspace ID for chunked dataset */ - hid_t implicit_sid; /* Dataspace ID for chunked dataset */ - hid_t fa_sid; /* Dataspace ID for chunked dataset */ - hid_t ea_sid; /* Dataspace ID for chunked dataset */ - hid_t bt2_sid; /* Dataspace ID for chunked dataset */ + hid_t compact_did; /* ID for compact dataset */ + hid_t contig_did; /* ID for contiguous dataset */ + hid_t single_did; /* ID for chunked dataset: single index */ + hid_t implicit_did; /* ID for chunked dataset: implicit index */ + hid_t fa_did; /* ID for chunked dataset: fixed array index */ + hid_t ea_did; /* ID for chunked dataset: extensible array index */ + hid_t bt2_did; /* ID for chunked dataset: version 2 btree index */ + hid_t compact_sid; /* Dataspace ID for compact dataset */ + hid_t contig_sid; /* Dataspace ID for contiguous dataset */ + hid_t single_sid; /* Dataspace ID for chunked dataset */ + hid_t implicit_sid; /* Dataspace ID for chunked dataset */ + hid_t fa_sid; /* Dataspace ID for chunked dataset */ + hid_t ea_sid; /* Dataspace ID for chunked dataset */ + hid_t bt2_sid; /* Dataspace ID for chunked dataset */ + hid_t obj_did; /* ID for object reference dataset */ + hid_t reg_did; /* ID for region reference dataset */ + H5R_ref_t *reg_buf; /* Buffer for holding the region references */ } dsets_state_t; /* Initializations for dsets_state_t */ @@ -98,7 +132,8 @@ typedef struct { .contig_sid = H5I_INVALID_HID, .single_did = H5I_INVALID_HID, .single_sid = H5I_INVALID_HID, \ .implicit_did = H5I_INVALID_HID, .implicit_sid = H5I_INVALID_HID, .fa_did = H5I_INVALID_HID, \ .fa_sid = H5I_INVALID_HID, .ea_did = H5I_INVALID_HID, .ea_sid = H5I_INVALID_HID, \ - .bt2_did = H5I_INVALID_HID, .bt2_sid = H5I_INVALID_HID \ + .bt2_did = H5I_INVALID_HID, .bt2_sid = H5I_INVALID_HID, .obj_did = H5I_INVALID_HID, \ + .reg_did = H5I_INVALID_HID, .reg_buf = NULL \ } /* Structure to hold info for named pipes */ @@ -131,8 +166,8 @@ static bool np_confirm_verify_notify(int fd, unsigned step, const state_t *s, np static bool create_dsets(const state_t *s, dsets_state_t *ds); static bool open_dsets(const state_t *s, dsets_state_t *ds); -static bool open_dset_real(const state_t *s, hid_t *did, hid_t *sid, const char *name); -static bool close_dsets(const dsets_state_t *ds); +static bool open_dset_real(const state_t *s, hid_t *did, hid_t *sid, const char *name, H5R_ref_t *obj_buf); +static bool close_dsets(dsets_state_t *ds); static bool close_dset_real(hid_t did, hid_t sid); static bool perform_dsets_operations(state_t *s, dsets_state_t *ds, H5F_vfd_swmr_config_t *config, @@ -140,17 +175,18 @@ static bool perform_dsets_operations(state_t *s, dsets_state_t *ds, H5F_vfd_swmr static bool dsets_action(unsigned action, const state_t *s, const dsets_state_t *ds, unsigned step); static bool dset_setup(unsigned action, unsigned which, const state_t *s, hsize_t *start, hsize_t *stride, hsize_t *count, hsize_t *block, hid_t *mem_sid, unsigned int **buf); -static bool write_dset(hid_t did, hid_t tid, hid_t mem_sid, hid_t file_sid, hsize_t *start, hsize_t *stride, - hsize_t *count, hsize_t *block, unsigned int *buf); +static bool write_dset(const state_t *s, const char *name, hid_t did, hid_t sid, hid_t mem_sid, + hsize_t *start, hsize_t *stride, hsize_t *count, hsize_t *block, unsigned int *buf, + H5R_ref_t *reg_buf); static bool write_dset_compact(const state_t *s, const dsets_state_t *ds); static bool verify_dsets_operations(state_t *s, dsets_state_t *ds, H5F_vfd_swmr_config_t *config, np_state_t *np, bool fileclosed); static bool verify_dsets_action(unsigned action, const state_t *s, const dsets_state_t *ds, unsigned which, bool fileclosed); -static bool verify_dset(hid_t did, hid_t tid, hid_t mem_sid, hid_t file_sid, hsize_t *start, hsize_t *stride, +static bool verify_dset(hid_t did, hid_t tid, hid_t sid, hid_t mem_sid, hsize_t *start, hsize_t *stride, hsize_t *count, hsize_t *block, unsigned int *vbuf, bool fileclosed, - bool flush_raw_data); + bool flush_raw_data, H5R_ref_t *rbuf); static bool verify_dset_compact(const state_t *s, const dsets_state_t *ds, bool fileclosed, bool flush_raw_data); @@ -168,6 +204,11 @@ static const hid_t badhid = H5I_INVALID_HID; #define DSET_EA_NAME "chunked_ea" #define DSET_BT2_NAME "chunked_bt2" +#define DSET_OBJ_REF_NAME "obj_ref_dset" /* Object reference dataset */ +#define DSET_REG_REF_NAME "reg_ref_dset" /* Region reference dataset */ +#define OBJ_REF_DIMS 7 /* Dimension size for object reference dataset */ +#define REG_REF_DIMS 6 /* Dimension size for region reference dataset */ + /* Action for writes */ #define SEQ_WRITE 1 /* Sequential write */ #define RANDOM_WRITE 2 /* Random write */ @@ -183,7 +224,7 @@ usage(const char *progname) " [-p] [-e elmts] [-o]\n" " [-g] [-k] [-m rows] [-n cols]\n" " [-s swrites] [-r rwrites] [-l lwrites] [-w writes]\n" - " [-u nticks] [-c csteps] [-U] [-S] [-N] [-q] [-b]\n" + " [-u nticks] [-c csteps] [-U] [-O] [-R] [-S] [-N] [-q] [-b]\n" "\n" "-p: create a dataset with compact layout\n" "-e elmts: # of <elmts> for the compact dataset\n" @@ -203,6 +244,8 @@ usage(const char *progname) "-c csteps: `csteps` steps communication interval between reader and writer\n" " (default is 1)\n" "-U: disable flush of raw data (default is flushing raw data)\n" + "-O: create object references to datasets\n" + "-R: create region references to datasets\n" "-S: do not use VFD SWMR\n" "-N: do not use named pipes for test synchronization\n" "-q: silence printouts, few messages\n" @@ -242,7 +285,7 @@ state_init(state_t *s, int argc, char **argv) tfile = NULL; } - while ((ch = getopt(argc, argv, "pte:gkm:n:s:r:l:w:bqSNUu:c:")) != -1) { + while ((ch = getopt(argc, argv, "pte:gkm:n:s:r:l:w:bqSNUORu:c:")) != -1) { switch (ch) { case 'p': /* compact dataset */ @@ -265,6 +308,14 @@ state_init(state_t *s, int argc, char **argv) s->flush_raw_data = false; break; + case 'O': /* Create object reference to datasets */ + s->obj_ref = true; + break; + + case 'R': /* Create region reference to datasets */ + s->reg_ref = true; + break; + case 'q': verbosity = 0; break; @@ -355,6 +406,20 @@ state_init(state_t *s, int argc, char **argv) goto error; } + /* Object reference applies to either compact or contiguous or chunked datasets */ + if (s->obj_ref && !(s->compact || s->contig || s->chunked)) { + HDprintf("Enable object reference without compact/contig/chunked dataset\n"); + usage(s->progname); + goto error; + } + + /* Region reference applies to either contiguous or chunked datasets */ + if (s->reg_ref && !(s->contig || s->chunked)) { + HDprintf("Enable region reference without contig/chunked dataset\n"); + usage(s->progname); + goto error; + } + /* Enable sequential/random/hyperslab/raw data writes (-s/-r/-l/-w) without contiguous/chunked dataset * (-g/-k) */ if ((s->swrites || s->rwrites || s->lwrites || s->wwrites) && !(s->contig || s->chunked)) { @@ -412,12 +477,17 @@ error: static bool create_dsets(const state_t *s, dsets_state_t *ds) { - hid_t dcpl = badhid; - hid_t dtid = badhid; + hid_t dcpl = badhid; + hid_t dtid = badhid; + unsigned i; + H5R_ref_t *obj_buf = NULL; /* Buffer for object references */ + hid_t sid = badhid; + hsize_t obj_dims = OBJ_REF_DIMS; /* Dimension for object reference dataset */ + hsize_t reg_dims = REG_REF_DIMS; /* Dimension for region reference dataset */ *ds = DSETS_INITIALIZER; - /* Create the named datatype that will be used by compact and contiguous datasets */ + /* Create the named datatype that will be used by compact dataset */ if ((dtid = H5Tcopy(s->filetype)) < 0) { HDprintf("H5Tcopy failed\n"); TEST_ERROR; @@ -462,7 +532,7 @@ create_dsets(const state_t *s, dsets_state_t *ds) } } - /* Dataset with contiguous layout, 2d, named datatype */ + /* Dataset with contiguous layout, 2d */ if (s->contig) { hsize_t dims[2]; @@ -485,9 +555,9 @@ create_dsets(const state_t *s, dsets_state_t *ds) TEST_ERROR; } - /* Create the contiguous dataset with the named datatype */ - if ((ds->contig_did = H5Dcreate2(s->file, DSET_CONTIG_NAME, dtid, ds->contig_sid, H5P_DEFAULT, dcpl, - H5P_DEFAULT)) < 0) { + /* Create the contiguous dataset with DEFAULT datatype */ + if ((ds->contig_did = H5Dcreate2(s->file, DSET_CONTIG_NAME, s->filetype, ds->contig_sid, H5P_DEFAULT, + dcpl, H5P_DEFAULT)) < 0) { HDprintf("H5Dcreate2 contiguous dataset failed\n"); TEST_ERROR; } @@ -498,7 +568,7 @@ create_dsets(const state_t *s, dsets_state_t *ds) } } - /* Dataset with chunked layout, 2d, named datatype */ + /* Dataset with chunked layout, 2d */ if (s->chunked) { hsize_t dims[2]; hsize_t max_dims[2]; @@ -534,9 +604,9 @@ create_dsets(const state_t *s, dsets_state_t *ds) TEST_ERROR; } - /* Create the chunked dataset (single index) with the named datatype */ - if ((ds->single_did = H5Dcreate2(s->file, DSET_SINGLE_NAME, dtid, ds->single_sid, H5P_DEFAULT, dcpl, - H5P_DEFAULT)) < 0) { + /* Create the chunked dataset (single index) with the default datatype */ + if ((ds->single_did = H5Dcreate2(s->file, DSET_SINGLE_NAME, s->filetype, ds->single_sid, H5P_DEFAULT, + dcpl, H5P_DEFAULT)) < 0) { HDprintf("H5Dcreate2 chunked dataset:single index failed\n"); TEST_ERROR; } @@ -559,9 +629,9 @@ create_dsets(const state_t *s, dsets_state_t *ds) TEST_ERROR; } - /* Create the chunked dataset (implicit index) with the named datatype */ - if ((ds->implicit_did = H5Dcreate2(s->file, DSET_IMPLICIT_NAME, dtid, ds->implicit_sid, H5P_DEFAULT, - dcpl, H5P_DEFAULT)) < 0) { + /* Create the chunked dataset (implicit index) with the default datatype */ + if ((ds->implicit_did = H5Dcreate2(s->file, DSET_IMPLICIT_NAME, s->filetype, ds->implicit_sid, + H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) { HDprintf("H5Dcreate2 chunked dataset:implicit index failed\n"); TEST_ERROR; } @@ -592,9 +662,9 @@ create_dsets(const state_t *s, dsets_state_t *ds) TEST_ERROR; } - /* Create the chunked dataset (fixed array index) with the named datatype */ - if ((ds->fa_did = - H5Dcreate2(s->file, DSET_FA_NAME, dtid, ds->fa_sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) { + /* Create the chunked dataset (fixed array index) with the default datatype */ + if ((ds->fa_did = H5Dcreate2(s->file, DSET_FA_NAME, s->filetype, ds->fa_sid, H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) { HDprintf("H5Dcreaet2 chunked dataset: fa index failed\n"); TEST_ERROR; } @@ -608,9 +678,9 @@ create_dsets(const state_t *s, dsets_state_t *ds) TEST_ERROR; } - /* Create the chunked dataset (extensible array index) with the named datatype */ - if ((ds->ea_did = - H5Dcreate2(s->file, DSET_EA_NAME, dtid, ds->ea_sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) { + /* Create the chunked dataset (extensible array index) with the default datatype */ + if ((ds->ea_did = H5Dcreate2(s->file, DSET_EA_NAME, s->filetype, ds->ea_sid, H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) { HDprintf("H5Dcreate2 chunked dataset: ea index failed\n"); TEST_ERROR; } @@ -624,9 +694,9 @@ create_dsets(const state_t *s, dsets_state_t *ds) TEST_ERROR; } - /* Create the chunked dataset (btree2 index) with the named datatype */ - if ((ds->bt2_did = - H5Dcreate2(s->file, DSET_BT2_NAME, dtid, ds->bt2_sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) { + /* Create the chunked dataset (btree2 index) with the default datatype */ + if ((ds->bt2_did = H5Dcreate2(s->file, DSET_BT2_NAME, s->filetype, ds->bt2_sid, H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) { HDprintf("H5Dcreate2 chunked dataset: bt2 index failed\n"); TEST_ERROR; } @@ -642,6 +712,104 @@ create_dsets(const state_t *s, dsets_state_t *ds) TEST_ERROR; } + /* If object reference is enabled: */ + /* create the object reference dataset and the references to the dataset objects */ + if (s->obj_ref) { + if ((obj_buf = HDcalloc(sizeof(H5R_ref_t), OBJ_REF_DIMS)) == NULL) { + HDprintf("HDcalloc failed\n"); + TEST_ERROR; + } + if ((sid = H5Screate_simple(1, &obj_dims, NULL)) < 0) { + HDprintf("H5Screate_simple failed\n"); + TEST_ERROR; + } + if ((ds->obj_did = H5Dcreate2(s->file, DSET_OBJ_REF_NAME, H5T_STD_REF, sid, H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT)) < 0) { + HDprintf("H5Dcreate2 object reference dataset failed\n"); + TEST_ERROR; + } + + if (s->compact) + if (H5Rcreate_object(s->file, DSET_COMPACT_NAME, H5P_DEFAULT, &obj_buf[0]) < 0) { + HDprintf("H5Rcreate_object failed\n"); + TEST_ERROR; + } + if (s->contig) + if (H5Rcreate_object(s->file, DSET_CONTIG_NAME, H5P_DEFAULT, &obj_buf[1]) < 0) { + HDprintf("H5Rcreate_object failed\n"); + TEST_ERROR; + } + if (s->chunked) { + if (H5Rcreate_object(s->file, DSET_SINGLE_NAME, H5P_DEFAULT, &obj_buf[2]) < 0) { + HDprintf("H5Rcreate_object failed\n"); + TEST_ERROR; + } + + if (H5Rcreate_object(s->file, DSET_IMPLICIT_NAME, H5P_DEFAULT, &obj_buf[3]) < 0) { + HDprintf("H5Rcreate_object failed\n"); + TEST_ERROR; + } + if (H5Rcreate_object(s->file, DSET_FA_NAME, H5P_DEFAULT, &obj_buf[4]) < 0) { + HDprintf("H5Rcreate_object failed\n"); + TEST_ERROR; + } + if (H5Rcreate_object(s->file, DSET_EA_NAME, H5P_DEFAULT, &obj_buf[5]) < 0) { + HDprintf("H5Rcreate_object failed\n"); + TEST_ERROR; + } + if (H5Rcreate_object(s->file, DSET_BT2_NAME, H5P_DEFAULT, &obj_buf[6]) < 0) { + HDprintf("H5Rcreate_object failed\n"); + TEST_ERROR; + } + } + + /* Write the object references to the reference dataset */ + if (H5Dwrite(ds->obj_did, H5T_STD_REF, H5S_ALL, H5S_ALL, H5P_DEFAULT, obj_buf) < 0) { + HDprintf("H5Dwrite object reference dataset failed\n"); + TEST_ERROR; + } + + /* Destroy the references */ + for (i = 0; i < OBJ_REF_DIMS; i++) + if (H5Rdestroy(&obj_buf[i]) < 0) { + HDprintf("H5Rdestroy object reference failed\n"); + TEST_ERROR; + } + + if (H5Sclose(sid) < 0) { + HDprintf("H5Sclose \n"); + TEST_ERROR; + } + + if (obj_buf) + HDfree(obj_buf); + } + + /* If region reference is enabled: */ + /* create the region reference dataset and allocate the buffer for holding the references */ + if (s->reg_ref) { + if ((sid = H5Screate_simple(1, ®_dims, NULL)) < 0) { + HDprintf("H5Screate_simple failed\n"); + TEST_ERROR; + } + + if ((ds->reg_did = H5Dcreate2(s->file, DSET_REG_REF_NAME, H5T_STD_REF, sid, H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT)) < 0) { + HDprintf("H5Dcreate2 failed\n"); + TEST_ERROR; + } + + if (H5Sclose(sid) < 0) { + HDprintf("H5Sclose failed\n"); + TEST_ERROR; + } + + if ((ds->reg_buf = HDcalloc(sizeof(H5R_ref_t), REG_REF_DIMS)) == NULL) { + HDprintf("HDcalloc failed\n"); + TEST_ERROR; + } + } + return true; error: @@ -656,16 +824,24 @@ error: H5Sclose(ds->fa_sid); H5Sclose(ds->ea_sid); H5Sclose(ds->bt2_sid); + H5Sclose(sid); H5Dclose(ds->compact_did); H5Dclose(ds->contig_did); H5Dclose(ds->single_did); - H5Dclose(ds->implicit_did); + H5Sclose(ds->implicit_did); H5Dclose(ds->fa_did); H5Dclose(ds->ea_did); H5Dclose(ds->bt2_did); + H5Dclose(ds->obj_did); + H5Dclose(ds->reg_did); } H5E_END_TRY; + if (obj_buf) + HDfree(obj_buf); + if (ds->reg_buf) + HDfree(ds->reg_buf); + return false; } /* create_dsets() */ @@ -676,10 +852,54 @@ error: static bool open_dsets(const state_t *s, dsets_state_t *ds) { + H5R_ref_t *obj_buf = NULL; /* Buffer for holding object references */ + unsigned i; + *ds = DSETS_INITIALIZER; + /* If object reference is specified: */ + /* open the object reference dataset and retrieve the dataset object references */ + if (s->obj_ref) { + if ((ds->obj_did = H5Dopen2(s->file, DSET_OBJ_REF_NAME, H5P_DEFAULT)) < 0) { + HDprintf("HDopen2 object reference dataset failed\n"); + TEST_ERROR; + } + + if (H5Drefresh(ds->obj_did) < 0) { + HDprintf("HDrefresh failed\n"); + TEST_ERROR; + } + + if ((obj_buf = HDcalloc(sizeof(H5R_ref_t), OBJ_REF_DIMS)) == NULL) { + HDprintf("HDcalloc failed\n"); + TEST_ERROR; + } + + /* Obtain the object references */ + if (H5Dread(ds->obj_did, H5T_STD_REF, H5S_ALL, H5S_ALL, H5P_DEFAULT, obj_buf) < 0) { + HDprintf("HDread failed\n"); + TEST_ERROR; + } + } + + /* If region reference is specified: */ + /* open the region reference dataset and allocate buffer for holding the region references */ + if (s->reg_ref) { + if ((ds->reg_did = H5Dopen2(s->file, DSET_REG_REF_NAME, H5P_DEFAULT)) < 0) { + HDprintf("HDopen2 region reference dataset failed\n"); + TEST_ERROR; + } + + /* Allocate the buffer for holding the region references */ + if ((ds->reg_buf = HDcalloc(sizeof(H5R_ref_t), REG_REF_DIMS)) == NULL) { + HDprintf("HDcalloc failed\n"); + TEST_ERROR; + } + } + if (s->compact) { - if (!open_dset_real(s, &ds->compact_did, &ds->compact_sid, DSET_COMPACT_NAME)) { + if (!open_dset_real(s, &ds->compact_did, &ds->compact_sid, DSET_COMPACT_NAME, + s->obj_ref ? &obj_buf[0] : NULL)) { HDprintf("open_dset_real() for compact dataset failed\n"); TEST_ERROR; } @@ -687,7 +907,8 @@ open_dsets(const state_t *s, dsets_state_t *ds) if (s->contig) { - if (!open_dset_real(s, &ds->contig_did, &ds->contig_sid, DSET_CONTIG_NAME)) { + if (!open_dset_real(s, &ds->contig_did, &ds->contig_sid, DSET_CONTIG_NAME, + s->obj_ref ? &obj_buf[1] : NULL)) { HDprintf("open_dset_real() for contiguous dataset failed\n"); TEST_ERROR; } @@ -695,35 +916,61 @@ open_dsets(const state_t *s, dsets_state_t *ds) if (s->chunked) { - if (!open_dset_real(s, &ds->single_did, &ds->single_sid, DSET_SINGLE_NAME)) { + if (!open_dset_real(s, &ds->single_did, &ds->single_sid, DSET_SINGLE_NAME, + s->obj_ref ? &obj_buf[2] : NULL)) { HDprintf("open_dset_real() for chunked dataset: single index failed\n"); TEST_ERROR; } - if (!open_dset_real(s, &ds->implicit_did, &ds->implicit_sid, DSET_IMPLICIT_NAME)) { + if (!open_dset_real(s, &ds->implicit_did, &ds->implicit_sid, DSET_IMPLICIT_NAME, + s->obj_ref ? &obj_buf[3] : NULL)) { HDprintf("open_dset_real() for chunked dataset: implicit index failed\n"); TEST_ERROR; } - if (!open_dset_real(s, &ds->fa_did, &ds->fa_sid, DSET_FA_NAME)) { + if (!open_dset_real(s, &ds->fa_did, &ds->fa_sid, DSET_FA_NAME, s->obj_ref ? &obj_buf[4] : NULL)) { HDprintf("open_dset_real() for chunked dataset: fa index failed\n"); TEST_ERROR; } - if (!open_dset_real(s, &ds->ea_did, &ds->ea_sid, DSET_EA_NAME)) { + if (!open_dset_real(s, &ds->ea_did, &ds->ea_sid, DSET_EA_NAME, s->obj_ref ? &obj_buf[5] : NULL)) { HDprintf("open_dset_real() for chunked dataset: ea index failed\n"); TEST_ERROR; } - if (!open_dset_real(s, &ds->bt2_did, &ds->bt2_sid, DSET_BT2_NAME)) { + if (!open_dset_real(s, &ds->bt2_did, &ds->bt2_sid, DSET_BT2_NAME, s->obj_ref ? &obj_buf[6] : NULL)) { HDprintf("open_dset_real() for chunked dataset: bt2 index failed\n"); TEST_ERROR; } } + /* Destroy the object references */ + if (s->obj_ref) { + for (i = 0; i < OBJ_REF_DIMS; i++) + if (H5Rdestroy(&obj_buf[i]) < 0) { + HDprintf("H5Rdestroy object references\n"); + TEST_ERROR; + } + + if (obj_buf) + HDfree(obj_buf); + } + return true; error: + H5E_BEGIN_TRY + { + H5Dclose(ds->obj_did); + H5Dclose(ds->reg_did); + } + H5E_END_TRY; + + if (obj_buf) + HDfree(obj_buf); + if (ds->reg_buf) + HDfree(ds->reg_buf); + return false; } /* open_dsets() */ @@ -733,13 +980,30 @@ error: * Verify the dimension sizes are as expected. */ static bool -open_dset_real(const state_t *s, hid_t *did, hid_t *sid, const char *name) +open_dset_real(const state_t *s, hid_t *did, hid_t *sid, const char *name, H5R_ref_t *obj_buf) { hsize_t dims[2]; - if ((*did = H5Dopen2(s->file, name, H5P_DEFAULT)) < 0) { - HDprintf("H5Dopen dataset failed\n"); - TEST_ERROR; + /* If object reference is enabled, obtain the dataset object ID + via H5Ropen_object(). */ + if (s->obj_ref) { + H5E_BEGIN_TRY + { + *did = H5Ropen_object(obj_buf, H5P_DEFAULT, H5P_DEFAULT); + } + H5E_END_TRY; + if (*did < 0) { + HDprintf("H5Ropen_object failed\n"); + TEST_ERROR; + } + } + else { + HDassert(obj_buf == NULL); + + if ((*did = H5Dopen2(s->file, name, H5P_DEFAULT)) < 0) { + HDprintf("H5Dopen dataset failed\n"); + TEST_ERROR; + } } if ((*sid = H5Dget_space(*did)) < 0) { @@ -777,7 +1041,7 @@ error: * Close all the datasets and dataspaces as specified. */ static bool -close_dsets(const dsets_state_t *ds) +close_dsets(dsets_state_t *ds) { if (!close_dset_real(ds->compact_did, ds->compact_sid)) { HDprintf("H5Dclose compact dataset failed\n"); @@ -814,6 +1078,11 @@ close_dsets(const dsets_state_t *ds) TEST_ERROR; } + if (ds->reg_buf) { + HDfree(ds->reg_buf); + ds->reg_buf = NULL; + } + return true; error: @@ -979,6 +1248,7 @@ dsets_action(unsigned action, const state_t *s, const dsets_state_t *ds, unsigne hsize_t block[2]; hid_t mem_sid; unsigned int *wbuf = NULL; + unsigned i; /* Set up selection, dataspace and data buffer according to the specified action */ if (!dset_setup(action, which, s, start, stride, count, block, &mem_sid, &wbuf)) { @@ -989,8 +1259,8 @@ dsets_action(unsigned action, const state_t *s, const dsets_state_t *ds, unsigne /* Write to the contiguous dataset */ if (s->contig) { - if (!write_dset(ds->contig_did, s->filetype, mem_sid, ds->contig_sid, start, stride, count, block, - wbuf)) { + if (!write_dset(s, DSET_CONTIG_NAME, ds->contig_did, ds->contig_sid, mem_sid, start, stride, count, + block, wbuf, s->reg_ref ? &ds->reg_buf[0] : NULL)) { HDprintf("H5Dwrite to contiguous dataset failed\n"); TEST_ERROR; } @@ -999,36 +1269,53 @@ dsets_action(unsigned action, const state_t *s, const dsets_state_t *ds, unsigne /* Write to the 5 chunked datasets */ if (s->chunked) { - if (!write_dset(ds->single_did, s->filetype, mem_sid, ds->single_sid, start, stride, count, block, - wbuf)) { + if (!write_dset(s, DSET_SINGLE_NAME, ds->single_did, ds->single_sid, mem_sid, start, stride, count, + block, wbuf, s->reg_ref ? &ds->reg_buf[1] : NULL)) { HDprintf("H5Dwrite to chunked dataset: single index dataset failed\n"); TEST_ERROR; } - if (!write_dset(ds->implicit_did, s->filetype, mem_sid, ds->implicit_sid, start, stride, count, block, - wbuf)) { + if (!write_dset(s, DSET_IMPLICIT_NAME, ds->implicit_did, ds->implicit_sid, mem_sid, start, stride, + count, block, wbuf, s->reg_ref ? &ds->reg_buf[2] : NULL)) { HDprintf("H5Dwrite to chunked dataset: implicit index dataset failed\n"); TEST_ERROR; } - if (!write_dset(ds->fa_did, s->filetype, mem_sid, ds->fa_sid, start, stride, count, block, wbuf)) { + if (!write_dset(s, DSET_FA_NAME, ds->fa_did, ds->fa_sid, mem_sid, start, stride, count, block, wbuf, + s->reg_ref ? &ds->reg_buf[3] : NULL)) { HDprintf("H5Dwrite to chunked dataset: fa index dataset failed\n"); TEST_ERROR; } - if (!write_dset(ds->ea_did, s->filetype, mem_sid, ds->ea_sid, start, stride, count, block, wbuf)) { + if (!write_dset(s, DSET_EA_NAME, ds->ea_did, ds->ea_sid, mem_sid, start, stride, count, block, wbuf, + s->reg_ref ? &ds->reg_buf[4] : NULL)) { HDprintf("H5Dwrite to chunked dataset: ea index dataset failed\n"); TEST_ERROR; } - if (!write_dset(ds->bt2_did, s->filetype, mem_sid, ds->bt2_sid, start, stride, count, block, wbuf)) { + if (!write_dset(s, DSET_BT2_NAME, ds->bt2_did, ds->bt2_sid, mem_sid, start, stride, count, block, + wbuf, s->reg_ref ? &ds->reg_buf[5] : NULL)) { HDprintf("H5Dwrite to chunked dataset: bt2 index dataset failed\n"); TEST_ERROR; } } + /* If region reference is enabled, store the region references to the reference dataset */ + if (s->reg_ref) { + if (H5Dwrite(ds->reg_did, H5T_STD_REF, H5S_ALL, H5S_ALL, H5P_DEFAULT, ds->reg_buf) < 0) { + HDprintf("H5Dwrite failed\n"); + TEST_ERROR; + } + for (i = 0; i < 6; i++) + if (H5Rdestroy(&ds->reg_buf[i]) < 0) { + HDprintf("H5Rdestroy failed\n"); + TEST_ERROR; + } + } + if (wbuf) HDfree(wbuf); + return true; error: @@ -1126,20 +1413,28 @@ error: * Make the selection and then write to the dataset. */ static bool -write_dset(hid_t did, hid_t tid, hid_t mem_sid, hid_t file_sid, hsize_t *start, hsize_t *stride, - hsize_t *count, hsize_t *block, unsigned int *buf) +write_dset(const state_t *s, const char *name, hid_t did, hid_t sid, hid_t mem_sid, hsize_t *start, + hsize_t *stride, hsize_t *count, hsize_t *block, unsigned int *buf, H5R_ref_t *reg_buf) { - - if (H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, start, stride, count, block) < 0) { + if (H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, stride, count, block) < 0) { HDprintf("H5Sselect to dataset failed\n"); TEST_ERROR; } - if (H5Dwrite(did, tid, mem_sid, file_sid, H5P_DEFAULT, buf) < 0) { + if (H5Dwrite(did, s->filetype, mem_sid, sid, H5P_DEFAULT, buf) < 0) { HDprintf("H5Dwrite to dataset failed\n"); TEST_ERROR; } + /* If region reference is enabled, store the region reference to the reference buffer */ + if (s->reg_ref) { + HDassert(reg_buf != NULL); + if (H5Rcreate_region(s->file, name, sid, H5P_DEFAULT, reg_buf) < 0) { + HDprintf("H5Rcreate_region failed\n"); + TEST_ERROR; + } + } + return true; error: @@ -1218,7 +1513,7 @@ verify_dsets_operations(state_t *s, dsets_state_t *ds, H5F_vfd_swmr_config_t *co result = verify_dset_compact(s, ds, fileclosed, config->flush_raw_data); if (s->use_np && !np_reader(result, 0, s, np)) { - HDprintf("np_reader() for verifying addition failed\n"); + HDprintf("np_reader() for verifying writes failed\n"); TEST_ERROR; } else if (!result) @@ -1250,7 +1545,7 @@ verify_dsets_operations(state_t *s, dsets_state_t *ds, H5F_vfd_swmr_config_t *co result = verify_dsets_action(SEQ_WRITE, s, ds, step, fileclosed); if (s->use_np && !np_reader(result, step, s, np)) { - HDprintf("np_reader() for verifying addition failed\n"); + HDprintf("np_reader() for verifying writes failed\n"); TEST_ERROR; } else if (!result) @@ -1286,7 +1581,7 @@ verify_dsets_operations(state_t *s, dsets_state_t *ds, H5F_vfd_swmr_config_t *co result = verify_dsets_action(RANDOM_WRITE, s, ds, newstep, fileclosed); if (s->use_np && !np_reader(result, step, s, np)) { - HDprintf("np_reader() for verifying addition failed\n"); + HDprintf("np_reader() for verifying writes failed\n"); TEST_ERROR; } else if (!result) @@ -1316,7 +1611,7 @@ verify_dsets_operations(state_t *s, dsets_state_t *ds, H5F_vfd_swmr_config_t *co result = verify_dsets_action(HYPER_WRITE, s, ds, k, fileclosed); if (s->use_np && !np_reader(result, step, s, np)) { - HDprintf("np_reader() for verifying addition failed\n"); + HDprintf("np_reader() for verifying writes failed\n"); TEST_ERROR; } else if (!result) @@ -1345,7 +1640,7 @@ verify_dsets_operations(state_t *s, dsets_state_t *ds, H5F_vfd_swmr_config_t *co result = verify_dsets_action(MODIFY_DATA, s, ds, step, fileclosed); if (s->use_np && !np_reader(result, step, s, np)) { - HDprintf("np_reader() for verifying addition failed\n"); + HDprintf("np_reader() for verifying writes failed\n"); TEST_ERROR; } else if (!result) @@ -1359,7 +1654,7 @@ verify_dsets_operations(state_t *s, dsets_state_t *ds, H5F_vfd_swmr_config_t *co error: return false; -} /* Verify_dsets_operations() */ +} /* verify_dsets_operations() */ /* * Verify the data read from each of the datasets specified on the command line @@ -1379,6 +1674,19 @@ verify_dsets_action(unsigned action, const state_t *s, const dsets_state_t *ds, hsize_t block[2]; hid_t mem_sid; unsigned int *vbuf = NULL; + unsigned i; + + /* If region reference is enabled, obtain the region references from the reference dataset */ + if (s->reg_ref) { + if (H5Drefresh(ds->reg_did) < 0) { + HDprintf("H5Drefresh region reference dataset failed\n"); + TEST_ERROR; + } + if (H5Dread(ds->reg_did, H5T_STD_REF, H5S_ALL, H5S_ALL, H5P_DEFAULT, ds->reg_buf) < 0) { + HDprintf("H5Dread failed\n"); + TEST_ERROR; + } + } /* Set up selection, dataspace and data buffer according to the specified action */ if (!dset_setup(action, which, s, start, stride, count, block, &mem_sid, &vbuf)) { @@ -1388,8 +1696,9 @@ verify_dsets_action(unsigned action, const state_t *s, const dsets_state_t *ds, /* Verify the data read for the contiguous dataset */ if (s->contig) { - if (!verify_dset(ds->contig_did, s->filetype, mem_sid, ds->contig_sid, start, stride, count, block, - vbuf, fileclosed, s->flush_raw_data)) { + + if (!verify_dset(ds->contig_did, s->filetype, ds->contig_sid, mem_sid, start, stride, count, block, + vbuf, fileclosed, s->flush_raw_data, s->reg_ref ? &ds->reg_buf[0] : NULL)) { HDprintf("verify_dset() to contiguous dataset failed\n"); TEST_ERROR; } @@ -1398,32 +1707,32 @@ verify_dsets_action(unsigned action, const state_t *s, const dsets_state_t *ds, /* Verify the data read for the chunked datasets */ if (s->chunked) { - if (!verify_dset(ds->single_did, s->filetype, mem_sid, ds->single_sid, start, stride, count, block, - vbuf, fileclosed, s->flush_raw_data)) { + if (!verify_dset(ds->single_did, s->filetype, ds->single_sid, mem_sid, start, stride, count, block, + vbuf, fileclosed, s->flush_raw_data, s->reg_ref ? &ds->reg_buf[1] : NULL)) { HDprintf("verify_dset() to chunked dataset: single index dataset failed\n"); TEST_ERROR; } - if (!verify_dset(ds->implicit_did, s->filetype, mem_sid, ds->implicit_sid, start, stride, count, - block, vbuf, fileclosed, s->flush_raw_data)) { + if (!verify_dset(ds->implicit_did, s->filetype, ds->implicit_sid, mem_sid, start, stride, count, + block, vbuf, fileclosed, s->flush_raw_data, s->reg_ref ? &ds->reg_buf[2] : NULL)) { HDprintf("verify_dset() to chunked dataset: implicit index dataset failed\n"); TEST_ERROR; } - if (!verify_dset(ds->fa_did, s->filetype, mem_sid, ds->fa_sid, start, stride, count, block, vbuf, - fileclosed, s->flush_raw_data)) { + if (!verify_dset(ds->fa_did, s->filetype, ds->fa_sid, mem_sid, start, stride, count, block, vbuf, + fileclosed, s->flush_raw_data, s->reg_ref ? &ds->reg_buf[3] : NULL)) { HDprintf("verify_dset() to chunked dataset: fa index dataset failed\n"); TEST_ERROR; } - if (!verify_dset(ds->ea_did, s->filetype, mem_sid, ds->ea_sid, start, stride, count, block, vbuf, - fileclosed, s->flush_raw_data)) { + if (!verify_dset(ds->ea_did, s->filetype, ds->ea_sid, mem_sid, start, stride, count, block, vbuf, + fileclosed, s->flush_raw_data, s->reg_ref ? &ds->reg_buf[4] : NULL)) { HDprintf("verify_dset() to chunked dataset: ea index dataset failed\n"); TEST_ERROR; } - if (!verify_dset(ds->bt2_did, s->filetype, mem_sid, ds->bt2_sid, start, stride, count, block, vbuf, - fileclosed, s->flush_raw_data)) { + if (!verify_dset(ds->bt2_did, s->filetype, ds->bt2_sid, mem_sid, start, stride, count, block, vbuf, + fileclosed, s->flush_raw_data, s->reg_ref ? &ds->reg_buf[5] : NULL)) { HDprintf("verify_dset() to chunked dataset: bt2 index dataset failed\n"); TEST_ERROR; } @@ -1432,6 +1741,14 @@ verify_dsets_action(unsigned action, const state_t *s, const dsets_state_t *ds, if (vbuf) HDfree(vbuf); + if (s->reg_ref) { + for (i = 0; i < 6; i++) + if (H5Rdestroy(&ds->reg_buf[i]) < 0) { + HDprintf("H5Rdestroy failed\n"); + TEST_ERROR; + } + } + return true; error: @@ -1447,8 +1764,8 @@ error: * `vbuf` contains the data expected from the read. */ static bool -verify_dset(hid_t did, hid_t tid, hid_t mem_sid, hid_t file_sid, hsize_t *start, hsize_t *stride, - hsize_t *count, hsize_t *block, unsigned int *vbuf, bool fileclosed, bool flush_raw_data) +verify_dset(hid_t did, hid_t tid, hid_t sid, hid_t mem_sid, hsize_t *start, hsize_t *stride, hsize_t *count, + hsize_t *block, unsigned int *vbuf, bool fileclosed, bool flush_raw_data, H5R_ref_t *reg_buf) { unsigned int *rbuf = NULL; unsigned i; @@ -1465,14 +1782,32 @@ verify_dset(hid_t did, hid_t tid, hid_t mem_sid, hid_t file_sid, hsize_t *start, TEST_ERROR; } - /* Make the selection the file dataspace */ - if (H5Sselect_hyperslab(file_sid, H5S_SELECT_SET, start, stride, count, block) < 0) { - HDprintf("H5Sselect to dataset failed\n"); - TEST_ERROR; + /* If region reference is enabled, obtain the dataset selection from the reference buffer */ + if (reg_buf != NULL) { /* Imply region reference is enabled */ + hid_t temp_sid = badhid; + + H5E_BEGIN_TRY + { + temp_sid = H5Ropen_region(reg_buf, H5P_DEFAULT, H5P_DEFAULT); + } + H5E_END_TRY; + + if (temp_sid < 0) { + HDprintf("H5Ropen_region failed\n"); + TEST_ERROR; + } + sid = temp_sid; + } + else { + /* Make the selection the file dataspace */ + if (H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, stride, count, block) < 0) { + HDprintf("H5Sselect_hyperslab failed\n"); + TEST_ERROR; + } } /* Read the data from the dataset into `rbuf` */ - if (H5Dread(did, tid, mem_sid, file_sid, H5P_DEFAULT, rbuf) < 0) { + if (H5Dread(did, tid, mem_sid, sid, H5P_DEFAULT, rbuf) < 0) { HDprintf("H5Dread from dataset failed\n"); TEST_ERROR; } diff --git a/test/vfd_swmr_generator.c b/test/vfd_swmr_generator.c index 1c549ca..e11d92f 100644 --- a/test/vfd_swmr_generator.c +++ b/test/vfd_swmr_generator.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/vfd_swmr_gfail_writer.c b/test/vfd_swmr_gfail_writer.c index aef39a0..84a9e80 100644 --- a/test/vfd_swmr_gfail_writer.c +++ b/test/vfd_swmr_gfail_writer.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/vfd_swmr_gperf_writer.c b/test/vfd_swmr_gperf_writer.c index 1ca4483..8632bf8 100644 --- a/test/vfd_swmr_gperf_writer.c +++ b/test/vfd_swmr_gperf_writer.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index eb7bb0e..3dea01d 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/vfd_swmr_indep_rw_writer.c b/test/vfd_swmr_indep_rw_writer.c index f452e89..75da581 100644 --- a/test/vfd_swmr_indep_rw_writer.c +++ b/test/vfd_swmr_indep_rw_writer.c @@ -1,15 +1,15 @@ -/* - * Copyright by The HDF Group. - * Copyright by the Board of Trustees of the University of Illinois. - * All rights reserved. - * - * This file is part of HDF5. The full HDF5 copyright notice, including - * terms governing use, modification, and redistribution, is contained in - * the COPYING file, which can be found at the root of the source code - * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. - * If you do not have access to either file, you may request a copy from - * help@hdfgroup.org. - */ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* This program checks independence of writer and reader for VFD SWMR. * A writer can also be a reader of another writer. A reader can also be a diff --git a/test/vfd_swmr_reader.c b/test/vfd_swmr_reader.c index d6a87dd..c04182c 100644 --- a/test/vfd_swmr_reader.c +++ b/test/vfd_swmr_reader.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/vfd_swmr_remove_reader.c b/test/vfd_swmr_remove_reader.c index bc00756..ed2de5f 100644 --- a/test/vfd_swmr_remove_reader.c +++ b/test/vfd_swmr_remove_reader.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/vfd_swmr_remove_writer.c b/test/vfd_swmr_remove_writer.c index 6b0c13f..a35d303 100644 --- a/test/vfd_swmr_remove_writer.c +++ b/test/vfd_swmr_remove_writer.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/vfd_swmr_sparse_reader.c b/test/vfd_swmr_sparse_reader.c index 6a7ba2f..d0a03a8 100644 --- a/test/vfd_swmr_sparse_reader.c +++ b/test/vfd_swmr_sparse_reader.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/vfd_swmr_sparse_writer.c b/test/vfd_swmr_sparse_writer.c index f78bf8b..2b976d3 100644 --- a/test/vfd_swmr_sparse_writer.c +++ b/test/vfd_swmr_sparse_writer.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/vfd_swmr_vlstr_reader.c b/test/vfd_swmr_vlstr_reader.c index 2554207..4107bfc 100644 --- a/test/vfd_swmr_vlstr_reader.c +++ b/test/vfd_swmr_vlstr_reader.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/vfd_swmr_vlstr_writer.c b/test/vfd_swmr_vlstr_writer.c index e3905e8..8fc3f95 100644 --- a/test/vfd_swmr_vlstr_writer.c +++ b/test/vfd_swmr_vlstr_writer.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/vfd_swmr_writer.c b/test/vfd_swmr_writer.c index a630532..db0c90b 100644 --- a/test/vfd_swmr_writer.c +++ b/test/vfd_swmr_writer.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/test/vfd_swmr_zoo_writer.c b/test/vfd_swmr_zoo_writer.c index d40f2d9..97c4db3 100644 --- a/test/vfd_swmr_zoo_writer.c +++ b/test/vfd_swmr_zoo_writer.c @@ -1,5 +1,6 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * + * Copyright by Akadio, Inc. * * All rights reserved. * * * * This file is part of HDF5. The full HDF5 copyright notice, including * diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 2d5626e..d6a5afd 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -2,3 +2,4 @@ cmake_minimum_required (VERSION 3.10) project (HDF5_UTILS C) add_subdirectory (mirror_vfd) +add_subdirectory (vfd_swmr) diff --git a/utils/Makefile.am b/utils/Makefile.am index 288da37..fa877ef 100644 --- a/utils/Makefile.am +++ b/utils/Makefile.am @@ -21,6 +21,6 @@ include $(top_srcdir)/config/commence.am CONFIG=ordered # All subdirectories -SUBDIRS=mirror_vfd +SUBDIRS=mirror_vfd vfd_swmr include $(top_srcdir)/config/conclude.am diff --git a/utils/vfd_swmr b/utils/vfd_swmr -Subproject 5f4b020713f3194222311b2e87b29f0de0722ed +Subproject 121da9672cebea43dca05c89adaae5d05ba56e4 |